Unity JSON with mixed media binary objects like audioStream buffers

You may want to provide more information specific to your case.

The solution depends on whether you are:

1. Going to serialize that data yourself and then deserialize them.

You could use scriptable objects to store audio, sprite, prefab and in general visual-centric data. Then create an editor extension for that particular scriptable object type to expose and edit the JSON data and store them in a .json file in project's assets.

2. You already have everything serialized in JSON and just need a way to deserialize them.

In this case, you should probably create the Data class to hold those data, with the serializable types in mind. Then create those data and try to import the stream byte array to either an audio file in the file system or an audio clip in memory.

Here is an example that caches the file in a directory:

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;

using UnityEngine;

using Newtonsoft.Json; // JSON .NET For Unity
using Newtonsoft.Json.Serialization;

[Serializable]
public class MovieModel
{
    [Serializable]
    public class SlotsData
    {
        public string Celebrity { get; set; }
        public string Genre { get; set; }
    }

    [Serializable]
    public class AudioStreamData
    {
        public string Type { get; set; }
        public byte[] Data { get; set; }
    }

    public string UserId { get; set; }
    public string ContentType { get; set; }
    public string IntentName { get; set; }
    public SlotsData Slots { get; set; }
    public string Message { get; set; }
    public string DialogState { get; set; }
    public AudioStreamData AudioStream { get; set; }

    public override string ToString()
    {
        return string.Format("[{0}, {1}, {2}, {3}]", UserId, ContentType, IntentName, Message);
    }
}

public class MovieWithAudioClip
{
    public MovieModel Model { get; set; }
    public string CachedFileName { get; set; }
    public AudioClip Clip { get; set; }
}

public class AudioClipJSONImporter : MonoBehaviour
{
    private static readonly JsonSerializerSettings SERIALIZATION_SETTINGS = new JsonSerializerSettings()
    {
        // Read the docs to configure the settings based on your data classes and the JSON file itself.
        ContractResolver = new CamelCasePropertyNamesContractResolver()
    };

    private static readonly Dictionary<string, AudioType> WHITELISTED_CONTENT_TYPE_TO_UNITY_AUDIO_TYPE = new Dictionary<string, AudioType>()
    {
        // Append all the supported content types here with their corresponding type, so that Unity can read them.
        { "audio/mpeg", AudioType.MPEG}
    };
    private static readonly Dictionary<string, string> CONTENT_TYPE_TO_FILE_EXTENSION = new Dictionary<string, string>()
    {
        { "audio/mpeg", ".mp3"}
    };

    [Header("Drag and drop a JSON movie entry here")]
    [SerializeField]
    private TextAsset m_MovieEntryJson;

    [SerializeField]
    private string m_ClipCacheDirectory = "Clips";

    [Header("Drag and drop an Audio source here, to preview the current movie entry")]
    [SerializeField] AudioSource m_AudioPreviewer;

    // Click on the top right of the script when in edit mode to call this procedure.
    [ContextMenu("Import JSON entry")]
    private void ImportJsonEntry()
    {
        if (m_MovieEntryJson == null || string.IsNullOrEmpty(m_MovieEntryJson.text))
        {
            Debug.LogError("Drag and drop a JSON movie entry in the inspector.");
            return;
        }

        MovieModel movieModel = JsonConvert.DeserializeObject<MovieModel>(m_MovieEntryJson.text, SERIALIZATION_SETTINGS);

        Debug.LogFormat("Movie entry {0} imported.", movieModel);

        Debug.Assert(movieModel != null, "Failed to load movie entry.");
        Debug.AssertFormat(movieModel.AudioStream != null, "Failed to load audio stream for movie entry {0}", movieModel);
        Debug.AssertFormat(movieModel.AudioStream.Data != null, "Failed to load audio stream byte array for movie entry {0}", movieModel);
        if (movieModel == null || movieModel.AudioStream == null || movieModel.AudioStream.Data == null)
        {
            return;
        }

        string clipCacheDirName = Application.isPlaying ? Application.persistentDataPath : Application.streamingAssetsPath;
        if (!string.IsNullOrEmpty(m_ClipCacheDirectory))
        {
            clipCacheDirName = Path.Combine(clipCacheDirName, m_ClipCacheDirectory);
        }

        AudioType supportedAudioType;
        string fileExtension = null;
        if (!WHITELISTED_CONTENT_TYPE_TO_UNITY_AUDIO_TYPE.TryGetValue(movieModel.ContentType, out supportedAudioType))
        {
            Debug.LogErrorFormat(
                "Failed to load movie {0} with mime type: {1} as it is not in the mime type to extension whitelist.",
                movieModel, movieModel.ContentType
            );

            return;
        }
        CONTENT_TYPE_TO_FILE_EXTENSION.TryGetValue(movieModel.ContentType, out fileExtension);

        StartCoroutine(
            GenerateAudioMovie(clipCacheDirName, fileExtension, supportedAudioType, movieModel, (MovieWithAudioClip movie) =>
            {
                if (m_AudioPreviewer != null)
                {
                    m_AudioPreviewer.clip = movie.Clip;
                    m_AudioPreviewer.Play();
                }
            })
        );
    }

    private IEnumerator GenerateAudioMovie(
        string rootDirName,
        string fileExtension,
        AudioType audioType,
        MovieModel movieModel,
        Action<MovieWithAudioClip> onDone,
        Action<string> onError = null
    )
    {
        // Remove this is you can be sure the directory exists.
        Directory.CreateDirectory(rootDirName);

        // If you can create a non random ID based on the JSON data, that is better.
        //
        // Mainly, because you can check the file system in case it has already been downloaded and load the clip directly.
        // Although, that makes sense only if you have a 'light' route to receive the movie data without the audio stream (which is now cached).
        string cachedFileId = Guid.NewGuid().ToString();
        string cachedFileName = Path.Combine(rootDirName, cachedFileId + fileExtension);
        MovieWithAudioClip audioMovie = new MovieWithAudioClip()
        {
            Model = movieModel,
            CachedFileName = cachedFileName
        };

        // Source: https://answers.unity.com/questions/686240/audioclip-oggmp3-loaded-from-byte-array.html
        //
        File.WriteAllBytes(cachedFileName, movieModel.AudioStream.Data);
        Debug.LogFormat("Movie audio file exported at: {0}", cachedFileName);

        WWW loader = new WWW(string.Format("file://{0}", cachedFileName));

        yield return loader;

        if (!System.String.IsNullOrEmpty(loader.error))
        {
            Debug.LogErrorFormat("Failed to load movie {0} at file {1} with error {2}.", movieModel, cachedFileName, loader.error);

            if (onError != null)
                onError(loader.error);
        }
        else
        {
            audioMovie.Clip = loader.GetAudioClip(false, false, audioType);
            Debug.AssertFormat(audioMovie.Clip != null, "Failed to generate audio clip for movie entry {0}", movieModel);

            if (audioMovie.Clip != null)
            {
                if (onDone != null)
                    onDone(audioMovie);
            }
            else
            {
                if (onError != null)
                    onError(loader.error);
            }
        }
    }
}

The code above does not do any processing, it just tries to retrieve an audio clip out of the stream in the JSON and it fails for mp3. However, an audio file will be created in your file system and you can play it and make sure that the JSON parser works.

From there, you need to process the data based on what types you are going to support. Here is a relevant post. Check out the three steps of Kurt-Dekker's answer.

Now, it's up to you to handle the different audio types you want to support, platform compatibility .etc.

Good luck!