Initially I just had the audio playing as a single track along side the video, but quickly realized that processing video frames takes much more work than processing audio. So therefore, running into frame drift where the audio would slowly get out of sync by the end creating a disorienting and horrible experience.
I needed to do something where I could take all of the individual clips that I was already lining up within my DAW so they would just sync with the timecode of the video. That way, if frames were taking just a fraction longer to render, all sound effects would be perceivably in sync. Not to mention, this gave me a chance to throw in some random audio to liven up "repetitive" sounds making a slightly different experience every time. In the end, I had more control over the final mix and better experience over all.
From the start, I knew that I needed a list of objects that could be collected and understood in one place so I could easily sequence them. I was also seeing a lot on scriptable objects with their ability to store data and functionality. The result was AudioObject.cs. A simple scriptable object script, but I needed it to have different functionality for different child classes. I knew I would be using inheritance, but what I didn’t know at the time was I was stumbling upon something called dependency injection, or at least something like it.
[Serializable]
public class AudioObject : ScriptableObject
{
public bool interupt = false;
public AudioSource source;
public Coroutine coroutine;
//having override-able functions allows for some dependency injection
public virtual IEnumerator Play()
{
yield return null;
}
public virtual void Stop()
{
}
}So from here I was able to make a handful of scripts that overrode the Play() and Stop() functions for custom purposes.
[Serializable]
[CreateAssetMenu(fileName = "AudioSequence", menuName = "AudioObjects/CreateAudioSequence", order = 1)]
public class AudioSequence : AudioObject //inherits from AudioObject
{
public List<m_AudioClip> clipQueue;
public override IEnumerator Play()
{
string sSourceName = string.Format("{0} (1)", source.name);
for (int i = 0; i < clipQueue.Count; i++)
{
source.clip = clipQueue[i].clip;
source.Play();
source.loop = clipQueue[i].loop;
yield return new WaitUntil(() => source.time == clipQueue[i].clip.length); //not complete until clip is through
}
yield break;
}
public override void Stop()
{
if(source != null) source.Stop();
}
}[Serializable]
[CreateAssetMenu(fileName = "AudioRandom", menuName = "AudioObjects/CreateAudioRandom", order = 1)]
public class AudioRandom : AudioObject
{
public List<AudioClip> clips;
public override IEnumerator Play()
{
source.clip = RandomClip();
source.Play();
yield break;
}
public override void Stop()
{
source.Stop();
}
public AudioClip RandomClip() //returns random clip from list
{
int rando = UnityEngine.Random.Range(0, clips.Count);
return clips[rando];
}
}[Serializable]
[CreateAssetMenu(fileName = "AudioFadeIn", menuName = "AudioObjects/CreateAudioFadeIn", order = 1)]
public class AudioFadeIn : AudioObject
{
public float fadeSpeed = 1;
public override IEnumerator Play()
{
//tell source to fade in at fadeSpeed
while (source.volume < 1f)
{
source.volume += Time.deltaTime * fadeSpeed;
yield return null;
}
yield break;
}
}[Serializable]
[CreateAssetMenu(fileName = "AudioFadeOut", menuName = "AudioObjects/CreateAudioFadeOut", order = 1)]
public class AudioFadeOut : AudioObject
{
public float fadeSpeed = 1.5f;
public override IEnumerator Play()
{
//tell source to fade out
while (source.volume > 0f)
{
source.volume -= Time.deltaTime * fadeSpeed;
yield return null;
}
yield break;
}
}These last two (above) were nice, because I didn’t have to necessarily play a clip, but I could throw these into the sequence and modify a specific audio source for some simple fading functions.
Below is how it all got implemented. I had a few items that would also be in a list, so I would have nested lists that I could populate within the inspector. The main part of this class sorts a list and loops through them all and activating them based on the current timecode of the playing video.
public class AudioPlaybackControl : MonoBehaviour
{
public List<AudioClip> clips;
public MediaPlayerCtrl media;
public GameObject soundPrefab;
public string fileName;
public List<AudioEventContainer> audioEvents;
public AudioMixer mixer;
public List<AudioMixerGroup> mixerGroups;
public List<AudioSource> activeSources;
public AudioEventObject EventObject(AudioObject obj, AudioSource source)
{
return new AudioEventObject(obj, source);
}
// Use this for initialization
void Start () {
//access all mixer groups under master bus
mixerGroups = mixer.FindMatchingGroups("Master").ToList();
clips = new List<AudioClip>();
for (int i = 0; i < audioEvents.Count; i++)
{
for(int j = 0; j< audioEvents[i].events.Count; j++)
{
//All events are made within the inspector so this checks if anyone is missing important data and where it's missing
if (audioEvents[i].events[j].audio == null)
{
Debug.LogErrorFormat("Event : {0} is missing an AudioObject on subevent : {1}", i, j);
}
if (audioEvents[i].events[j].source == null)
{
Debug.LogErrorFormat("Event : {0} is missing an AudioSource on subevent : {1}", i, j);
}
}
}
SortAudioEvents();//puts all events into chronological order in case any were added out of order.
StartCoroutine(AudioSequence());//run the sequence
}
//Sort audio events by point in timeline to be in chronological order
public void SortAudioEvents()//puts all events into chronological order in case any were added out of order.
{
audioEvents.Sort(delegate (AudioEventContainer x, AudioEventContainer y)
{
if (x.audioTime == 0 && y.audioTime == 0) return 0;
else if (x.audioTime == 0) return -1;
else if (y.audioTime == 0) return 1;
else return x.audioTime.CompareTo(y.audioTime);
});
}
//filler coroutine so that length is longer than zero avoiding out of range exceptions
private IEnumerator Temp()
{
yield return new WaitUntil(() => media.m_frameTime > audioEvents[0].audioTime);
Debug.Log("This will happen if wait takes Too Long");
yield break;
}
//Check if event needs to be interupted (i.e. a looping sfx)
private bool DoInteruptEvent(int routineId, int eventsId, List<CoroutineEventsContainer> routineEvents)
{
return (routineEvents[routineId].audioSource == audioEvents[eventsId].events[0].source && audioEvents[eventsId].events[0].audio.coroutine != null && audioEvents[eventsId].events[0].audio.interupt == true);
}
//coroutine for running all audio events in sequence based on their timeline
private IEnumerator AudioSequence()
{
while (true)
{
yield return new WaitUntil(() => media.m_strFileName == fileName);//wait for correct video before beginning
AudioSource preSource = audioEvents[0].events[0].source;
List<Coroutine> routines = new List<Coroutine>();
routines.Add(StartCoroutine(Temp())); //filler to avoid out of range exceptions
routines.Add(StartCoroutine(Temp())); //filler to avoid out of range exceptions
List<CoroutineEventsContainer> routineEvents = new List<CoroutineEventsContainer>(); //stores last source and last coroutines ran for potential stopping of events
// go through audio events
for (int i = 0; i < audioEvents.Count; i++)
{
routineEvents.Add(new CoroutineEventsContainer(routines, preSource));
//wait for current event's point in timeline to run
yield return new WaitUntil(() => media.m_frameTime > audioEvents[i].audioTime);
for (int j = 0; j < routineEvents.Count; j++)
{
if (DoInteruptEvent(i, j, routineEvents) == true;) //Check if event needs to be interupted (i.e. a looping sfx)
{
//clear any corutines
foreach (Coroutine coroutine in routineEvents[j].coroutines)
{
StopCoroutine(coroutine);
}
//stop any playing audio.
foreach (AudioEventObject eventObject in audioEvents[i].events)
{
eventObject.audio.Stop();
}
}
}
//add their sources to play from
audioEvents[i].FillSources();
preSource = audioEvents[i].events[0].source;
foreach (AudioEventObject eventObject in audioEvents[i].events)
{
routines.Add(StartCoroutine(eventObject.audio.Play()));
}
}
yield return new WaitUntil(() => media.m_strFileName != fileName);// wait for new video to be loaded to clear events
routineEvents.Clear();
yield return null;
}
}This is how it all looked within the inspector:
In conclusion, I was able to have a usable interface and place audio on a timeline similar to a DAW that was organized and editable. Unfortunately each list item is labeled “Element X.” If I were to do things differently, I would probably give it a better editor UI so that it at least is labeled accurately or even better, develop a UI window that is more like a DAW where I could bring in clips and see them represented visually.