From d9f9b4e7f199cf22a14317942e5bf60f1c5e8674 Mon Sep 17 00:00:00 2001 From: Anton Franzluebbers Date: Mon, 19 Dec 2022 23:01:37 -0500 Subject: [PATCH] added auto-start option for microphone, formatting and documentation changes --- .../VelNetUnity/Runtime/Util/VelVoice.cs | 477 +++++++++--------- .../Runtime/Util/VelVoicePlayer.cs | 70 ++- .../Packages/VelNetUnity/package.json | 53 +- TestVelGameServer/Packages/manifest.json | 9 +- TestVelGameServer/Packages/packages-lock.json | 20 +- 5 files changed, 308 insertions(+), 321 deletions(-) diff --git a/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs b/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs index 8ddcd7d..733b0b7 100644 --- a/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs +++ b/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs @@ -1,262 +1,265 @@ -using System.Collections; using System.Collections.Generic; using UnityEngine; -using System.IO; using Concentus.Structs; using System.Threading; using System; +using System.Linq; + namespace VelNet { - public class VelVoice : MonoBehaviour - { - public class FixedArray - { + public class VelVoice : MonoBehaviour + { + public class FixedArray + { + public readonly byte[] array; + public int count; - public byte[] array; - public int count; + public FixedArray(int max) + { + array = new byte[max]; + count = 0; + } + } - public FixedArray(int max) - { - array = new byte[max]; - count = 0; - } - } - OpusEncoder opusEncoder; - OpusDecoder opusDecoder; - //StreamWriter sw; - AudioClip clip; - float[] tempData; - float[] encoderBuffer; - List frameBuffer; + private OpusEncoder opusEncoder; + private OpusDecoder opusDecoder; - List sendQueue = new List(); - List encoderArrayPool = new List(); - List decoderArrayPool = new List(); - int lastUsedEncoderPool = 0; - int lastUsedDecoderPool = 0; - int encoderBufferIndex = 0; - int size = 0; - int lastPosition = 0; - string device = ""; - int encoder_frame_size = 640; - double micSampleTime; - int opusFreq = 16000; - double encodeTime = 1 / (double)16000;//16000.0; - double lastMicSample; //holds the last mic sample, in case we need to interpolate it - double sampleTimer = 0; //increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time - EventWaitHandle waiter; - public float silenceThreshold = .01f; //average volume of packet - int numSilent = 0; //number of silent packets detected - public int minSilencePacketsToStop = 5; - double averageVolume = 0; - Thread t; - public Action encodedFrameAvailable = delegate { }; + //StreamWriter sw; + private AudioClip clip; + private float[] tempData; + private float[] encoderBuffer; + private List frameBuffer; - // Start is called before the first frame update - void Start() - { - opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP); - opusDecoder = new OpusDecoder(opusFreq, 1); - encoderBuffer = new float[opusFreq]; - frameBuffer = new List(); - //string path = Application.persistentDataPath + "/" + "mic.csv"; //this was for writing mic samples - //sw = new StreamWriter(path, false); + private readonly List sendQueue = new List(); + private readonly List encoderArrayPool = new List(); + private readonly List decoderArrayPool = new List(); + private int lastUsedEncoderPool; + private int lastUsedDecoderPool; + private int encoderBufferIndex; + private int lastPosition; + private string device = ""; + private const int encoderFrameSize = 640; + private double micSampleTime; + private const int opusFreq = 16000; + private const double encodeTime = 1 / (double)16000; + + /// + /// holds the last mic sample, in case we need to interpolate it + /// + private double lastMicSample; + + /// + /// increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time + /// + private double sampleTimer; + + private EventWaitHandle waiter; + + /// + /// average volume of packet + /// + public float silenceThreshold = .01f; + + /// + /// number of silent packets detected + /// + private int numSilent; + + public int minSilencePacketsToStop = 5; + private double averageVolume; + private Thread t; + public Action encodedFrameAvailable = delegate { }; + + public bool autostartMicrophone = true; + + private void Start() + { + opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP); + opusDecoder = new OpusDecoder(opusFreq, 1); + encoderBuffer = new float[opusFreq]; + frameBuffer = new List(); + + // pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2) + for (int i = 0; i < 100; i++) + { + encoderArrayPool.Add(new float[encoderFrameSize]); + decoderArrayPool.Add(new FixedArray(encoderFrameSize)); + } + + t = new Thread(EncodeThread); + waiter = new EventWaitHandle(true, EventResetMode.AutoReset); + t.Start(); + + if (autostartMicrophone) + { + StartMicrophone(Microphone.devices.FirstOrDefault()); + } + } + + public void StartMicrophone(string micDeviceName) + { + Debug.Log("Starting with microphone: " + micDeviceName); + if (micDeviceName == null) return; + device = micDeviceName; + Microphone.GetDeviceCaps(device, out int minFreq, out int maxFreq); + Debug.Log("Freq: " + minFreq + ":" + maxFreq); + clip = Microphone.Start(device, true, 10, 48000); + micSampleTime = 1.0 / clip.frequency; + + Debug.Log("Frequency:" + clip.frequency); + tempData = new float[clip.samples * clip.channels]; + Debug.Log("channels: " + clip.channels); + } + + private void OnApplicationQuit() + { + t.Abort(); + + //sw.Flush(); + //sw.Close(); + } + + private float[] GetNextEncoderPool() + { + lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count; + return encoderArrayPool[lastUsedEncoderPool]; + } + + private FixedArray GetNextDecoderPool() + { + lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count; + + FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool]; + toReturn.count = 0; + return toReturn; + } + + // Update is called once per frame + private void Update() + { + if (clip != null) + { + int micPosition = Microphone.GetPosition(device); + if (micPosition == lastPosition) + { + return; //sometimes the microphone will not advance + } + + int numSamples; + float[] temp; + if (micPosition > lastPosition) + { + numSamples = micPosition - lastPosition; + } + else + { + //whatever was left + numSamples = (tempData.Length - lastPosition) + micPosition; + } + + // this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic) + temp = new float[numSamples]; + clip.GetData(temp, lastPosition); + lastPosition = micPosition; + // this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets + // iterate through temp, which contains that mic samples at 44.1khz + foreach (float sample in temp) + { + sampleTimer += micSampleTime; + if (sampleTimer > encodeTime) + { + //take a sample between the last sample and the current sample + + double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is + double t = diff / micSampleTime; //this should be between 0 and 1 + double v = lastMicSample * (1 - t) + sample * t; + sampleTimer -= encodeTime; + + encoderBuffer[encoderBufferIndex++] = (float)v; + averageVolume += v > 0 ? v : -v; + if (encoderBufferIndex > encoderFrameSize) //this is when a new packet gets created + { + averageVolume = averageVolume / encoderFrameSize; + + if (averageVolume < silenceThreshold) + { + numSilent++; + } + else + { + numSilent = 0; + } + + averageVolume = 0; + + if (numSilent < minSilencePacketsToStop) + { + float[] frame = GetNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array + //lock the frame buffer + + System.Array.Copy(encoderBuffer, frame, encoderFrameSize); //nice and fast - for (int i = 0; i < 100; i++) //pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2) - { - encoderArrayPool.Add(new float[encoder_frame_size]); - decoderArrayPool.Add(new FixedArray(encoder_frame_size)); + lock (frameBuffer) + { + frameBuffer.Add(frame); + waiter.Set(); //signal the encode frame + } + } - } + encoderBufferIndex = 0; + } + } - t = new Thread(encodeThread); - waiter = new EventWaitHandle(true, EventResetMode.AutoReset); - t.Start(); - } + lastMicSample = sample; //remember the last sample, just in case this is the first one next time + } + } - public void startMicrophone(string mic) - { - Debug.Log(mic); - device = mic; - int minFreq, maxFreq; - Microphone.GetDeviceCaps(device, out minFreq, out maxFreq); - Debug.Log("Freq: " + minFreq + ":" + maxFreq); - clip = Microphone.Start(device, true, 10, 48000); - micSampleTime = 1.0 / clip.frequency; + lock (sendQueue) + { + foreach (FixedArray f in sendQueue) + { + encodedFrameAvailable(f); + } - Debug.Log("Frequency:" + clip.frequency); - tempData = new float[clip.samples * clip.channels]; - Debug.Log("channels: " + clip.channels); - } + sendQueue.Clear(); + } + } - private void OnApplicationQuit() - { - t.Abort(); + public float[] DecodeOpusData(byte[] data, int count) + { + float[] t = GetNextEncoderPool(); + opusDecoder.Decode(data, 0, count, t, 0, encoderFrameSize); + return t; + } - //sw.Flush(); - //sw.Close(); - - } - - float[] getNextEncoderPool() - { - lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count; - return encoderArrayPool[lastUsedEncoderPool]; - } - - FixedArray getNextDecoderPool() - { - lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count; - - FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool]; - toReturn.count = 0; - return toReturn; - } - // Update is called once per frame - void Update() - { - - if (clip != null) - { - int micPosition = Microphone.GetPosition(device); - if (micPosition == lastPosition) - { - return; //sometimes the microphone will not advance - } - int numSamples = 0; - float[] temp; - if (micPosition > lastPosition) - { - numSamples = micPosition - lastPosition; - } - else - { - //whatever was left - numSamples = (tempData.Length - lastPosition) + micPosition; - } + private void EncodeThread() + { + while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal + { + List toEncode = new List(); - //Debug.Log(micPosition); - temp = new float[numSamples]; //this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic) - clip.GetData(temp, lastPosition); - lastPosition = micPosition; + lock (frameBuffer) + { + toEncode.AddRange(frameBuffer); + frameBuffer.Clear(); + } - - //this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets - for (int i = 0; i < temp.Length; i++) //iterate through temp, which contans that mic samples at 44.1khz - { - sampleTimer += micSampleTime; - if (sampleTimer > encodeTime) - { - - //take a sample between the last sample and the current sample - - double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is - double t = diff / micSampleTime; //this should be between 0 and 1 - double v = lastMicSample * (1 - t) + temp[i] * t; - sampleTimer -= encodeTime; - - encoderBuffer[encoderBufferIndex++] = (float)v; - averageVolume += v > 0 ? v : -v; - if (encoderBufferIndex > encoder_frame_size) //this is when a new packet gets created - { - - - - averageVolume = averageVolume / encoder_frame_size; - - if (averageVolume < silenceThreshold) - { - numSilent++; - } - else - { - numSilent = 0; - } - averageVolume = 0; - - if (numSilent < minSilencePacketsToStop) - { - - float[] frame = getNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array - //lock the frame buffer - - System.Array.Copy(encoderBuffer, frame, encoder_frame_size); //nice and fast - - - lock (frameBuffer) - { - - frameBuffer.Add(frame); - waiter.Set(); //signal the encode frame - } - } - encoderBufferIndex = 0; - - } - } - lastMicSample = temp[i]; //remember the last sample, just in case this is the first one next time - } - } - - lock (sendQueue) - { - foreach (FixedArray f in sendQueue) - { - encodedFrameAvailable(f); - - } - sendQueue.Clear(); - } - - } - - public float[] decodeOpusData(byte[] data, int count) - { - float[] t = getNextEncoderPool(); - opusDecoder.Decode(data, 0, count, t, 0, encoder_frame_size); - return t; - } - - void encodeThread() - { - - while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal - { - - List toEncode = new List(); - - - lock (frameBuffer) - { - foreach (float[] frame in frameBuffer) - { - toEncode.Add(frame); - } - frameBuffer.Clear(); - } - - foreach (float[] frame in toEncode) - { - FixedArray a = getNextDecoderPool(); - int out_data_size = opusEncoder.Encode(frame, 0, encoder_frame_size, a.array, 0, a.array.Length); - a.count = out_data_size; - //add frame to the send buffer - lock (sendQueue) - { - sendQueue.Add(a); - } - } - - } - - } - - - - - } + foreach (float[] frame in toEncode) + { + FixedArray a = GetNextDecoderPool(); + int outDataSize = opusEncoder.Encode(frame, 0, encoderFrameSize, a.array, 0, a.array.Length); + a.count = outDataSize; + //add frame to the send buffer + lock (sendQueue) + { + sendQueue.Add(a); + } + } + } + } + } } \ No newline at end of file diff --git a/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs b/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs index 3b32fed..07eea1a 100644 --- a/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs +++ b/TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs @@ -1,24 +1,35 @@ -using System.Collections; -using System.Collections.Generic; using System.IO; using UnityEngine; + namespace VelNet { public class VelVoicePlayer : NetworkComponent { + /// + /// must be set for the player only + /// + public VelVoice voiceSystem; + + /// + /// must be set for the clone only + /// + public AudioSource source; + + private AudioClip myClip; + public int bufferedAmount; + public int playedAmount; + private int lastTime; + + /// + /// a buffer of 0s to force silence, because playing doesn't stop on demand + /// + private readonly float[] empty = new float[1000]; + + private float delayStartTime; - public VelVoice voiceSystem; //must be set for the player only - public AudioSource source; //must be set for the clone only - AudioClip myClip; - public int bufferedAmount = 0; - public int playedAmount = 0; - int lastTime = 0; - float[] empty = new float[1000]; //a buffer of 0s to force silence, because playing doesn't stop on demand - float delayStartTime; public override void ReceiveBytes(byte[] message) { - - float[] temp = voiceSystem.decodeOpusData(message, message.Length); + float[] temp = voiceSystem.DecodeOpusData(message, message.Length); myClip.SetData(temp, bufferedAmount % source.clip.samples); bufferedAmount += temp.Length; myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample) @@ -30,7 +41,7 @@ namespace VelNet } // Start is called before the first frame update - void Start() + private void Start() { voiceSystem = GameObject.FindObjectOfType(); if (voiceSystem == null) @@ -38,19 +49,17 @@ namespace VelNet Debug.LogError("No microphone found. Make sure you have one in the scene."); return; } + if (networkObject.IsMine) { voiceSystem.encodedFrameAvailable += (frame) => { - //float[] temp = new float[frame.count]; - //System.Array.Copy(frame.array, temp, frame.count); + //float[] temp = new float[frame.count]; + //System.Array.Copy(frame.array, temp, frame.count); MemoryStream mem = new MemoryStream(); BinaryWriter writer = new BinaryWriter(mem); writer.Write(frame.array, 0, frame.count); this.SendBytes(mem.ToArray(), false); - - - }; } @@ -58,26 +67,20 @@ namespace VelNet source.clip = myClip; source.loop = true; source.Pause(); - } - // Update is called once per frame - void Update() + private void Update() { - - - if (bufferedAmount > playedAmount) { - var offset = bufferedAmount - playedAmount; if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better { - var temp = Mathf.Max(0, offset - 2000); - source.pitch = Mathf.Min(2,1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind - + var temp = Mathf.Max(0, offset - 2000); + source.pitch = Mathf.Min(2, 1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind + if (!source.isPlaying) { @@ -86,7 +89,6 @@ namespace VelNet } else { - return; } } @@ -96,6 +98,7 @@ namespace VelNet source.Pause(); source.timeSamples = bufferedAmount % source.clip.samples; } + //Debug.Log(playedAmount); if (source.timeSamples >= lastTime) { @@ -103,18 +106,11 @@ namespace VelNet } else //repeated { - int total = source.clip.samples - lastTime + source.timeSamples; playedAmount += total; } + lastTime = source.timeSamples; - - } - - - - } - } \ No newline at end of file diff --git a/TestVelGameServer/Packages/VelNetUnity/package.json b/TestVelGameServer/Packages/VelNetUnity/package.json index be3358c..33fd65c 100644 --- a/TestVelGameServer/Packages/VelNetUnity/package.json +++ b/TestVelGameServer/Packages/VelNetUnity/package.json @@ -1,31 +1,28 @@ { - "name": "edu.uga.engr.vel.velnet", - "displayName": "VelNet", - "version": "1.1.4", - "unity": "2019.1", - "description": "A custom networking library for Unity.", - "keywords": [ - "networking", - "self-hosted" - ], - "author": { - "name": "Virtual Experiences Laboratory", - "email": "kjohnsen@uga.edu", - "url": "https://vel.engr.uga.edu" + "name": "edu.uga.engr.vel.velnet", + "displayName": "VelNet", + "version": "1.1.5", + "unity": "2019.1", + "description": "A custom networking library for Unity.", + "keywords": [ + "networking", + "self-hosted" + ], + "author": { + "name": "Virtual Experiences Laboratory", + "email": "kjohnsen@uga.edu", + "url": "https://vel.engr.uga.edu" + }, + "samples": [ + { + "displayName": "Example Dissonance", + "description": "Example Scene with Dissonance Integration Required", + "path": "Samples~/ExampleDissonance" }, - "samples": [ - { - "displayName": "Example Dissonance", - "description": "Example Scene with Dissonance Integration Required", - "path": "Samples~/ExampleDissonance" - }, - { - "displayName": "Example VEL Voice", - "description": "Example Scene using Built-in VEL Voice", - "path": "Samples~/ExampleVelVoice" - } - ], - "dependencies": { + { + "displayName": "Example VEL Voice", + "description": "Example Scene using Built-in VEL Voice", + "path": "Samples~/ExampleVelVoice" } - } - + ] +} \ No newline at end of file diff --git a/TestVelGameServer/Packages/manifest.json b/TestVelGameServer/Packages/manifest.json index 41b217c..3dcf57e 100644 --- a/TestVelGameServer/Packages/manifest.json +++ b/TestVelGameServer/Packages/manifest.json @@ -1,15 +1,14 @@ { "dependencies": { "com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git", - "com.unity.collab-proxy": "1.15.15", - "com.unity.ide.rider": "3.0.13", - "com.unity.ide.visualstudio": "2.0.15", + "com.unity.collab-proxy": "1.17.7", + "com.unity.ide.rider": "3.0.17", + "com.unity.ide.visualstudio": "2.0.17", "com.unity.ide.vscode": "1.2.5", - "com.unity.test-framework": "1.1.31", + "com.unity.test-framework": "1.3.2", "com.unity.textmeshpro": "3.0.6", "com.unity.timeline": "1.6.4", "com.unity.ugui": "1.0.0", - "edu.uga.engr.vel.velnet.dissonance": "file:S:/git_repo/VelNetDissonanceIntegration", "com.unity.modules.ai": "1.0.0", "com.unity.modules.androidjni": "1.0.0", "com.unity.modules.animation": "1.0.0", diff --git a/TestVelGameServer/Packages/packages-lock.json b/TestVelGameServer/Packages/packages-lock.json index c6630af..5686633 100644 --- a/TestVelGameServer/Packages/packages-lock.json +++ b/TestVelGameServer/Packages/packages-lock.json @@ -11,7 +11,7 @@ "hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d" }, "com.unity.collab-proxy": { - "version": "1.15.15", + "version": "1.17.7", "depth": 0, "source": "registry", "dependencies": { @@ -20,14 +20,14 @@ "url": "https://packages.unity.com" }, "com.unity.ext.nunit": { - "version": "1.0.6", + "version": "2.0.3", "depth": 1, "source": "registry", "dependencies": {}, "url": "https://packages.unity.com" }, "com.unity.ide.rider": { - "version": "3.0.13", + "version": "3.0.17", "depth": 0, "source": "registry", "dependencies": { @@ -36,7 +36,7 @@ "url": "https://packages.unity.com" }, "com.unity.ide.visualstudio": { - "version": "2.0.15", + "version": "2.0.17", "depth": 0, "source": "registry", "dependencies": { @@ -68,11 +68,11 @@ "url": "https://packages.unity.com" }, "com.unity.test-framework": { - "version": "1.1.31", + "version": "1.3.2", "depth": 0, "source": "registry", "dependencies": { - "com.unity.ext.nunit": "1.0.6", + "com.unity.ext.nunit": "2.0.3", "com.unity.modules.imgui": "1.0.0", "com.unity.modules.jsonserialize": "1.0.0" }, @@ -124,14 +124,6 @@ "source": "embedded", "dependencies": {} }, - "edu.uga.engr.vel.velnet.dissonance": { - "version": "file:S:/git_repo/VelNetDissonanceIntegration", - "depth": 0, - "source": "local", - "dependencies": { - "edu.uga.engr.vel.velnet": "1.1.0" - } - }, "com.unity.modules.ai": { "version": "1.0.0", "depth": 0,