vel
/
VelNet-Unity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added auto-start option for microphone, formatting and documentation changes

main
Anton Franzluebbers 2022-12-19 23:01:37 -05:00
parent 5ad76f69bc
commit d9f9b4e7f1
5 changed files with 308 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs
View File

@ -1,18 +1,17 @@
using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using UnityEngine; using UnityEngine;
using System.IO;
using Concentus.Structs; using Concentus.Structs;
using System.Threading; using System.Threading;
using System; using System;
using System.Linq;
namespace VelNet namespace VelNet
{ {
public class VelVoice : MonoBehaviour public class VelVoice : MonoBehaviour
{ {
public class FixedArray public class FixedArray
{ {
public readonly byte[] array;
public byte[] array;
public int count; public int count;
public FixedArray(int max) public FixedArray(int max)
@ -21,68 +20,88 @@ namespace VelNet
count = 0; count = 0;
} }
} }
OpusEncoder opusEncoder;
OpusDecoder opusDecoder;
//StreamWriter sw;
AudioClip clip;
float[] tempData;
float[] encoderBuffer;
List<float[]> frameBuffer;
List<FixedArray> sendQueue = new List<FixedArray>(); private OpusEncoder opusEncoder;
List<float[]> encoderArrayPool = new List<float[]>(); private OpusDecoder opusDecoder;
List<FixedArray> decoderArrayPool = new List<FixedArray>();
int lastUsedEncoderPool = 0; //StreamWriter sw;
int lastUsedDecoderPool = 0; private AudioClip clip;
int encoderBufferIndex = 0; private float[] tempData;
int size = 0; private float[] encoderBuffer;
int lastPosition = 0; private List<float[]> frameBuffer;
string device = "";
int encoder_frame_size = 640; private readonly List<FixedArray> sendQueue = new List<FixedArray>();
double micSampleTime; private readonly List<float[]> encoderArrayPool = new List<float[]>();
int opusFreq = 16000; private readonly List<FixedArray> decoderArrayPool = new List<FixedArray>();
double encodeTime = 1 / (double)16000;//16000.0; private int lastUsedEncoderPool;
double lastMicSample; //holds the last mic sample, in case we need to interpolate it private int lastUsedDecoderPool;
double sampleTimer = 0; //increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time private int encoderBufferIndex;
EventWaitHandle waiter; private int lastPosition;
public float silenceThreshold = .01f; //average volume of packet private string device = "";
int numSilent = 0; //number of silent packets detected private const int encoderFrameSize = 640;
private double micSampleTime;
private const int opusFreq = 16000;
private const double encodeTime = 1 / (double)16000;
/// <summary>
/// holds the last mic sample, in case we need to interpolate it
/// </summary>
private double lastMicSample;
/// <summary>
/// increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time
/// </summary>
private double sampleTimer;
private EventWaitHandle waiter;
/// <summary>
/// average volume of packet
/// </summary>
public float silenceThreshold = .01f;
/// <summary>
/// number of silent packets detected
/// </summary>
private int numSilent;
public int minSilencePacketsToStop = 5; public int minSilencePacketsToStop = 5;
double averageVolume = 0; private double averageVolume;
Thread t; private Thread t;
public Action<FixedArray> encodedFrameAvailable = delegate { }; public Action<FixedArray> encodedFrameAvailable = delegate { };
// Start is called before the first frame update public bool autostartMicrophone = true;
void Start()
private void Start()
{ {
opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP); opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP);
opusDecoder = new OpusDecoder(opusFreq, 1); opusDecoder = new OpusDecoder(opusFreq, 1);
encoderBuffer = new float[opusFreq]; encoderBuffer = new float[opusFreq];
frameBuffer = new List<float[]>(); frameBuffer = new List<float[]>();
//string path = Application.persistentDataPath + "/" + "mic.csv"; //this was for writing mic samples
//sw = new StreamWriter(path, false);
// pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2)
for (int i = 0; i < 100; i++)
for (int i = 0; i < 100; i++) //pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2)
{ {
encoderArrayPool.Add(new float[encoder_frame_size]); encoderArrayPool.Add(new float[encoderFrameSize]);
decoderArrayPool.Add(new FixedArray(encoder_frame_size)); decoderArrayPool.Add(new FixedArray(encoderFrameSize));
} }
t = new Thread(encodeThread); t = new Thread(EncodeThread);
waiter = new EventWaitHandle(true, EventResetMode.AutoReset); waiter = new EventWaitHandle(true, EventResetMode.AutoReset);
t.Start(); t.Start();
if (autostartMicrophone)
{
StartMicrophone(Microphone.devices.FirstOrDefault());
}
} }
public void startMicrophone(string mic) public void StartMicrophone(string micDeviceName)
{ {
Debug.Log(mic); Debug.Log("Starting with microphone: " + micDeviceName);
device = mic; if (micDeviceName == null) return;
int minFreq, maxFreq; device = micDeviceName;
Microphone.GetDeviceCaps(device, out minFreq, out maxFreq); Microphone.GetDeviceCaps(device, out int minFreq, out int maxFreq);
Debug.Log("Freq: " + minFreq + ":" + maxFreq); Debug.Log("Freq: " + minFreq + ":" + maxFreq);
clip = Microphone.Start(device, true, 10, 48000); clip = Microphone.Start(device, true, 10, 48000);
micSampleTime = 1.0 / clip.frequency; micSampleTime = 1.0 / clip.frequency;
@ -98,16 +117,15 @@ namespace VelNet
//sw.Flush(); //sw.Flush();
//sw.Close(); //sw.Close();
} }
float[] getNextEncoderPool() private float[] GetNextEncoderPool()
{ {
lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count; lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count;
return encoderArrayPool[lastUsedEncoderPool]; return encoderArrayPool[lastUsedEncoderPool];
} }
FixedArray getNextDecoderPool() private FixedArray GetNextDecoderPool()
{ {
lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count; lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count;
@ -115,10 +133,10 @@ namespace VelNet
toReturn.count = 0; toReturn.count = 0;
return toReturn; return toReturn;
} }
// Update is called once per frame
void Update()
{
// Update is called once per frame
private void Update()
{
if (clip != null) if (clip != null)
{ {
int micPosition = Microphone.GetPosition(device); int micPosition = Microphone.GetPosition(device);
@ -126,7 +144,8 @@ namespace VelNet
{ {
return; //sometimes the microphone will not advance return; //sometimes the microphone will not advance
} }
int numSamples = 0;
int numSamples;
float[] temp; float[] temp;
if (micPosition > lastPosition) if (micPosition > lastPosition)
{ {
@ -138,35 +157,31 @@ namespace VelNet
numSamples = (tempData.Length - lastPosition) + micPosition; numSamples = (tempData.Length - lastPosition) + micPosition;
} }
// this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic)
//Debug.Log(micPosition); temp = new float[numSamples];
temp = new float[numSamples]; //this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic)
clip.GetData(temp, lastPosition); clip.GetData(temp, lastPosition);
lastPosition = micPosition; lastPosition = micPosition;
// this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets // this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets
for (int i = 0; i < temp.Length; i++) //iterate through temp, which contans that mic samples at 44.1khz // iterate through temp, which contains that mic samples at 44.1khz
foreach (float sample in temp)
{ {
sampleTimer += micSampleTime; sampleTimer += micSampleTime;
if (sampleTimer > encodeTime) if (sampleTimer > encodeTime)
{ {
//take a sample between the last sample and the current sample //take a sample between the last sample and the current sample
double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is
double t = diff / micSampleTime; //this should be between 0 and 1 double t = diff / micSampleTime; //this should be between 0 and 1
double v = lastMicSample * (1 - t) + temp[i] * t; double v = lastMicSample * (1 - t) + sample * t;
sampleTimer -= encodeTime; sampleTimer -= encodeTime;
encoderBuffer[encoderBufferIndex++] = (float)v; encoderBuffer[encoderBufferIndex++] = (float)v;
averageVolume += v > 0 ? v : -v; averageVolume += v > 0 ? v : -v;
if (encoderBufferIndex > encoder_frame_size) //this is when a new packet gets created if (encoderBufferIndex > encoderFrameSize) //this is when a new packet gets created
{ {
averageVolume = averageVolume / encoderFrameSize;
averageVolume = averageVolume / encoder_frame_size;
if (averageVolume < silenceThreshold) if (averageVolume < silenceThreshold)
{ {
@ -176,29 +191,29 @@ namespace VelNet
{ {
numSilent = 0; numSilent = 0;
} }
averageVolume = 0; averageVolume = 0;
if (numSilent < minSilencePacketsToStop) if (numSilent < minSilencePacketsToStop)
{ {
float[] frame = GetNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
float[] frame = getNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
//lock the frame buffer //lock the frame buffer
System.Array.Copy(encoderBuffer, frame, encoder_frame_size); //nice and fast System.Array.Copy(encoderBuffer, frame, encoderFrameSize); //nice and fast
lock (frameBuffer) lock (frameBuffer)
{ {
frameBuffer.Add(frame); frameBuffer.Add(frame);
waiter.Set(); //signal the encode frame waiter.Set(); //signal the encode frame
} }
} }
encoderBufferIndex = 0;
encoderBufferIndex = 0;
} }
} }
lastMicSample = temp[i]; //remember the last sample, just in case this is the first one next time
lastMicSample = sample; //remember the last sample, just in case this is the first one next time
} }
} }
@ -207,56 +222,44 @@ namespace VelNet
foreach (FixedArray f in sendQueue) foreach (FixedArray f in sendQueue)
{ {
encodedFrameAvailable(f); encodedFrameAvailable(f);
} }
sendQueue.Clear(); sendQueue.Clear();
} }
} }
public float[] decodeOpusData(byte[] data, int count) public float[] DecodeOpusData(byte[] data, int count)
{ {
float[] t = getNextEncoderPool(); float[] t = GetNextEncoderPool();
opusDecoder.Decode(data, 0, count, t, 0, encoder_frame_size); opusDecoder.Decode(data, 0, count, t, 0, encoderFrameSize);
return t; return t;
} }
void encodeThread() private void EncodeThread()
{ {
while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal
{ {
List<float[]> toEncode = new List<float[]>(); List<float[]> toEncode = new List<float[]>();
lock (frameBuffer) lock (frameBuffer)
{ {
foreach (float[] frame in frameBuffer) toEncode.AddRange(frameBuffer);
{
toEncode.Add(frame);
}
frameBuffer.Clear(); frameBuffer.Clear();
} }
foreach (float[] frame in toEncode) foreach (float[] frame in toEncode)
{ {
FixedArray a = getNextDecoderPool(); FixedArray a = GetNextDecoderPool();
int out_data_size = opusEncoder.Encode(frame, 0, encoder_frame_size, a.array, 0, a.array.Length); int outDataSize = opusEncoder.Encode(frame, 0, encoderFrameSize, a.array, 0, a.array.Length);
a.count = out_data_size; a.count = outDataSize;
//add frame to the send buffer //add frame to the send buffer
lock (sendQueue) lock (sendQueue)
{ {
sendQueue.Add(a); sendQueue.Add(a);
} }
} }
}
} }
}
} }
} }

60
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs
View File

@ -1,24 +1,35 @@
using System.Collections;
using System.Collections.Generic;
using System.IO; using System.IO;
using UnityEngine; using UnityEngine;
namespace VelNet namespace VelNet
{ {
public class VelVoicePlayer : NetworkComponent public class VelVoicePlayer : NetworkComponent
{ {
/// <summary>
/// must be set for the player only
/// </summary>
public VelVoice voiceSystem;
/// <summary>
/// must be set for the clone only
/// </summary>
public AudioSource source;
private AudioClip myClip;
public int bufferedAmount;
public int playedAmount;
private int lastTime;
/// <summary>
/// a buffer of 0s to force silence, because playing doesn't stop on demand
/// </summary>
private readonly float[] empty = new float[1000];
private float delayStartTime;
public VelVoice voiceSystem; //must be set for the player only
public AudioSource source; //must be set for the clone only
AudioClip myClip;
public int bufferedAmount = 0;
public int playedAmount = 0;
int lastTime = 0;
float[] empty = new float[1000]; //a buffer of 0s to force silence, because playing doesn't stop on demand
float delayStartTime;
public override void ReceiveBytes(byte[] message) public override void ReceiveBytes(byte[] message)
{ {
float[] temp = voiceSystem.DecodeOpusData(message, message.Length);
float[] temp = voiceSystem.decodeOpusData(message, message.Length);
myClip.SetData(temp, bufferedAmount % source.clip.samples); myClip.SetData(temp, bufferedAmount % source.clip.samples);
bufferedAmount += temp.Length; bufferedAmount += temp.Length;
myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample) myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample)
@ -30,7 +41,7 @@ namespace VelNet
} }
// Start is called before the first frame update // Start is called before the first frame update
void Start() private void Start()
{ {
voiceSystem = GameObject.FindObjectOfType<VelVoice>(); voiceSystem = GameObject.FindObjectOfType<VelVoice>();
if (voiceSystem == null) if (voiceSystem == null)
@ -38,6 +49,7 @@ namespace VelNet
Debug.LogError("No microphone found. Make sure you have one in the scene."); Debug.LogError("No microphone found. Make sure you have one in the scene.");
return; return;
} }
if (networkObject.IsMine) if (networkObject.IsMine)
{ {
voiceSystem.encodedFrameAvailable += (frame) => voiceSystem.encodedFrameAvailable += (frame) =>
@ -48,9 +60,6 @@ namespace VelNet
BinaryWriter writer = new BinaryWriter(mem); BinaryWriter writer = new BinaryWriter(mem);
writer.Write(frame.array, 0, frame.count); writer.Write(frame.array, 0, frame.count);
this.SendBytes(mem.ToArray(), false); this.SendBytes(mem.ToArray(), false);
}; };
} }
@ -58,20 +67,14 @@ namespace VelNet
source.clip = myClip; source.clip = myClip;
source.loop = true; source.loop = true;
source.Pause(); source.Pause();
} }
// Update is called once per frame // Update is called once per frame
void Update() private void Update()
{ {
if (bufferedAmount > playedAmount) if (bufferedAmount > playedAmount)
{ {
var offset = bufferedAmount - playedAmount; var offset = bufferedAmount - playedAmount;
if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better
{ {
@ -86,7 +89,6 @@ namespace VelNet
} }
else else
{ {
return; return;
} }
} }
@ -96,6 +98,7 @@ namespace VelNet
source.Pause(); source.Pause();
source.timeSamples = bufferedAmount % source.clip.samples; source.timeSamples = bufferedAmount % source.clip.samples;
} }
//Debug.Log(playedAmount); //Debug.Log(playedAmount);
if (source.timeSamples >= lastTime) if (source.timeSamples >= lastTime)
{ {
@ -103,18 +106,11 @@ namespace VelNet
} }
else //repeated else //repeated
{ {
int total = source.clip.samples - lastTime + source.timeSamples; int total = source.clip.samples - lastTime + source.timeSamples;
playedAmount += total; playedAmount += total;
} }
lastTime = source.timeSamples; lastTime = source.timeSamples;
} }
} }
} }

7
TestVelGameServer/Packages/VelNetUnity/package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "edu.uga.engr.vel.velnet", "name": "edu.uga.engr.vel.velnet",
"displayName": "VelNet", "displayName": "VelNet",
"version": "1.1.4", "version": "1.1.5",
"unity": "2019.1", "unity": "2019.1",
"description": "A custom networking library for Unity.", "description": "A custom networking library for Unity.",
"keywords": [ "keywords": [
@ -24,8 +24,5 @@
"description": "Example Scene using Built-in VEL Voice", "description": "Example Scene using Built-in VEL Voice",
"path": "Samples~/ExampleVelVoice" "path": "Samples~/ExampleVelVoice"
} }
], ]
"dependencies": {
} }
}

9
TestVelGameServer/Packages/manifest.json
View File

@ -1,15 +1,14 @@
{ {
"dependencies": { "dependencies": {
"com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git", "com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git",
"com.unity.collab-proxy": "1.15.15", "com.unity.collab-proxy": "1.17.7",
"com.unity.ide.rider": "3.0.13", "com.unity.ide.rider": "3.0.17",
"com.unity.ide.visualstudio": "2.0.15", "com.unity.ide.visualstudio": "2.0.17",
"com.unity.ide.vscode": "1.2.5", "com.unity.ide.vscode": "1.2.5",
"com.unity.test-framework": "1.1.31", "com.unity.test-framework": "1.3.2",
"com.unity.textmeshpro": "3.0.6", "com.unity.textmeshpro": "3.0.6",
"com.unity.timeline": "1.6.4", "com.unity.timeline": "1.6.4",
"com.unity.ugui": "1.0.0", "com.unity.ugui": "1.0.0",
"edu.uga.engr.vel.velnet.dissonance": "file:S:/git_repo/VelNetDissonanceIntegration",
"com.unity.modules.ai": "1.0.0", "com.unity.modules.ai": "1.0.0",
"com.unity.modules.androidjni": "1.0.0", "com.unity.modules.androidjni": "1.0.0",
"com.unity.modules.animation": "1.0.0", "com.unity.modules.animation": "1.0.0",

20
TestVelGameServer/Packages/packages-lock.json
View File

@ -11,7 +11,7 @@
"hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d" "hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d"
}, },
"com.unity.collab-proxy": { "com.unity.collab-proxy": {
"version": "1.15.15", "version": "1.17.7",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -20,14 +20,14 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ext.nunit": { "com.unity.ext.nunit": {
"version": "1.0.6", "version": "2.0.3",
"depth": 1, "depth": 1,
"source": "registry", "source": "registry",
"dependencies": {}, "dependencies": {},
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ide.rider": { "com.unity.ide.rider": {
"version": "3.0.13", "version": "3.0.17",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -36,7 +36,7 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ide.visualstudio": { "com.unity.ide.visualstudio": {
"version": "2.0.15", "version": "2.0.17",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -68,11 +68,11 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.test-framework": { "com.unity.test-framework": {
"version": "1.1.31", "version": "1.3.2",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
"com.unity.ext.nunit": "1.0.6", "com.unity.ext.nunit": "2.0.3",
"com.unity.modules.imgui": "1.0.0", "com.unity.modules.imgui": "1.0.0",
"com.unity.modules.jsonserialize": "1.0.0" "com.unity.modules.jsonserialize": "1.0.0"
}, },
@ -124,14 +124,6 @@
"source": "embedded", "source": "embedded",
"dependencies": {} "dependencies": {}
}, },
"edu.uga.engr.vel.velnet.dissonance": {
"version": "file:S:/git_repo/VelNetDissonanceIntegration",
"depth": 0,
"source": "local",
"dependencies": {
"edu.uga.engr.vel.velnet": "1.1.0"
}
},
"com.unity.modules.ai": { "com.unity.modules.ai": {
"version": "1.0.0", "version": "1.0.0",
"depth": 0, "depth": 0,