vel
/
VelNet-Unity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added auto-start option for microphone, formatting and documentation changes

main
Anton Franzluebbers 2022-12-19 23:01:37 -05:00
parent 5ad76f69bc
commit d9f9b4e7f1
5 changed files with 308 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

477
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs
View File

@ -1,262 +1,265 @@
using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using UnityEngine; using UnityEngine;
using System.IO;
using Concentus.Structs; using Concentus.Structs;
using System.Threading; using System.Threading;
using System; using System;
using System.Linq;
namespace VelNet namespace VelNet
{ {
public class VelVoice : MonoBehaviour public class VelVoice : MonoBehaviour
{ {
public class FixedArray public class FixedArray
{ {
public readonly byte[] array;
public int count;
public byte[] array; public FixedArray(int max)
public int count; {
array = new byte[max];
count = 0;
}
}
public FixedArray(int max) private OpusEncoder opusEncoder;
{ private OpusDecoder opusDecoder;
array = new byte[max];
count = 0;
}
}
OpusEncoder opusEncoder;
OpusDecoder opusDecoder;
//StreamWriter sw;
AudioClip clip;
float[] tempData;
float[] encoderBuffer;
List<float[]> frameBuffer;
List<FixedArray> sendQueue = new List<FixedArray>(); //StreamWriter sw;
List<float[]> encoderArrayPool = new List<float[]>(); private AudioClip clip;
List<FixedArray> decoderArrayPool = new List<FixedArray>(); private float[] tempData;
int lastUsedEncoderPool = 0; private float[] encoderBuffer;
int lastUsedDecoderPool = 0; private List<float[]> frameBuffer;
int encoderBufferIndex = 0;
int size = 0;
int lastPosition = 0;
string device = "";
int encoder_frame_size = 640;
double micSampleTime;
int opusFreq = 16000;
double encodeTime = 1 / (double)16000;//16000.0;
double lastMicSample; //holds the last mic sample, in case we need to interpolate it
double sampleTimer = 0; //increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time
EventWaitHandle waiter;
public float silenceThreshold = .01f; //average volume of packet
int numSilent = 0; //number of silent packets detected
public int minSilencePacketsToStop = 5;
double averageVolume = 0;
Thread t;
public Action<FixedArray> encodedFrameAvailable = delegate { };
// Start is called before the first frame update private readonly List<FixedArray> sendQueue = new List<FixedArray>();
void Start() private readonly List<float[]> encoderArrayPool = new List<float[]>();
{ private readonly List<FixedArray> decoderArrayPool = new List<FixedArray>();
opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP); private int lastUsedEncoderPool;
opusDecoder = new OpusDecoder(opusFreq, 1); private int lastUsedDecoderPool;
encoderBuffer = new float[opusFreq]; private int encoderBufferIndex;
frameBuffer = new List<float[]>(); private int lastPosition;
//string path = Application.persistentDataPath + "/" + "mic.csv"; //this was for writing mic samples private string device = "";
//sw = new StreamWriter(path, false); private const int encoderFrameSize = 640;
private double micSampleTime;
private const int opusFreq = 16000;
private const double encodeTime = 1 / (double)16000;
/// <summary>
/// holds the last mic sample, in case we need to interpolate it
/// </summary>
private double lastMicSample;
/// <summary>
/// increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time
/// </summary>
private double sampleTimer;
private EventWaitHandle waiter;
/// <summary>
/// average volume of packet
/// </summary>
public float silenceThreshold = .01f;
/// <summary>
/// number of silent packets detected
/// </summary>
private int numSilent;
public int minSilencePacketsToStop = 5;
private double averageVolume;
private Thread t;
public Action<FixedArray> encodedFrameAvailable = delegate { };
public bool autostartMicrophone = true;
private void Start()
{
opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP);
opusDecoder = new OpusDecoder(opusFreq, 1);
encoderBuffer = new float[opusFreq];
frameBuffer = new List<float[]>();
// pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2)
for (int i = 0; i < 100; i++)
{
encoderArrayPool.Add(new float[encoderFrameSize]);
decoderArrayPool.Add(new FixedArray(encoderFrameSize));
}
t = new Thread(EncodeThread);
waiter = new EventWaitHandle(true, EventResetMode.AutoReset);
t.Start();
if (autostartMicrophone)
{
StartMicrophone(Microphone.devices.FirstOrDefault());
}
}
public void StartMicrophone(string micDeviceName)
{
Debug.Log("Starting with microphone: " + micDeviceName);
if (micDeviceName == null) return;
device = micDeviceName;
Microphone.GetDeviceCaps(device, out int minFreq, out int maxFreq);
Debug.Log("Freq: " + minFreq + ":" + maxFreq);
clip = Microphone.Start(device, true, 10, 48000);
micSampleTime = 1.0 / clip.frequency;
Debug.Log("Frequency:" + clip.frequency);
tempData = new float[clip.samples * clip.channels];
Debug.Log("channels: " + clip.channels);
}
private void OnApplicationQuit()
{
t.Abort();
//sw.Flush();
//sw.Close();
}
private float[] GetNextEncoderPool()
{
lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count;
return encoderArrayPool[lastUsedEncoderPool];
}
private FixedArray GetNextDecoderPool()
{
lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count;
FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool];
toReturn.count = 0;
return toReturn;
}
// Update is called once per frame
private void Update()
{
if (clip != null)
{
int micPosition = Microphone.GetPosition(device);
if (micPosition == lastPosition)
{
return; //sometimes the microphone will not advance
}
int numSamples;
float[] temp;
if (micPosition > lastPosition)
{
numSamples = micPosition - lastPosition;
}
else
{
//whatever was left
numSamples = (tempData.Length - lastPosition) + micPosition;
}
// this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic)
temp = new float[numSamples];
clip.GetData(temp, lastPosition);
lastPosition = micPosition;
// this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets
// iterate through temp, which contains that mic samples at 44.1khz
foreach (float sample in temp)
{
sampleTimer += micSampleTime;
if (sampleTimer > encodeTime)
{
//take a sample between the last sample and the current sample
double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is
double t = diff / micSampleTime; //this should be between 0 and 1
double v = lastMicSample * (1 - t) + sample * t;
sampleTimer -= encodeTime;
encoderBuffer[encoderBufferIndex++] = (float)v;
averageVolume += v > 0 ? v : -v;
if (encoderBufferIndex > encoderFrameSize) //this is when a new packet gets created
{
averageVolume = averageVolume / encoderFrameSize;
if (averageVolume < silenceThreshold)
{
numSilent++;
}
else
{
numSilent = 0;
}
averageVolume = 0;
if (numSilent < minSilencePacketsToStop)
{
float[] frame = GetNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
//lock the frame buffer
System.Array.Copy(encoderBuffer, frame, encoderFrameSize); //nice and fast
for (int i = 0; i < 100; i++) //pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2) lock (frameBuffer)
{ {
encoderArrayPool.Add(new float[encoder_frame_size]); frameBuffer.Add(frame);
decoderArrayPool.Add(new FixedArray(encoder_frame_size)); waiter.Set(); //signal the encode frame
}
}
} encoderBufferIndex = 0;
}
}
t = new Thread(encodeThread); lastMicSample = sample; //remember the last sample, just in case this is the first one next time
waiter = new EventWaitHandle(true, EventResetMode.AutoReset); }
t.Start(); }
}
public void startMicrophone(string mic) lock (sendQueue)
{ {
Debug.Log(mic); foreach (FixedArray f in sendQueue)
device = mic; {
int minFreq, maxFreq; encodedFrameAvailable(f);
Microphone.GetDeviceCaps(device, out minFreq, out maxFreq); }
Debug.Log("Freq: " + minFreq + ":" + maxFreq);
clip = Microphone.Start(device, true, 10, 48000);
micSampleTime = 1.0 / clip.frequency;
Debug.Log("Frequency:" + clip.frequency); sendQueue.Clear();
tempData = new float[clip.samples * clip.channels]; }
Debug.Log("channels: " + clip.channels); }
}
private void OnApplicationQuit() public float[] DecodeOpusData(byte[] data, int count)
{ {
t.Abort(); float[] t = GetNextEncoderPool();
opusDecoder.Decode(data, 0, count, t, 0, encoderFrameSize);
return t;
}
//sw.Flush(); private void EncodeThread()
//sw.Close(); {
while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal
} {
List<float[]> toEncode = new List<float[]>();
float[] getNextEncoderPool()
{
lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count;
return encoderArrayPool[lastUsedEncoderPool];
}
FixedArray getNextDecoderPool()
{
lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count;
FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool];
toReturn.count = 0;
return toReturn;
}
// Update is called once per frame
void Update()
{
if (clip != null)
{
int micPosition = Microphone.GetPosition(device);
if (micPosition == lastPosition)
{
return; //sometimes the microphone will not advance
}
int numSamples = 0;
float[] temp;
if (micPosition > lastPosition)
{
numSamples = micPosition - lastPosition;
}
else
{
//whatever was left
numSamples = (tempData.Length - lastPosition) + micPosition;
}
//Debug.Log(micPosition); lock (frameBuffer)
temp = new float[numSamples]; //this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic) {
clip.GetData(temp, lastPosition); toEncode.AddRange(frameBuffer);
lastPosition = micPosition; frameBuffer.Clear();
}
foreach (float[] frame in toEncode)
//this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets {
for (int i = 0; i < temp.Length; i++) //iterate through temp, which contans that mic samples at 44.1khz FixedArray a = GetNextDecoderPool();
{ int outDataSize = opusEncoder.Encode(frame, 0, encoderFrameSize, a.array, 0, a.array.Length);
sampleTimer += micSampleTime; a.count = outDataSize;
if (sampleTimer > encodeTime) //add frame to the send buffer
{ lock (sendQueue)
{
//take a sample between the last sample and the current sample sendQueue.Add(a);
}
double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is }
double t = diff / micSampleTime; //this should be between 0 and 1 }
double v = lastMicSample * (1 - t) + temp[i] * t; }
sampleTimer -= encodeTime; }
encoderBuffer[encoderBufferIndex++] = (float)v;
averageVolume += v > 0 ? v : -v;
if (encoderBufferIndex > encoder_frame_size) //this is when a new packet gets created
{
averageVolume = averageVolume / encoder_frame_size;
if (averageVolume < silenceThreshold)
{
numSilent++;
}
else
{
numSilent = 0;
}
averageVolume = 0;
if (numSilent < minSilencePacketsToStop)
{
float[] frame = getNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
//lock the frame buffer
System.Array.Copy(encoderBuffer, frame, encoder_frame_size); //nice and fast
lock (frameBuffer)
{
frameBuffer.Add(frame);
waiter.Set(); //signal the encode frame
}
}
encoderBufferIndex = 0;
}
}
lastMicSample = temp[i]; //remember the last sample, just in case this is the first one next time
}
}
lock (sendQueue)
{
foreach (FixedArray f in sendQueue)
{
encodedFrameAvailable(f);
}
sendQueue.Clear();
}
}
public float[] decodeOpusData(byte[] data, int count)
{
float[] t = getNextEncoderPool();
opusDecoder.Decode(data, 0, count, t, 0, encoder_frame_size);
return t;
}
void encodeThread()
{
while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal
{
List<float[]> toEncode = new List<float[]>();
lock (frameBuffer)
{
foreach (float[] frame in frameBuffer)
{
toEncode.Add(frame);
}
frameBuffer.Clear();
}
foreach (float[] frame in toEncode)
{
FixedArray a = getNextDecoderPool();
int out_data_size = opusEncoder.Encode(frame, 0, encoder_frame_size, a.array, 0, a.array.Length);
a.count = out_data_size;
//add frame to the send buffer
lock (sendQueue)
{
sendQueue.Add(a);
}
}
}
}
}
} }

70
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs
View File

@ -1,24 +1,35 @@
using System.Collections;
using System.Collections.Generic;
using System.IO; using System.IO;
using UnityEngine; using UnityEngine;
namespace VelNet namespace VelNet
{ {
public class VelVoicePlayer : NetworkComponent public class VelVoicePlayer : NetworkComponent
{ {
/// <summary>
/// must be set for the player only
/// </summary>
public VelVoice voiceSystem;
/// <summary>
/// must be set for the clone only
/// </summary>
public AudioSource source;
private AudioClip myClip;
public int bufferedAmount;
public int playedAmount;
private int lastTime;
/// <summary>
/// a buffer of 0s to force silence, because playing doesn't stop on demand
/// </summary>
private readonly float[] empty = new float[1000];
private float delayStartTime;
public VelVoice voiceSystem; //must be set for the player only
public AudioSource source; //must be set for the clone only
AudioClip myClip;
public int bufferedAmount = 0;
public int playedAmount = 0;
int lastTime = 0;
float[] empty = new float[1000]; //a buffer of 0s to force silence, because playing doesn't stop on demand
float delayStartTime;
public override void ReceiveBytes(byte[] message) public override void ReceiveBytes(byte[] message)
{ {
float[] temp = voiceSystem.DecodeOpusData(message, message.Length);
float[] temp = voiceSystem.decodeOpusData(message, message.Length);
myClip.SetData(temp, bufferedAmount % source.clip.samples); myClip.SetData(temp, bufferedAmount % source.clip.samples);
bufferedAmount += temp.Length; bufferedAmount += temp.Length;
myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample) myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample)
@ -30,7 +41,7 @@ namespace VelNet
} }
// Start is called before the first frame update // Start is called before the first frame update
void Start() private void Start()
{ {
voiceSystem = GameObject.FindObjectOfType<VelVoice>(); voiceSystem = GameObject.FindObjectOfType<VelVoice>();
if (voiceSystem == null) if (voiceSystem == null)
@ -38,19 +49,17 @@ namespace VelNet
Debug.LogError("No microphone found. Make sure you have one in the scene."); Debug.LogError("No microphone found. Make sure you have one in the scene.");
return; return;
} }
if (networkObject.IsMine) if (networkObject.IsMine)
{ {
voiceSystem.encodedFrameAvailable += (frame) => voiceSystem.encodedFrameAvailable += (frame) =>
{ {
//float[] temp = new float[frame.count]; //float[] temp = new float[frame.count];
//System.Array.Copy(frame.array, temp, frame.count); //System.Array.Copy(frame.array, temp, frame.count);
MemoryStream mem = new MemoryStream(); MemoryStream mem = new MemoryStream();
BinaryWriter writer = new BinaryWriter(mem); BinaryWriter writer = new BinaryWriter(mem);
writer.Write(frame.array, 0, frame.count); writer.Write(frame.array, 0, frame.count);
this.SendBytes(mem.ToArray(), false); this.SendBytes(mem.ToArray(), false);
}; };
} }
@ -58,26 +67,20 @@ namespace VelNet
source.clip = myClip; source.clip = myClip;
source.loop = true; source.loop = true;
source.Pause(); source.Pause();
} }
// Update is called once per frame // Update is called once per frame
void Update() private void Update()
{ {
if (bufferedAmount > playedAmount) if (bufferedAmount > playedAmount)
{ {
var offset = bufferedAmount - playedAmount; var offset = bufferedAmount - playedAmount;
if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better
{ {
var temp = Mathf.Max(0, offset - 2000); var temp = Mathf.Max(0, offset - 2000);
source.pitch = Mathf.Min(2,1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind source.pitch = Mathf.Min(2, 1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind
if (!source.isPlaying) if (!source.isPlaying)
{ {
@ -86,7 +89,6 @@ namespace VelNet
} }
else else
{ {
return; return;
} }
} }
@ -96,6 +98,7 @@ namespace VelNet
source.Pause(); source.Pause();
source.timeSamples = bufferedAmount % source.clip.samples; source.timeSamples = bufferedAmount % source.clip.samples;
} }
//Debug.Log(playedAmount); //Debug.Log(playedAmount);
if (source.timeSamples >= lastTime) if (source.timeSamples >= lastTime)
{ {
@ -103,18 +106,11 @@ namespace VelNet
} }
else //repeated else //repeated
{ {
int total = source.clip.samples - lastTime + source.timeSamples; int total = source.clip.samples - lastTime + source.timeSamples;
playedAmount += total; playedAmount += total;
} }
lastTime = source.timeSamples; lastTime = source.timeSamples;
} }
} }
} }

53
TestVelGameServer/Packages/VelNetUnity/package.json
View File

@ -1,31 +1,28 @@
{ {
"name": "edu.uga.engr.vel.velnet", "name": "edu.uga.engr.vel.velnet",
"displayName": "VelNet", "displayName": "VelNet",
"version": "1.1.4", "version": "1.1.5",
"unity": "2019.1", "unity": "2019.1",
"description": "A custom networking library for Unity.", "description": "A custom networking library for Unity.",
"keywords": [ "keywords": [
"networking", "networking",
"self-hosted" "self-hosted"
], ],
"author": { "author": {
"name": "Virtual Experiences Laboratory", "name": "Virtual Experiences Laboratory",
"email": "kjohnsen@uga.edu", "email": "kjohnsen@uga.edu",
"url": "https://vel.engr.uga.edu" "url": "https://vel.engr.uga.edu"
},
"samples": [
{
"displayName": "Example Dissonance",
"description": "Example Scene with Dissonance Integration Required",
"path": "Samples~/ExampleDissonance"
}, },
"samples": [ {
{ "displayName": "Example VEL Voice",
"displayName": "Example Dissonance", "description": "Example Scene using Built-in VEL Voice",
"description": "Example Scene with Dissonance Integration Required", "path": "Samples~/ExampleVelVoice"
"path": "Samples~/ExampleDissonance"
},
{
"displayName": "Example VEL Voice",
"description": "Example Scene using Built-in VEL Voice",
"path": "Samples~/ExampleVelVoice"
}
],
"dependencies": {
} }
} ]
}

9
TestVelGameServer/Packages/manifest.json
View File

@ -1,15 +1,14 @@
{ {
"dependencies": { "dependencies": {
"com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git", "com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git",
"com.unity.collab-proxy": "1.15.15", "com.unity.collab-proxy": "1.17.7",
"com.unity.ide.rider": "3.0.13", "com.unity.ide.rider": "3.0.17",
"com.unity.ide.visualstudio": "2.0.15", "com.unity.ide.visualstudio": "2.0.17",
"com.unity.ide.vscode": "1.2.5", "com.unity.ide.vscode": "1.2.5",
"com.unity.test-framework": "1.1.31", "com.unity.test-framework": "1.3.2",
"com.unity.textmeshpro": "3.0.6", "com.unity.textmeshpro": "3.0.6",
"com.unity.timeline": "1.6.4", "com.unity.timeline": "1.6.4",
"com.unity.ugui": "1.0.0", "com.unity.ugui": "1.0.0",
"edu.uga.engr.vel.velnet.dissonance": "file:S:/git_repo/VelNetDissonanceIntegration",
"com.unity.modules.ai": "1.0.0", "com.unity.modules.ai": "1.0.0",
"com.unity.modules.androidjni": "1.0.0", "com.unity.modules.androidjni": "1.0.0",
"com.unity.modules.animation": "1.0.0", "com.unity.modules.animation": "1.0.0",

20
TestVelGameServer/Packages/packages-lock.json
View File

@ -11,7 +11,7 @@
"hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d" "hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d"
}, },
"com.unity.collab-proxy": { "com.unity.collab-proxy": {
"version": "1.15.15", "version": "1.17.7",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -20,14 +20,14 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ext.nunit": { "com.unity.ext.nunit": {
"version": "1.0.6", "version": "2.0.3",
"depth": 1, "depth": 1,
"source": "registry", "source": "registry",
"dependencies": {}, "dependencies": {},
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ide.rider": { "com.unity.ide.rider": {
"version": "3.0.13", "version": "3.0.17",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -36,7 +36,7 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.ide.visualstudio": { "com.unity.ide.visualstudio": {
"version": "2.0.15", "version": "2.0.17",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
@ -68,11 +68,11 @@
"url": "https://packages.unity.com" "url": "https://packages.unity.com"
}, },
"com.unity.test-framework": { "com.unity.test-framework": {
"version": "1.1.31", "version": "1.3.2",
"depth": 0, "depth": 0,
"source": "registry", "source": "registry",
"dependencies": { "dependencies": {
"com.unity.ext.nunit": "1.0.6", "com.unity.ext.nunit": "2.0.3",
"com.unity.modules.imgui": "1.0.0", "com.unity.modules.imgui": "1.0.0",
"com.unity.modules.jsonserialize": "1.0.0" "com.unity.modules.jsonserialize": "1.0.0"
}, },
@ -124,14 +124,6 @@
"source": "embedded", "source": "embedded",
"dependencies": {} "dependencies": {}
}, },
"edu.uga.engr.vel.velnet.dissonance": {
"version": "file:S:/git_repo/VelNetDissonanceIntegration",
"depth": 0,
"source": "local",
"dependencies": {
"edu.uga.engr.vel.velnet": "1.1.0"
}
},
"com.unity.modules.ai": { "com.unity.modules.ai": {
"version": "1.0.0", "version": "1.0.0",
"depth": 0, "depth": 0,