vel
/
VelNet-Unity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added auto-start option for microphone, formatting and documentation changes

main
Anton Franzluebbers 2022-12-19 23:01:37 -05:00
parent 5ad76f69bc
commit d9f9b4e7f1
5 changed files with 308 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

477
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoice.cs
View File

@ -1,262 +1,265 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.IO;
using Concentus.Structs;
using System.Threading;
using System;
using System.Linq;
namespace VelNet
{
public class VelVoice : MonoBehaviour
{
public class FixedArray
{
public class VelVoice : MonoBehaviour
{
public class FixedArray
{
public readonly byte[] array;
public int count;
public byte[] array;
public int count;
public FixedArray(int max)
{
array = new byte[max];
count = 0;
}
}
public FixedArray(int max)
{
array = new byte[max];
count = 0;
}
}
OpusEncoder opusEncoder;
OpusDecoder opusDecoder;
//StreamWriter sw;
AudioClip clip;
float[] tempData;
float[] encoderBuffer;
List<float[]> frameBuffer;
private OpusEncoder opusEncoder;
private OpusDecoder opusDecoder;
List<FixedArray> sendQueue = new List<FixedArray>();
List<float[]> encoderArrayPool = new List<float[]>();
List<FixedArray> decoderArrayPool = new List<FixedArray>();
int lastUsedEncoderPool = 0;
int lastUsedDecoderPool = 0;
int encoderBufferIndex = 0;
int size = 0;
int lastPosition = 0;
string device = "";
int encoder_frame_size = 640;
double micSampleTime;
int opusFreq = 16000;
double encodeTime = 1 / (double)16000;//16000.0;
double lastMicSample; //holds the last mic sample, in case we need to interpolate it
double sampleTimer = 0; //increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time
EventWaitHandle waiter;
public float silenceThreshold = .01f; //average volume of packet
int numSilent = 0; //number of silent packets detected
public int minSilencePacketsToStop = 5;
double averageVolume = 0;
Thread t;
public Action<FixedArray> encodedFrameAvailable = delegate { };
//StreamWriter sw;
private AudioClip clip;
private float[] tempData;
private float[] encoderBuffer;
private List<float[]> frameBuffer;
// Start is called before the first frame update
void Start()
{
opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP);
opusDecoder = new OpusDecoder(opusFreq, 1);
encoderBuffer = new float[opusFreq];
frameBuffer = new List<float[]>();
//string path = Application.persistentDataPath + "/" + "mic.csv"; //this was for writing mic samples
//sw = new StreamWriter(path, false);
private readonly List<FixedArray> sendQueue = new List<FixedArray>();
private readonly List<float[]> encoderArrayPool = new List<float[]>();
private readonly List<FixedArray> decoderArrayPool = new List<FixedArray>();
private int lastUsedEncoderPool;
private int lastUsedDecoderPool;
private int encoderBufferIndex;
private int lastPosition;
private string device = "";
private const int encoderFrameSize = 640;
private double micSampleTime;
private const int opusFreq = 16000;
private const double encodeTime = 1 / (double)16000;
/// <summary>
/// holds the last mic sample, in case we need to interpolate it
/// </summary>
private double lastMicSample;
/// <summary>
/// increments with every mic sample, but when over the encodeTime, causes a sample and subtracts that encode time
/// </summary>
private double sampleTimer;
private EventWaitHandle waiter;
/// <summary>
/// average volume of packet
/// </summary>
public float silenceThreshold = .01f;
/// <summary>
/// number of silent packets detected
/// </summary>
private int numSilent;
public int minSilencePacketsToStop = 5;
private double averageVolume;
private Thread t;
public Action<FixedArray> encodedFrameAvailable = delegate { };
public bool autostartMicrophone = true;
private void Start()
{
opusEncoder = new OpusEncoder(opusFreq, 1, Concentus.Enums.OpusApplication.OPUS_APPLICATION_VOIP);
opusDecoder = new OpusDecoder(opusFreq, 1);
encoderBuffer = new float[opusFreq];
frameBuffer = new List<float[]>();
// pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2)
for (int i = 0; i < 100; i++)
{
encoderArrayPool.Add(new float[encoderFrameSize]);
decoderArrayPool.Add(new FixedArray(encoderFrameSize));
}
t = new Thread(EncodeThread);
waiter = new EventWaitHandle(true, EventResetMode.AutoReset);
t.Start();
if (autostartMicrophone)
{
StartMicrophone(Microphone.devices.FirstOrDefault());
}
}
public void StartMicrophone(string micDeviceName)
{
Debug.Log("Starting with microphone: " + micDeviceName);
if (micDeviceName == null) return;
device = micDeviceName;
Microphone.GetDeviceCaps(device, out int minFreq, out int maxFreq);
Debug.Log("Freq: " + minFreq + ":" + maxFreq);
clip = Microphone.Start(device, true, 10, 48000);
micSampleTime = 1.0 / clip.frequency;
Debug.Log("Frequency:" + clip.frequency);
tempData = new float[clip.samples * clip.channels];
Debug.Log("channels: " + clip.channels);
}
private void OnApplicationQuit()
{
t.Abort();
//sw.Flush();
//sw.Close();
}
private float[] GetNextEncoderPool()
{
lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count;
return encoderArrayPool[lastUsedEncoderPool];
}
private FixedArray GetNextDecoderPool()
{
lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count;
FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool];
toReturn.count = 0;
return toReturn;
}
// Update is called once per frame
private void Update()
{
if (clip != null)
{
int micPosition = Microphone.GetPosition(device);
if (micPosition == lastPosition)
{
return; //sometimes the microphone will not advance
}
int numSamples;
float[] temp;
if (micPosition > lastPosition)
{
numSamples = micPosition - lastPosition;
}
else
{
//whatever was left
numSamples = (tempData.Length - lastPosition) + micPosition;
}
// this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic)
temp = new float[numSamples];
clip.GetData(temp, lastPosition);
lastPosition = micPosition;
// this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets
// iterate through temp, which contains that mic samples at 44.1khz
foreach (float sample in temp)
{
sampleTimer += micSampleTime;
if (sampleTimer > encodeTime)
{
//take a sample between the last sample and the current sample
double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is
double t = diff / micSampleTime; //this should be between 0 and 1
double v = lastMicSample * (1 - t) + sample * t;
sampleTimer -= encodeTime;
encoderBuffer[encoderBufferIndex++] = (float)v;
averageVolume += v > 0 ? v : -v;
if (encoderBufferIndex > encoderFrameSize) //this is when a new packet gets created
{
averageVolume = averageVolume / encoderFrameSize;
if (averageVolume < silenceThreshold)
{
numSilent++;
}
else
{
numSilent = 0;
}
averageVolume = 0;
if (numSilent < minSilencePacketsToStop)
{
float[] frame = GetNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
//lock the frame buffer
System.Array.Copy(encoderBuffer, frame, encoderFrameSize); //nice and fast
for (int i = 0; i < 100; i++) //pre allocate a bunch of arrays for microphone frames (probably will only need 1 or 2)
{
encoderArrayPool.Add(new float[encoder_frame_size]);
decoderArrayPool.Add(new FixedArray(encoder_frame_size));
lock (frameBuffer)
{
frameBuffer.Add(frame);
waiter.Set(); //signal the encode frame
}
}
}
encoderBufferIndex = 0;
}
}
t = new Thread(encodeThread);
waiter = new EventWaitHandle(true, EventResetMode.AutoReset);
t.Start();
}
lastMicSample = sample; //remember the last sample, just in case this is the first one next time
}
}
public void startMicrophone(string mic)
{
Debug.Log(mic);
device = mic;
int minFreq, maxFreq;
Microphone.GetDeviceCaps(device, out minFreq, out maxFreq);
Debug.Log("Freq: " + minFreq + ":" + maxFreq);
clip = Microphone.Start(device, true, 10, 48000);
micSampleTime = 1.0 / clip.frequency;
lock (sendQueue)
{
foreach (FixedArray f in sendQueue)
{
encodedFrameAvailable(f);
}
Debug.Log("Frequency:" + clip.frequency);
tempData = new float[clip.samples * clip.channels];
Debug.Log("channels: " + clip.channels);
}
sendQueue.Clear();
}
}
private void OnApplicationQuit()
{
t.Abort();
public float[] DecodeOpusData(byte[] data, int count)
{
float[] t = GetNextEncoderPool();
opusDecoder.Decode(data, 0, count, t, 0, encoderFrameSize);
return t;
}
//sw.Flush();
//sw.Close();
}
float[] getNextEncoderPool()
{
lastUsedEncoderPool = (lastUsedEncoderPool + 1) % encoderArrayPool.Count;
return encoderArrayPool[lastUsedEncoderPool];
}
FixedArray getNextDecoderPool()
{
lastUsedDecoderPool = (lastUsedDecoderPool + 1) % decoderArrayPool.Count;
FixedArray toReturn = decoderArrayPool[lastUsedDecoderPool];
toReturn.count = 0;
return toReturn;
}
// Update is called once per frame
void Update()
{
if (clip != null)
{
int micPosition = Microphone.GetPosition(device);
if (micPosition == lastPosition)
{
return; //sometimes the microphone will not advance
}
int numSamples = 0;
float[] temp;
if (micPosition > lastPosition)
{
numSamples = micPosition - lastPosition;
}
else
{
//whatever was left
numSamples = (tempData.Length - lastPosition) + micPosition;
}
private void EncodeThread()
{
while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal
{
List<float[]> toEncode = new List<float[]>();
//Debug.Log(micPosition);
temp = new float[numSamples]; //this has to be dynamically allocated because of the way clip.GetData works (annoying...maybe use native mic)
clip.GetData(temp, lastPosition);
lastPosition = micPosition;
lock (frameBuffer)
{
toEncode.AddRange(frameBuffer);
frameBuffer.Clear();
}
//this code does 2 things. 1) it samples the microphone data to be exactly what the encoder wants, 2) it forms encoder packets
for (int i = 0; i < temp.Length; i++) //iterate through temp, which contans that mic samples at 44.1khz
{
sampleTimer += micSampleTime;
if (sampleTimer > encodeTime)
{
//take a sample between the last sample and the current sample
double diff = sampleTimer - encodeTime; //this represents how far past this sample actually is
double t = diff / micSampleTime; //this should be between 0 and 1
double v = lastMicSample * (1 - t) + temp[i] * t;
sampleTimer -= encodeTime;
encoderBuffer[encoderBufferIndex++] = (float)v;
averageVolume += v > 0 ? v : -v;
if (encoderBufferIndex > encoder_frame_size) //this is when a new packet gets created
{
averageVolume = averageVolume / encoder_frame_size;
if (averageVolume < silenceThreshold)
{
numSilent++;
}
else
{
numSilent = 0;
}
averageVolume = 0;
if (numSilent < minSilencePacketsToStop)
{
float[] frame = getNextEncoderPool(); //these are predefined sizes, so we don't have to allocate a new array
//lock the frame buffer
System.Array.Copy(encoderBuffer, frame, encoder_frame_size); //nice and fast
lock (frameBuffer)
{
frameBuffer.Add(frame);
waiter.Set(); //signal the encode frame
}
}
encoderBufferIndex = 0;
}
}
lastMicSample = temp[i]; //remember the last sample, just in case this is the first one next time
}
}
lock (sendQueue)
{
foreach (FixedArray f in sendQueue)
{
encodedFrameAvailable(f);
}
sendQueue.Clear();
}
}
public float[] decodeOpusData(byte[] data, int count)
{
float[] t = getNextEncoderPool();
opusDecoder.Decode(data, 0, count, t, 0, encoder_frame_size);
return t;
}
void encodeThread()
{
while (waiter.WaitOne(Timeout.Infinite)) //better to wait on signal
{
List<float[]> toEncode = new List<float[]>();
lock (frameBuffer)
{
foreach (float[] frame in frameBuffer)
{
toEncode.Add(frame);
}
frameBuffer.Clear();
}
foreach (float[] frame in toEncode)
{
FixedArray a = getNextDecoderPool();
int out_data_size = opusEncoder.Encode(frame, 0, encoder_frame_size, a.array, 0, a.array.Length);
a.count = out_data_size;
//add frame to the send buffer
lock (sendQueue)
{
sendQueue.Add(a);
}
}
}
}
}
foreach (float[] frame in toEncode)
{
FixedArray a = GetNextDecoderPool();
int outDataSize = opusEncoder.Encode(frame, 0, encoderFrameSize, a.array, 0, a.array.Length);
a.count = outDataSize;
//add frame to the send buffer
lock (sendQueue)
{
sendQueue.Add(a);
}
}
}
}
}
}

70
TestVelGameServer/Packages/VelNetUnity/Runtime/Util/VelVoicePlayer.cs
View File

@ -1,24 +1,35 @@
using System.Collections;
using System.Collections.Generic;
using System.IO;
using UnityEngine;
namespace VelNet
{
public class VelVoicePlayer : NetworkComponent
{
/// <summary>
/// must be set for the player only
/// </summary>
public VelVoice voiceSystem;
/// <summary>
/// must be set for the clone only
/// </summary>
public AudioSource source;
private AudioClip myClip;
public int bufferedAmount;
public int playedAmount;
private int lastTime;
/// <summary>
/// a buffer of 0s to force silence, because playing doesn't stop on demand
/// </summary>
private readonly float[] empty = new float[1000];
private float delayStartTime;
public VelVoice voiceSystem; //must be set for the player only
public AudioSource source; //must be set for the clone only
AudioClip myClip;
public int bufferedAmount = 0;
public int playedAmount = 0;
int lastTime = 0;
float[] empty = new float[1000]; //a buffer of 0s to force silence, because playing doesn't stop on demand
float delayStartTime;
public override void ReceiveBytes(byte[] message)
{
float[] temp = voiceSystem.decodeOpusData(message, message.Length);
float[] temp = voiceSystem.DecodeOpusData(message, message.Length);
myClip.SetData(temp, bufferedAmount % source.clip.samples);
bufferedAmount += temp.Length;
myClip.SetData(empty, bufferedAmount % source.clip.samples); //buffer some empty data because otherwise you'll hear sound (but it'll be overwritten by the next sample)
@ -30,7 +41,7 @@ namespace VelNet
}
// Start is called before the first frame update
void Start()
private void Start()
{
voiceSystem = GameObject.FindObjectOfType<VelVoice>();
if (voiceSystem == null)
@ -38,19 +49,17 @@ namespace VelNet
Debug.LogError("No microphone found. Make sure you have one in the scene.");
return;
}
if (networkObject.IsMine)
{
voiceSystem.encodedFrameAvailable += (frame) =>
{
//float[] temp = new float[frame.count];
//System.Array.Copy(frame.array, temp, frame.count);
//float[] temp = new float[frame.count];
//System.Array.Copy(frame.array, temp, frame.count);
MemoryStream mem = new MemoryStream();
BinaryWriter writer = new BinaryWriter(mem);
writer.Write(frame.array, 0, frame.count);
this.SendBytes(mem.ToArray(), false);
};
}
@ -58,26 +67,20 @@ namespace VelNet
source.clip = myClip;
source.loop = true;
source.Pause();
}
// Update is called once per frame
void Update()
private void Update()
{
if (bufferedAmount > playedAmount)
{
var offset = bufferedAmount - playedAmount;
if ((offset > 1000) || (Time.time - delayStartTime) > .1f) //this seems to make the quality better
{
var temp = Mathf.Max(0, offset - 2000);
source.pitch = Mathf.Min(2,1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind
var temp = Mathf.Max(0, offset - 2000);
source.pitch = Mathf.Min(2, 1 + temp / 18000.0f); //okay to behind by 2000. These numbers correspond to about 2x speed at a seconds behind
if (!source.isPlaying)
{
@ -86,7 +89,6 @@ namespace VelNet
}
else
{
return;
}
}
@ -96,6 +98,7 @@ namespace VelNet
source.Pause();
source.timeSamples = bufferedAmount % source.clip.samples;
}
//Debug.Log(playedAmount);
if (source.timeSamples >= lastTime)
{
@ -103,18 +106,11 @@ namespace VelNet
}
else //repeated
{
int total = source.clip.samples - lastTime + source.timeSamples;
playedAmount += total;
}
lastTime = source.timeSamples;
}
}
}

53
TestVelGameServer/Packages/VelNetUnity/package.json
View File

@ -1,31 +1,28 @@
{
"name": "edu.uga.engr.vel.velnet",
"displayName": "VelNet",
"version": "1.1.4",
"unity": "2019.1",
"description": "A custom networking library for Unity.",
"keywords": [
"networking",
"self-hosted"
],
"author": {
"name": "Virtual Experiences Laboratory",
"email": "kjohnsen@uga.edu",
"url": "https://vel.engr.uga.edu"
"name": "edu.uga.engr.vel.velnet",
"displayName": "VelNet",
"version": "1.1.5",
"unity": "2019.1",
"description": "A custom networking library for Unity.",
"keywords": [
"networking",
"self-hosted"
],
"author": {
"name": "Virtual Experiences Laboratory",
"email": "kjohnsen@uga.edu",
"url": "https://vel.engr.uga.edu"
},
"samples": [
{
"displayName": "Example Dissonance",
"description": "Example Scene with Dissonance Integration Required",
"path": "Samples~/ExampleDissonance"
},
"samples": [
{
"displayName": "Example Dissonance",
"description": "Example Scene with Dissonance Integration Required",
"path": "Samples~/ExampleDissonance"
},
{
"displayName": "Example VEL Voice",
"description": "Example Scene using Built-in VEL Voice",
"path": "Samples~/ExampleVelVoice"
}
],
"dependencies": {
{
"displayName": "Example VEL Voice",
"description": "Example Scene using Built-in VEL Voice",
"path": "Samples~/ExampleVelVoice"
}
}
]
}

9
TestVelGameServer/Packages/manifest.json
View File

@ -1,15 +1,14 @@
{
"dependencies": {
"com.franzco.unityutilities": "https://github.com/AntonFranzluebbers/unityutilities.git",
"com.unity.collab-proxy": "1.15.15",
"com.unity.ide.rider": "3.0.13",
"com.unity.ide.visualstudio": "2.0.15",
"com.unity.collab-proxy": "1.17.7",
"com.unity.ide.rider": "3.0.17",
"com.unity.ide.visualstudio": "2.0.17",
"com.unity.ide.vscode": "1.2.5",
"com.unity.test-framework": "1.1.31",
"com.unity.test-framework": "1.3.2",
"com.unity.textmeshpro": "3.0.6",
"com.unity.timeline": "1.6.4",
"com.unity.ugui": "1.0.0",
"edu.uga.engr.vel.velnet.dissonance": "file:S:/git_repo/VelNetDissonanceIntegration",
"com.unity.modules.ai": "1.0.0",
"com.unity.modules.androidjni": "1.0.0",
"com.unity.modules.animation": "1.0.0",

20
TestVelGameServer/Packages/packages-lock.json
View File

@ -11,7 +11,7 @@
"hash": "330ff73d80febbf9b505dd0ebd86d370d12fc73d"
},
"com.unity.collab-proxy": {
"version": "1.15.15",
"version": "1.17.7",
"depth": 0,
"source": "registry",
"dependencies": {
@ -20,14 +20,14 @@
"url": "https://packages.unity.com"
},
"com.unity.ext.nunit": {
"version": "1.0.6",
"version": "2.0.3",
"depth": 1,
"source": "registry",
"dependencies": {},
"url": "https://packages.unity.com"
},
"com.unity.ide.rider": {
"version": "3.0.13",
"version": "3.0.17",
"depth": 0,
"source": "registry",
"dependencies": {
@ -36,7 +36,7 @@
"url": "https://packages.unity.com"
},
"com.unity.ide.visualstudio": {
"version": "2.0.15",
"version": "2.0.17",
"depth": 0,
"source": "registry",
"dependencies": {
@ -68,11 +68,11 @@
"url": "https://packages.unity.com"
},
"com.unity.test-framework": {
"version": "1.1.31",
"version": "1.3.2",
"depth": 0,
"source": "registry",
"dependencies": {
"com.unity.ext.nunit": "1.0.6",
"com.unity.ext.nunit": "2.0.3",
"com.unity.modules.imgui": "1.0.0",
"com.unity.modules.jsonserialize": "1.0.0"
},
@ -124,14 +124,6 @@
"source": "embedded",
"dependencies": {}
},
"edu.uga.engr.vel.velnet.dissonance": {
"version": "file:S:/git_repo/VelNetDissonanceIntegration",
"depth": 0,
"source": "local",
"dependencies": {
"edu.uga.engr.vel.velnet": "1.1.0"
}
},
"com.unity.modules.ai": {
"version": "1.0.0",
"depth": 0,