Rudimentary VR implementation: head/hand position tracked and quest

transmits voice.

Known issues:
- Quest probably can't hear others.
- Hand bone rotations seemingly not being transmitted.

TestVelGameServer/Assets/Samples/VelNet/1.0.4/Example/VRPlayerPrefab.prefab

@@ -10,7 +10,6 @@ GameObject:
   m_Component:
   - component: {fileID: 749297529733664309}
   - component: {fileID: 2292958742615076072}
-  - component: {fileID: 3384017775560023782}
   m_Layer: 0
   m_Name: Left Hand
   m_TagString: Untagged
@@ -48,27 +47,7 @@ MonoBehaviour:
   networkObject: {fileID: 1210055141702294256}
   serializationRateHz: 10
   hand: {fileID: 0}
-  toSync: {fileID: 3384017775560023782}
+  toSync:
-  targets: []
-  smoothness: 0.1
---- !u!114 &3384017775560023782
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 956485485189628882}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 674a40251fe8ad841b18517ac5209957, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  _skeletonType: 0
-  _updateRootPose: 0
-  _updateRootScale: 0
-  _enablePhysicsCapsules: 0
-  _applyBoneTranslations: 1
-  _customBones_V2:
   - {fileID: 6270683889218838997}
   - {fileID: 6270683889218839035}
   - {fileID: 6270683889218839007}
@@ -88,37 +67,8 @@ MonoBehaviour:
   - {fileID: 6270683889218839023}
   - {fileID: 6270683889218839021}
   - {fileID: 6270683889218839011}
-  - {fileID: 6270683889218838633}
+  targets: []
-  - {fileID: 6270683889218838989}
+  smoothness: 0.1
-  - {fileID: 6270683889218838967}
-  - {fileID: 6270683889218838913}
-  - {fileID: 6270683889218838943}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
 --- !u!1 &2007684774674236866
 GameObject:
   m_ObjectHideFlags: 0
@@ -274,7 +224,6 @@ GameObject:
   m_Component:
   - component: {fileID: 3512402255783560565}
   - component: {fileID: 8288445267447093230}
-  - component: {fileID: 7998315652155547491}
   m_Layer: 0
   m_Name: Right Hand
   m_TagString: Untagged
@@ -312,27 +261,7 @@ MonoBehaviour:
   networkObject: {fileID: 1210055141702294256}
   serializationRateHz: 10
   hand: {fileID: 0}
-  toSync: {fileID: 7998315652155547491}
+  toSync:
-  targets: []
-  smoothness: 0.1
---- !u!114 &7998315652155547491
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4956616402311533684}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 674a40251fe8ad841b18517ac5209957, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  _skeletonType: 1
-  _updateRootPose: 0
-  _updateRootScale: 0
-  _enablePhysicsCapsules: 0
-  _applyBoneTranslations: 1
-  _customBones_V2:
   - {fileID: 8015258244566099424}
   - {fileID: 8015258244566099406}
   - {fileID: 8015258244566099434}
@@ -352,37 +281,8 @@ MonoBehaviour:
   - {fileID: 8015258244566099418}
   - {fileID: 8015258244566099416}
   - {fileID: 8015258244566099414}
-  - {fileID: 8015258244566099034}
+  targets: []
-  - {fileID: 8015258244566099446}
+  smoothness: 0.1
-  - {fileID: 8015258244566099328}
-  - {fileID: 8015258244566099378}
-  - {fileID: 8015258244566099368}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
-  - {fileID: 0}
 --- !u!1 &6527810452308992911
 GameObject:
   m_ObjectHideFlags: 0
@@ -443,6 +343,7 @@ MonoBehaviour:
   - {fileID: 8171177894477629926}
   - {fileID: 8571378120727598095}
   - {fileID: 8288445267447093230}
+  - {fileID: 6169608247249244442}
 --- !u!114 &5057794648895309284
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -681,11 +582,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400006, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218838989 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400054, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &6270683889218839025 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400010, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
@@ -701,26 +597,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400014, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218838967 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400076, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218839017 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400018, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218839023 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400020, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218838633 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400146, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &6270683889218838993 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400042, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
@@ -741,11 +617,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400036, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218838913 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400122, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &6270683889218839003 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400032, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
@@ -756,9 +627,9 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400028, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218838943 stripped
+--- !u!4 &6270683889218839011 stripped
 Transform:
-  m_CorrespondingSourceObject: {fileID: 400100, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
+  m_CorrespondingSourceObject: {fileID: 400024, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
 --- !u!4 &6270683889218839013 stripped
@@ -766,16 +637,21 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400030, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &6270683889218839011 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400024, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
-  m_PrefabInstance: {fileID: 6270683889218452347}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &6270683889218839021 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400022, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
   m_PrefabInstance: {fileID: 6270683889218452347}
   m_PrefabAsset: {fileID: 0}
+--- !u!4 &6270683889218839023 stripped
+Transform:
+  m_CorrespondingSourceObject: {fileID: 400020, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
+  m_PrefabInstance: {fileID: 6270683889218452347}
+  m_PrefabAsset: {fileID: 0}
+--- !u!4 &6270683889218839017 stripped
+Transform:
+  m_CorrespondingSourceObject: {fileID: 400018, guid: 442d5b4503ed83c4897ef44659446587, type: 3}
+  m_PrefabInstance: {fileID: 6270683889218452347}
+  m_PrefabAsset: {fileID: 0}
 --- !u!1001 &8015258244566237006
 PrefabInstance:
   m_ObjectHideFlags: 0
@@ -848,11 +724,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400004, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099400 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400006, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
-  m_PrefabInstance: {fileID: 8015258244566237006}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &8015258244566099454 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400048, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
@@ -873,11 +744,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400014, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099328 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400078, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
-  m_PrefabInstance: {fileID: 8015258244566237006}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &8015258244566099420 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400018, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
@@ -898,14 +764,9 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400024, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099368 stripped
+--- !u!4 &8015258244566099400 stripped
 Transform:
-  m_CorrespondingSourceObject: {fileID: 400102, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
+  m_CorrespondingSourceObject: {fileID: 400006, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
-  m_PrefabInstance: {fileID: 8015258244566237006}
-  m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099446 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400056, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
 --- !u!4 &8015258244566099408 stripped
@@ -918,11 +779,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400032, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099378 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400124, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
-  m_PrefabInstance: {fileID: 8015258244566237006}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &8015258244566099434 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400036, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
@@ -943,11 +799,6 @@ Transform:
   m_CorrespondingSourceObject: {fileID: 400042, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
   m_PrefabInstance: {fileID: 8015258244566237006}
   m_PrefabAsset: {fileID: 0}
---- !u!4 &8015258244566099034 stripped
-Transform:
-  m_CorrespondingSourceObject: {fileID: 400148, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}
-  m_PrefabInstance: {fileID: 8015258244566237006}
-  m_PrefabAsset: {fileID: 0}
 --- !u!4 &8015258244566099424 stripped
 Transform:
   m_CorrespondingSourceObject: {fileID: 400046, guid: 88e2e1bfe11ec6c438b0ade27eaeded1, type: 3}

TestVelGameServer/Assets/Samples/VelNet/1.0.4/Example/VelNetSyncHand.cs

@@ -7,14 +7,41 @@ using VelNet;
 public class VelNetSyncHand : NetworkSerializedObject
 {
     public OVRSkeleton hand;
-    public OVRCustomSkeleton toSync;
+    public Transform[] toSync;
     public Quaternion[] targets;
     public float smoothness = .1f;
+    #region bonesToSync
+    /*
+    public Transform Hand;
+    public Transform Wrist;
+    public Transform Index1;
+    public Transform Index2;
+    public Transform Index3;
+
+    public Transform Middle1;
+    public Transform Middle2;
+    public Transform Middle3;
+
+    public Transform Ring1;
+    public Transform Ring2;
+    public Transform Ring3;
+
+    public Transform Pinky0;
+    public Transform Pinky1;
+    public Transform Pinky2;
+    public Transform Pinky3;
+
+    public Transform Thumb0;
+    public Transform Thumb1;
+    public Transform Thumb2;
+    public Transform Thumb3;
+    */
+    #endregion
     protected override void ReceiveState(byte[] message)
     {
         using MemoryStream mem = new MemoryStream(message);
         using BinaryReader reader = new BinaryReader(mem);
-        for (int i =0; i<toSync.Bones.Count; i++)
+        for (int i =0; i<toSync.Length; i++)
         {
             targets[i] = reader.ReadQuaternion();
         }
@@ -24,9 +51,9 @@ public class VelNetSyncHand : NetworkSerializedObject
     {
         using MemoryStream mem = new MemoryStream();
         using BinaryWriter writer = new BinaryWriter(mem);
-        for(int i = 0; i<toSync.Bones.Count; i++)
+        for(int i = 0; i<toSync.Length; i++)
         {
-            writer.Write(toSync.Bones[0].Transform.rotation); //TODO: optimize to just one float for some bones
+            writer.Write(toSync[0].rotation); //TODO: optimize to just one float for some bones
         }
 
         return mem.ToArray();
@@ -35,24 +62,24 @@ public class VelNetSyncHand : NetworkSerializedObject
     // Start is called before the first frame update
     void Start()
     {
-        targets = new Quaternion[toSync.Bones.Count];
+        targets = new Quaternion[toSync.Length];
     }
 
     // Update is called once per frame
     void Update()
     {
-        if (IsMine && hand) //need to set values from tracked hand to this networkobject
+        if (IsMine && hand?.Bones != null && hand?.Bones.Count == toSync.Length) //need to set values from tracked hand to this networkobject
         {
-            for(int i = 0; i < hand.Bones.Count; i++)
+            for(int i = 0; i < toSync.Length; i++)
             {
-                toSync.Bones[i].Transform.rotation = hand.Bones[i].Transform.rotation;
+                toSync[i].rotation = hand.Bones[i].Transform.rotation;
             }
             
             return;
         }
         for(int i = 0;i<targets.Length;i++)
         {
-            toSync.Bones[i].Transform.rotation = Quaternion.Slerp(transform.rotation, targets[i], 1 / smoothness / serializationRateHz);
+            toSync[i].rotation = Quaternion.Slerp(transform.rotation, targets[i], 1 / smoothness / serializationRateHz);
         }
     }
 }

TestVelGameServer/Packages/VelNetUnity/Runtime/NetworkObject.cs

@@ -55,9 +55,16 @@ namespace VelNet
 			owner.SendGroupMessage(this, group, index.ToString(), message, reliable);
 		}
 
-		public void ReceiveBytes(string identifier, byte[] message)
+		public void ReceiveBytes(string identifier, byte[] message, string str_message = "")
 		{
 			// send the message to the right component
+			var index = int.Parse(identifier);
+			if(index < 0 || index >= syncedComponents.Count)
+            {
+				Debug.LogError("Got message for NetworkComponent that doesn't exist: " + identifier + " on " + prefabName);
+				Debug.Log(str_message);
+				return;
+            }
 			syncedComponents[int.Parse(identifier)].ReceiveBytes(message);
 		}
 	}

TestVelGameServer/Packages/VelNetUnity/Runtime/VelNetPlayer.cs

@@ -79,7 +79,7 @@ namespace VelNet
 						{
 							if (manager.objects[objectKey].owner == this)
 							{
-								manager.objects[objectKey].ReceiveBytes(identifier, messageBytes);
+								manager.objects[objectKey].ReceiveBytes(identifier, messageBytes, m.text);
 							}
 						}
 

TestVelGameServer/ProjectSettings/ProjectSettings.asset

@@ -144,6 +144,8 @@ PlayerSettings:
   - {fileID: 0}
   - {fileID: 0}
   - {fileID: 0}
+  - {fileID: 11400000, guid: 851187c2a28053f4e806e95f3ff7ed7d, type: 2}
+  - {fileID: -332507616495366349, guid: c614bbfd853c73e4cb7fcf7b7f69a45e, type: 2}
   metroInputSource: 0
   wsaTransparentSwapchain: 0
   m_HolographicPauseOnTrackingLoss: 1