使用 SocketIO 和 Unity Quest 2 vr 作为客户端和 Node.JS 作为服务器

Question

Me and my team are building a VR game for the Quest 2 with Unity written in C#.

We have a program that is the server that is written in JavaScript (Node.JS), the way that they are both communicating is through SocketIO. With in the Unity editor both applications are able to talk to each other, but when I try to build the app as an APK and load it onto the Quest 2, it appears that the server does not talk to the VR game.

Any advice on what I could do to have them talk to each other? Here is the NetworkManager file that I have been using.

Note: Anything with SocketIO in the file always has a red line under and says "The type or namespace name 'SocketIOCommunicator' could not be found (are you missing a using directive or an assembly reference?) [Assembly-CSharp]csharp(CS0246)"

I dont know what this means and what to do.

using System;
using System.Collections;
using Firesplash.UnityAssets.SocketIO;
using Newtonsoft.Json;
using TMPro;
using UnityEngine;
using UnityEngine.SceneManagement;
using UnityEngine.UI;
using static PlayerManager;
using Classes.Managers;

namespace Network
{
    public class JoinRoom
    { 
        public string clinician;
    }
    public class StartGame
    {
        public string game;
    }

    public static class Pausing 
    {
        public static Boolean isPaused = false;
    }

    public class Position
    {
        public float xPos;
        public float zPos;
        public float yPos;
    }

    [RequireComponent(typeof(SocketIOCommunicator))]
    public class NetworkManager : MonoBehaviour
    {
        private SocketIOCommunicator _communicator;
        private SocketIOInstance _socket;
    
        [SerializeField] private TMP_InputField patientNameInput;
        [SerializeField] private TextMeshProUGUI patientName;
        [SerializeField] private TextMeshProUGUI clinicianName;
        [SerializeField] private GameObject setName;
        [SerializeField] private TextMeshProUGUI connText;
        private void Awake()
        { 
            DontDestroyOnLoad(this);
            _communicator = GetComponent<SocketIOCommunicator>();
            _socket = _communicator.Instance;
        }

        private void Start()
        {
            _socket.Connect();
            HandleServerEvents();
            StartCoroutine(PatientConnect());
        }

        private IEnumerator PatientConnect()
        {
            yield return new WaitUntil(() => _socket.IsConnected());
            FetchName();
        }


        private void FetchName()
        {
            if (PlayerPrefs.HasKey("PatientName"))
            {
                PlayerPrefs.SetString("PatientName", "Test Subject FetchName()");
                var name = PlayerPrefs.GetString("PatientName");
                Debug.Log("Has Name: " + name);
                _socket.Emit("unityConnect", name);
                patientName.SetText(name);
                SwitchToViewName();
            }
            else
            {
                SwitchToSetName();
            }
        }
        
        private void HandleServerEvents()
        {
             _socket.On("userJoined", (string payload) =>
            {
                Debug.Log("Joined Room!");
                var obj = JsonConvert.DeserializeObject<JoinRoom>(payload);
                Debug.Log(obj.clinician);
                clinicianName.SetText(obj.clinician);
                connText.gameObject.SetActive(true);
            });
             _socket.On("startGame", (string payload) =>
             {
                 Debug.Log("Started Game");
                 var obj = JsonConvert.DeserializeObject<StartGame>(payload);
                 Debug.Log(obj.game);
                 // connText.SetText("Started Planes");
                 switch(obj.game) {
                    case "3":
                        SceneManager.LoadScene("Planes");
                        break;
                    case "2":
                        SceneManager.LoadScene("Balloons");
                        break;
                    case "1":
                        SceneManager.LoadScene("Blocks");
                        break;
                    default:
                        SceneManager.LoadScene("Init");
                        break;
                 }
             });
             _socket.On("pauseGame", (string payload) => {
                GameplayManager.getManager().ResumeGame();
                Debug.Log("Unpaused");             
            });
             _socket.On("resumeGame", (string payload) => {
                GameplayManager.getManager().PauseGame();
                Debug.Log("Paused");
             });
             _socket.On("updateClientPosition", (string payload) => {
                 Debug.Log(payload);
                 var obj = JsonConvert.DeserializeObject<Position>(payload);
                 float y = obj.yPos;
                 float x = obj.xPos;
                 float z = obj.zPos;
                 movePlayerX(x);
                 movePlayerY(y);
                 movePlayerZ(z);
             });

            _socket.On("kickPatient", (string payload) => {
                Debug.Log(payload);
                SceneManager.LoadScene("Init");
            });

             _socket.On("handMirror", (string payload) => {
                Debug.Log(payload);
                switch (payload) {
                    case "LEFT":
                        CalibrationManager.getManager().SetCalibrationType(2);
                        CalibrationManager.getManager().SetUnaffectedController(OVRInput.Controller.LTouch);
                        break;
                    case "RIGHT":
                        CalibrationManager.getManager().SetCalibrationType(2);
                        CalibrationManager.getManager().SetUnaffectedController(OVRInput.Controller.RTouch);
                        break;
                    default:
                        CalibrationManager.getManager().SetCalibrationType(0);
                        break;
                }                
             });
        }
        
        private void OnDestroy()
        {
            _socket.Close();
        }

        public void SetPatientName()
        {
            
            PlayerPrefs.SetString("PatientName", patientNameInput.text);
            FetchName();
        }

        public void SwitchToSetName()
        {
            setName.SetActive(true);
            patientName.gameObject.SetActive(false);
        }

        private void SwitchToViewName()
        {
            setName.SetActive(false);
            patientName.gameObject.SetActive(true);
        }

    }
}

And here is the JS server files

import express, { json } from "express";

const app = express();
import cors from "cors";

import http from "http";

const httpServer = http.createServer(app);

import { Server, Socket } from "socket.io";

import "./database";
import {
  addPatient,
  checkUserWithPassword,
  clearToken,
  deleteSession,
  getAllSessions,
  getSessionsByUsername,
  insertSession,
  removePatientFromSession,
} from "./database";
import { ISession } from "./ Models/ISession";
import { logout, validateToken } from "./auth";
import { generateUniqueId } from "./utils";
import { IUser } from "./ Models/IUser";

const port = 5000;

interface IStartGame {
  sessionKey: string;
  game: string;
}
const io = new Server(httpServer, {
  cors: {
    origin: "http://localhost:3000",
    methods: ["GET", "POST"],
  },
});

io.use(async (socket, next) => {
  const token = socket.handshake.headers.authorization;
  // const session = socket.handshake.headers.session;
  /*if (verifyToken(token)) {
        setTimeout(next, 1000);
    }*/
  next();
});

app.use(cors());
app.use(express.json());
app.use((req, res, next) => setTimeout(next, 1000));

app.post("/login", async (req, res) => {
  const params = req.body as IUser;

  checkUserWithPassword(params.username, params.shaPassword).then((token) => {
    if (token) {
      res.send({ success: true, token });
    } else {
      res.send({ success: false });
    }
  });
});

app.post("/logout", async (req, res) => {
  const token = req.headers.authorization;
  logout(token, () => {
    res.sendStatus(200);
  });
});

app.post("/loginWithToken", (req, res) => {
  const token = req.headers.authorization;
  validateToken(token, (success: boolean, user: IUser) => {
    if (success) {
      res.send({
        success: true,
        user,
      });
    } else {
      res.send({ success: false });
    }
  });
});

app.get("/sessions", (req, res) => {
  const cred = req.body.user as IUser;
  const token = req.headers.authorization;
  validateToken(token, (success: boolean, user: IUser) => {
    if (success) {
      getAllSessions()
        .then((sessions) => {
          res.send({ sessions });
        })
        .catch(() => {
          res.sendStatus(500);
        });
    } else {
      res.sendStatus(403);
    }
  });
});

app.post("/session", (req, res) => {
  const params = req.body as ISession;
  const token = req.headers.authorization;
  validateToken(token, (success: boolean, user: IUser) => {
    if (success) {
      getAllSessions().then((sessions: ISession[]) => {
        const id = generateUniqueId(
          5,
          sessions.map((s) => s.sessionKey)
        );
        insertSession({
          sessionName: params.sessionName,
          sessionKey: id,
          createdBy: user.username,
          patients: [],
        }).then((suc) => {
          if (suc) {
            res.send({ success: true });
          } else {
            res.sendStatus(500);
          }
        });
      });
    } else {
      res.sendStatus(403);
    }
  });
});
app.delete("/sessionDelete", (req, res) => {
  const params = req.body;
  const token = req.headers.authorization;
  validateToken(token, (success: boolean) => {
    if (success) {
      deleteSession(params.sessionKey);
      console.log(`Clinicain deleted session ${params.sessionKey}`);
      res.sendStatus(200);
    } else {
      res.sendStatus(403);
    }
  });
});



const unitySockets = {};
interface IUnitySocket {
  name: string;
  socket: Socket;
}

const addUnitySocket = (name: string, socket: Socket) => {
  unitySockets[socket.id] = {
    name,
    socket,
  };
};

io.on("connection", (socket) => {
  console.log(`Client ${socket.id} has connected`);
  socket.on("disconnect", () => {
    if (unitySockets[socket.id]) {
      unitySockets[socket.id].socket.disconnect();
      delete unitySockets[socket.id];
    }
    console.log(`Client ${socket.id} has disconnected`);
  });

  socket.on("unityConnect", (name: string) => {
    console.log(name + " joined unity");
    socket.join("waiting");
    addUnitySocket(name, socket);
  });
  socket.on("unityChangeName", (name: string) => {
    if (unitySockets[socket.id]) {
      unitySockets[socket.id].name = name;
    } else {
      addUnitySocket(name, socket);
    }
  });
  app.post("/startGame", (req, res) => {
    const params = req.body as IStartGame;
    console.log(params.sessionKey);
    console.log(params.game);
    socket.to(params.sessionKey).emit("startGame", { game: params.game });
    res.sendStatus(200);
  });
  app.patch("/pause", (req, res) => {
    const params = req.body;
    console.log("Clicked pause");
    if (params.isPaused) {
      socket.to(params.sessionKey).emit("resumeGame");
    } else {
      socket.to(params.sessionKey).emit("pauseGame");
    }
    res.sendStatus(200);
  });

  app.post("/updateClientPosition", (req, res) => {
    const params = req.body;
    console.log(params);
    socket.to(params.sessionKey).emit("updateClientPosition", params.value);
    res.sendStatus(200);
  });

  app.post("/handMirror", (req, res) => {
    const params = req.body;
    socket.to(params.sessionKey).emit("handMirror", params.hand);
    res.sendStatus(200);
  });

  app.delete("/removePatientFromSession", (req, res) => {
    const params = req.body;
    const token = req.headers.authorization;
    validateToken(token, (success: boolean) => {
      if (success) {
        removePatientFromSession(params.sessionKey, params.patientId);
        console.log(`Clinician kicked user ${params.patientId}`);
        socket.to(params.sessionKey).emit("kickPatient", params.patientId);
        res.sendStatus(200);
      } else {
        res.sendStatus(403);
      }
    });
  });

  app.post("/leaveAllRooms", (req, res) => {
    const rooms = io.sockets.adapter.sids[socket.id];
    console.log(rooms);
    // tslint:disable-next-line:forin
    for (const r in rooms) {
      socket.leave(r);
      console.log("left room " + r);
    }
    res.sendStatus(200);
  });
  app.post("/join", (req, res) => {
    const params = req.body as ISession;
    socket.join(params.sessionKey);
    console.log(`joined room ${params.sessionKey}`);
    socket.to(params.sessionKey).emit("userJoined");
    socket.emit("userJoined");
    res.send({ success: true });
  });
  app.get("/getWaitingClients", (req, res) => {
    const waitingSet = io.sockets.adapter.rooms.get("waiting");

    const retList = [];
    console.log(waitingSet);
    if (!waitingSet) {
      res.send({ waitingList: [] });
    } else {
      const waiting = Array.from(waitingSet);
      waiting.forEach((val) => {
        const unitySocket: IUnitySocket = unitySockets[val];
        if (unitySocket) {
          retList.push({
            name: unitySocket.name,
            socketId: val,
          });
        }
      });
      res.send({ waitingList: retList });
    }
  });
  app.get("/getPatientsInSession", (req, res) => {
    const roomKey = req.query.roomKey.toString();
    console.log("Get room " + roomKey);
    const roomSet = io.sockets.adapter.rooms.get(roomKey);
    if (roomSet) {
      const patients = Array.from(roomSet);
      const retList = [];
      patients.forEach((id) => {
        if (unitySockets[id]) {
          retList.push({
            name: unitySockets[id].name,
            socketId: id,
          });
        }
      });
      res.send({ patientList: retList });
    } else {
      res.send({ patientList: [] });
    }
  });
  app.post("/addClientsToSession", (req, res) => {
    const clientList = req.body.clientList;
    const roomKey = req.body.roomKey;
    const clinicianName = req.body.clinician;
    console.log("Add to room " + roomKey);
    clientList.forEach((id) => {
      if (unitySockets[id].socket) {
        unitySockets[id].socket.leave("waiting");
        unitySockets[id].socket.join(roomKey);
        unitySockets[id].socket.emit("userJoined", {
          clinician: clinicianName,
        });
      }
    });
    addPatient(clientList, roomKey);
    res.sendStatus(200);
  });
  app.post("/leave", (req, res) => {
    const params = req.body as ISession;
    socket.leave(params.sessionKey);
    console.log(`left room ${params.sessionKey}`);
    socket.to(params.sessionKey).emit("userLeft");
    socket.emit("userLeft");
    res.send({ success: true });
  });
});

httpServer.listen(port, () => {
  console.log(`server is listening on ${port}`);
});

The Unity scene interface: Unity Scence

Answer 1

I found how to do it, I used heroku to host my node.js server app and then had to change how the front end electron would connect to the heroku website. Same thing goes for the unity quest game.