Remote video not showing up on one end - WebRTC Video chat app

Question

I'm new to webrtc and react. I'm developing a peer to peer video chat app. On the calling side, both remote video and local video shows up. But on the callee side, only local video shows up. I've been trying to find out where I'm doing wrong but not able to figure it out. One thing I noticed when I console.log peerconnection variable inside handle ice candidate function is, the 'connectionState' is still 'connecting' on the callee side.('connected' on the caller side).

EDIT: I have modified code for readability and using async await. Now I am getting "Failed to execute 'addIceCandidate' on 'RTCPeerConnection': Error processing ICE candidate" error.

EDIT 2: Modified code according to answer

Here is my ORIGINAL code

//refs for my video tag
const localVideoRef = useRef("");
const remoteVideoRef = useRef("");

//video elements
<video ref={remoteVideoRef} playsInline autoPlay className="remoteVideo"></video>
<video ref={localVideoRef} playsInline autoPlay muted className="localVideo"></video>

//button to start call
<button onClick={handleCall}>Call</button>

const handleCall = async () => {

    createPeerConnection();

    navigator.mediaDevices
      .getUserMedia({
        audio: true,
        video: true
      })
      .then(function(localStream) {
        let videoObj = localVideoRef.current;
        videoObj.srcObject = localStream;

        localStream
          .getTracks()
          .forEach(track => myPeerConnection.addTrack(track, localStream));
      })
      .catch("getUserMedia() error: ");
};


let myPeerConnection = null;

  function createPeerConnection() {
    myPeerConnection = new RTCPeerConnection({
      iceServers: [
        {
          urls: "stun:stun2.1.google.com:19302"
        }
      ]
    });

    myPeerConnection.onnegotiationneeded = handleNegotiationNeededEvent;
    myPeerConnection.onicecandidate = handleICECandidateEvent;
    myPeerConnection.ontrack = handleTrackEvent;
    myPeerConnection.onsignalingstatechange = handleSignalingStateChangeEvent;
}

 var isNegotiating = false;

  const handleSignalingStateChangeEvent = () => {
    isNegotiating = myPeerConnection.signalingState != "stable";
  };



function handleNegotiationNeededEvent() {

    if (isNegotiating) {
      return;
    }

    isNegotiating = true;

    myPeerConnection
      .createOffer()
      .then(function(offer) {
        return myPeerConnection.setLocalDescription(offer);
      })
      .then(function() {
        socket.emit("video-offer", {
          from: authContext.user.name,
          to: connectedTo,
          sdp: myPeerConnection.localDescription
        });
      });  
}

//checking if socket is initialized
if (socket) {
    socket.on("gotOffer", data => {
      handleVideoOfferMsg(data);
    });

    socket.on("gotCandidate", data => {
      handleNewICECandidateMsg(data);
    });

    socket.on("gotAnswer", data => {
      console.log("inside got answer");
      handleGotAnswer(data);
    });
  }

function handleVideoOfferMsg(msg) {

    createPeerConnection();

    var desc = new RTCSessionDescription(msg.sdp);

    myPeerConnection
      .setRemoteDescription(desc)
      .then(function() {
        return navigator.mediaDevices.getUserMedia({
          audio: true,
          video: true
        });
      })
      .then(function(stream) {
        let localStream = stream;

        let videoObj = localVideoRef.current;
        videoObj.srcObject = stream;

        localStream
          .getTracks()
          .forEach(track => myPeerConnection.addTrack(track, localStream));
      })
      .then(function() {
        return myPeerConnection.createAnswer();
      })
      .then(function(answer) {
        return myPeerConnection.setLocalDescription(answer);
      })
      .then(function() {
        socket.emit("video-answer", {
          from: authContext.user.name,
          to: connectedTo,
          sdp: myPeerConnection.localDescription
        });

      })
      .catch("error");
  }

 async function handleGotAnswer(msg) {

    if (!myPeerConnection) return;

    // if (isNegotiating) return;
    //I don't know why it's not working (no remote video on the caller side too) when I add above line. So, I am checking signaling state in the below line

    if (myPeerConnection.signalingState == "stable") return;

    await myPeerConnection.setRemoteDescription(
      new RTCSessionDescription(msg.sdp)
    );

}

 function handleICECandidateEvent(event) {
    if (!myPeerConnection) return;

    if (isNegotiating) return;

    if (event.candidate) {
      socket.emit("candidate", {
        to: connectedTo,
        from: authContext.user.name,
        candidate: event.candidate
      });

    }
}

function handleNewICECandidateMsg(msg) {

    if (myPeerConnection.signalingState == "stable") return;

    var candidate = new RTCIceCandidate(msg.candidate);

    myPeerConnection.addIceCandidate(candidate).catch("error");

}

 function handleTrackEvent(event) {
    let videoObj = remoteVideoRef.current;
    videoObj.srcObject = event.streams[0];
}

Here is my NEW code:

let pc1 = new RTCPeerConnection({
    iceServers: [
      {
        urls: "stun:stun2.1.google.com:19302"
      }
    ]
  });

  let pc2 = new RTCPeerConnection({
    iceServers: [
      {
        urls: "stun:stun2.1.google.com:19302"
      }
    ]
  });

  const handleCall = async () => {

    let stream = await navigator.mediaDevices.getUserMedia({
      audio: true,
      video: true
    });

    let videoObj = localVideoRef.current;
    videoObj.srcObject = stream;

    let localStream = stream;

    stream
      .getTracks()
      .forEach(async track => await pc1.addTrack(track, localStream));

    pc1.onnegotiationneeded = async function() {
      let offer = await pc1.createOffer();
      await pc1.setLocalDescription(offer);
      socket.emit("video-offer", {
        from: authContext.user.name,
        to: connectedTo,
        sdp: pc1.localDescription
      });

      pc1.onicecandidate = function(event) {
        if (event.candidate) {
          socket.emit("candidate", {
            pc: "pc1",
            to: connectedTo,
            from: authContext.user.name,
            candidate: event.candidate
          });
        }
      };
    };

    pc1.ontrack = function(event) {
      let videoObj = remoteVideoRef.current;
      videoObj.srcObject = event.streams[0];
    };
  };

  //listening to socket emits from server related to video chat

  if (socket) {
    socket.on("gotOffer", data => {
      //step 1 of callee
      handleVideoOfferMsg(data);
    });

    socket.on("gotCandidate", data => {
      handleNewICECandidateMsg(data);
    });

    socket.on("gotAnswer", data => {
      handleGotAnswer(data);
    });

  }

  async function handleVideoOfferMsg(msg) {
    var desc = new RTCSessionDescription(msg.sdp);

    await pc2.setRemoteDescription(desc);

    let stream = await navigator.mediaDevices.getUserMedia({
      audio: true,
      video: true
    });

    let videoObj = localVideoRef.current;
    videoObj.srcObject = stream;

    let localStream = stream;

    stream
      .getTracks()
      .forEach(async track => await pc2.addTrack(track, localStream));

    let answer = await pc2.createAnswer();

    await pc2.setLocalDescription(answer);

    socket.emit("video-answer", {
      from: authContext.user.name,
      to: connectedTo,
      sdp: pc2.localDescription
    });

    pc2.ontrack = function(event) {
      let videoObj = remoteVideoRef.current;
      videoObj.srcObject = event.streams[0];
    };

    pc2.onicecandidate = function(event) {
      if (event.candidate) {
        socket.emit("candidate", {
          pc: "pc2",
          to: connectedTo,
          from: authContext.user.name,
          candidate: event.candidate
        });
      }
    };
  }

  async function handleGotAnswer(msg) {
    if (pc1.signalingState == "stable") {
      console.log("negotiating");
      return;
    }

    await pc1.setRemoteDescription(new RTCSessionDescription(msg.sdp));

    //INSERTED THIS
    if (candidatesArray.length) {
      candidatesArray.forEach(async msg => {
        var candidate = new RTCIceCandidate(msg.candidate);
        await pc1.addIceCandidate(candidate);
      });
    }

  }

 let candidatesArray = [];


  async function handleNewICECandidateMsg(msg) {
    if (msg.pc == "pc1") {
      var candidate = new RTCIceCandidate(msg.candidate);

      await pc2.addIceCandidate(candidate);
    }

    if (msg.pc == "pc2") {
      try {

        if (pc1.connectionState != "stable" && !pc1.remoteDescription) {
          candidatesArray.push(msg);
          return;
        }
        var candidate = new RTCIceCandidate(msg.candidate);

        await pc1.addIceCandidate(candidate);
      } catch (error) {
//this is where error is triggered.
        console.log("error adding ice candidate: " + error);
      }
    }
  }

I have not put my server side code, because I find no issues in it.

From what I understand, the error is because remotedescription is not being set when addicecandidate is called. May be because I am skipping setting remote description when signalingState is stable. But if I remove that line of code, I am getting another error - "Failed to set remote answer sdp: Called in wrong state: kStable"

Where am I going wrong?

Answer 1

Immediately after calling pc.setLocalDescription() , the PeerConnection will start emitting onicecandidate events, thanks to Trickle ICE. However, this means that maybe the first candidates are generated too fast and they get sent to the remote peer even before sending SDP Offer/Answer!

Maybe that's what happens in your case, and the first candidates are arriving too early from the other side. For this reason it's a good idea to check the PeerConnection signaling state: if it is stable and the remote description has been already set, then you can call pc.addIceCandidate() . If not, you store the candidate in a queue.

Later, when the remote description finally arrives, after setting it you manually add all the candidates that are waiting in the queue.

Here you can see code with this idea. Candidates are first queued, and later when the PeerConnection signaling state becomes stable , queued items are added .