WebRTC can't answer an offer with multiple tracks in the same connection

Question

I'm trying to establish a WebRTC connection with two video tracks in the same connection, just after receiving an offer.

The caller can't receive all the video tracks added by the callee when answering the call. However, the caller can start a connection offering two or more video tracks.

That's what the caller (sender) is doing:

const senderStreams = [localStream1];

senderStreams.forEach((stream) => {
  stream.getVideoTracks().forEach((track) => sender.addTrack(track, stream));
});

That's what the callee (receiver) is doing:

const receiverStreams = [localStream2, localStream3]

receiver.onsignalingstatechange = async () => {
  if (receiver.signalingState === "have-remote-offer") {
    receiverStreams.forEach((stream) => {
      stream
        .getVideoTracks()
        .forEach((track) => receiver.addTrack(track, stream));
    });

    const answer = await receiver.createAnswer();
    await receiver.setLocalDescription(answer);
    await sender.setRemoteDescription(answer);
  }
};

The caller (sender) should receive back two track events:

sender.ontrack = (e) => {
  console.log(`Sender received track:`, e.track.id);
  
  // ...
};

Here is the full POC implementation:

 "use strict"; let localStream1, localStream2, localStream3; let sender, receiver; main(); function main() { const btnOffer1 = document.getElementById("btnOffer1"); const btnOffer2 = document.getElementById("btnOffer2"); const buttons = document.querySelector(".buttons"); btnOffer1.addEventListener("click", () => { startCall(1); buttons.remove(); }); btnOffer2.addEventListener("click", () => { startCall(2); buttons.remove(); }); } function startCall(offerOptionNum) { localStream1 = createCanvasStream(); localStream2 = createCanvasStream(); localStream3 = createCanvasStream(); const senderStreams = offerOptionNum === 1? [localStream1]: [localStream1, localStream2]; const receiverStreams = offerOptionNum === 1? [localStream2, localStream3]: [localStream3]; document.getElementById("senderTotalLocalTracks").innerText = senderStreams.length; document.getElementById("receiverTotalLocalTracks").innerText = receiverStreams.length; sender = new RTCPeerConnection(); sender.onicecandidate = (e) => onIceCandidate(sender, e); receiver = new RTCPeerConnection(); receiver.onicecandidate = (e) => onIceCandidate(receiver, e); sender.onconnectionstatechange = () => onConnectionStateChange(sender); receiver.onconnectionstatechange = () => onConnectionStateChange(receiver); sender.onsignalingstatechange = async() => { console.log(`${getName(sender)} Signaling state: ${sender.signalingState}`); if (sender.signalingState === "have-local-offer") { await receiver.setRemoteDescription(sender.localDescription); } }; sender.onnegotiationneeded = async() => { await sender.setLocalDescription(await sender.createOffer()); }; receiver.onsignalingstatechange = async() => { console.log( `${getName(receiver)} Signaling state: ${receiver.signalingState}` ); if (receiver.signalingState === "have-remote-offer") { receiverStreams.forEach((stream) => { stream.getVideoTracks().forEach((track) => receiver.addTrack(track, stream)); }); const answer = await receiver.createAnswer(); await receiver.setLocalDescription(answer); await sender.setRemoteDescription(answer); } }; sender.ontrack = (e) => { console.log(`${getName(sender)} received track:`, e.track.id); const el = document.getElementById("senderTotalRemoteTracks"); el.innerText = Number(el.innerText) + 1; }; receiver.ontrack = (e) => { console.log(`${getName(receiver)} received track:`, e.track.id); const el = document.getElementById("receiverTotalRemoteTracks"); el.innerText = Number(el.innerText) + 1; }; senderStreams.forEach((stream) => { stream.getVideoTracks().forEach((track) => sender.addTrack(track, stream)); }); } function createCanvasStream() { const canvas = Object.assign( document.createElement("canvas", { width: 640, height: 480, }) ); const ctx = canvas.getContext("2d"); const stream = canvas.captureStream(1); const drawInCanvas = () => ctx.fillRect(0, 0, canvas.width, canvas.height); drawInCanvas(); setInterval(() => { drawInCanvas(); }, 1000); return stream; } async function onIceCandidate(pc, event) { if (event.candidate) { try { await getOtherPc(pc).addIceCandidate(event.candidate); } catch (error) { console.error(error, event.candidate); } } } function onConnectionStateChange(pc) { if (pc) { console.log(`${getName(pc)} Connection state: ${pc.connectionState}`); } } function getName(pc) { return pc === sender? "Sender": "Receiver"; } function getOtherPc(pc) { return pc === sender? receiver: sender; }

 html, body { margin: 0; font-family: system-ui, sans-serif; color: #222; background: #f8f8f8; } input, textarea { font-size: 1em; box-sizing: border-box; padding: 6px 8px; } button, code, kbd, pre { font-size: 1em; } code, kbd, pre { font-family: "Menlo", "Monaco", monospace; border-radius: 3px; box-sizing: border-box; padding: 2px 4px 1px 4px; background: rgba(0, 0, 0, 0.1); } pre { padding: 8px 12px; } p { line-height: 1.5em; } a { color: #222; } a:hover { color: #666; }.cards { display: flex; }.buttons { display: flex; }

 <,DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width. initial-scale=1:0"> <title>Document</title> <link rel="stylesheet" href="https.//unpkg.com/blocks.css/dist/blocks.min:css" /> </head> <body> <div class="cards"> <div class="card fixed block sender"> <h2>sender</h2> <p>Remote tracks: <span id="senderTotalRemoteTracks">0</span></p> <p>Local tracks: <span id="senderTotalLocalTracks">0</span></p> </div> <div class="card fixed block receiver"> <h2>receiver</h2> <p>Remote tracks: <span id="receiverTotalRemoteTracks">0</span></p> <p>Local tracks: <span id="receiverTotalLocalTracks">0</span></p> </div> </div> <div class="buttons"> <button class="block accent" id="btnOffer1">Offer 1 track / Receive 2 tracks</button> <button class="block" id="btnOffer2">Offer 2 tracks / Receive 1 track</button> </div> </body> </html>

Is that possible? What am I doing wrong?

Answer 1

Apparently, this is a case of WebRTC renegotiation.

Adding one more track to the receiver's side will fire its negotiationneeded event.

According to the docs ,

This occurs both during the initial setup of the connection as well as any time a change to the communication environment requires reconfiguring the connection.

So I modified the POC to support a bidirectional negotiation process, as follows:

async function onNegotiationNeeded(pc) {
  console.log(`${getName(pc)} negotiationneeded event`);

  await pc.setLocalDescription(await pc.createOffer());
}

async function onSignalingStateChange(pc) {
  console.log(`${getName(pc)} Signaling state: ${pc.signalingState}`);

  const otherPc = getOtherPc(pc);

  if (pc.signalingState === "have-local-offer") {
    await otherPc.setRemoteDescription(pc.localDescription);
  } else if (pc.signalingState === "have-remote-offer") {
    if (pc === receiver) {
      receiverStreams.forEach((stream) => {
        stream.getVideoTracks().forEach((track) => pc.addTrack(track, stream));
      });
    }

    const answer = await pc.createAnswer();
    await pc.setLocalDescription(answer);
    await otherPc.setRemoteDescription(answer);
  }
}

sender.onnegotiationneeded = () => onNegotiationNeeded(sender);
receiver.onnegotiationneeded = () => onNegotiationNeeded(receiver);
sender.onsignalingstatechange = () => onSignalingStateChange(sender);
receiver.onsignalingstatechange = () => onSignalingStateChange(receiver);

Here's the full implementation:

 "use strict"; let localStream1, localStream2, localStream3; let sender, receiver; let senderStreams = [], receiverStreams = []; main(); function main() { const btnOffer1 = document.getElementById("btnOffer1"); const btnOffer2 = document.getElementById("btnOffer2"); const buttons = document.querySelector(".buttons"); const btnTryAgain = document.getElementById("btnTryAgain"); btnOffer1.addEventListener("click", () => { startCall(1); buttons.remove(); }); btnOffer2.addEventListener("click", () => { startCall(2); buttons.remove(); }); btnTryAgain.addEventListener("click", () => { window.location.reload(); }); } function startCall(offerOptionNum) { localStream1 = createCanvasStream(); localStream2 = createCanvasStream(); localStream3 = createCanvasStream(); senderStreams = offerOptionNum === 1? [localStream1]: [localStream1, localStream2]; receiverStreams = offerOptionNum === 1? [localStream2, localStream3]: [localStream3]; document.getElementById("senderTotalLocalTracks").innerText = senderStreams.length; document.getElementById("receiverTotalLocalTracks").innerText = receiverStreams.length; sender = new RTCPeerConnection(); sender.onicecandidate = (e) => onIceCandidate(sender, e); receiver = new RTCPeerConnection(); receiver.onicecandidate = (e) => onIceCandidate(receiver, e); sender.onconnectionstatechange = () => onConnectionStateChange(sender); receiver.onconnectionstatechange = () => onConnectionStateChange(receiver); sender.onnegotiationneeded = () => onNegotiationNeeded(sender); receiver.onnegotiationneeded = () => onNegotiationNeeded(receiver); sender.onsignalingstatechange = () => onSignalingStateChange(sender); receiver.onsignalingstatechange = () => onSignalingStateChange(receiver); sender.ontrack = (e) => { console.log(`${getName(sender)} received track id:`, e.track.id); const el = document.getElementById("senderTotalRemoteTracks"); el.innerText = Number(el.innerText) + 1; }; receiver.ontrack = (e) => { console.log(`${getName(receiver)} received track id:`, e.track.id); const el = document.getElementById("receiverTotalRemoteTracks"); el.innerText = Number(el.innerText) + 1; }; senderStreams.forEach((stream) => { stream.getVideoTracks().forEach((track) => sender.addTrack(track, stream)); }); } function createCanvasStream() { const canvas = Object.assign( document.createElement("canvas", { width: 640, height: 480, }) ); const ctx = canvas.getContext("2d"); const stream = canvas.captureStream(1); const drawInCanvas = () => ctx.fillRect(0, 0, canvas.width, canvas.height); drawInCanvas(); setInterval(() => { drawInCanvas(); }, 1000); return stream; } async function onIceCandidate(pc, event) { if (event.candidate) { try { await getOtherPc(pc).addIceCandidate(event.candidate); } catch (error) { console.error(error, event.candidate); } } } function onConnectionStateChange(pc) { if (pc) { console.log(`${getName(pc)} Connection state: ${pc.connectionState}`); } } async function onNegotiationNeeded(pc) { console.log(`${getName(pc)} negotiationneeded event`); await pc.setLocalDescription(await pc.createOffer()); } async function onSignalingStateChange(pc) { console.log(`${getName(pc)} Signaling state: ${pc.signalingState}`); const otherPc = getOtherPc(pc); if (pc.signalingState === "have-local-offer") { await otherPc.setRemoteDescription(pc.localDescription); } else if (pc.signalingState === "have-remote-offer") { if (pc === receiver) { receiverStreams.forEach((stream) => { stream.getVideoTracks().forEach((track) => pc.addTrack(track, stream)); }); } const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); await otherPc.setRemoteDescription(answer); } } function getName(pc) { return pc === sender? "Sender": "Receiver"; } function getOtherPc(pc) { return pc === sender? receiver: sender; }

 html, body { margin: 0; font-family: system-ui, sans-serif; color: #222; background: #f8f8f8; } input, textarea { font-size: 1em; box-sizing: border-box; padding: 6px 8px; } button, code, kbd, pre { font-size: 1em; } code, kbd, pre { font-family: "Menlo", "Monaco", monospace; border-radius: 3px; box-sizing: border-box; padding: 2px 4px 1px 4px; background: rgba(0, 0, 0, 0.1); } pre { padding: 8px 12px; } p { line-height: 1.5em; } a { color: #222; } a:hover { color: #666; }

 <,DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width. initial-scale=1.0"> <title>Document</title> <link rel="stylesheet" href="reset:css" /> <link rel="stylesheet" href="https.//unpkg.com/blocks.css/dist/blocks.min.css" /> <style>:cards { display; flex. }:buttons { display; flex: } </style> </head> <body> <div class="cards"> <div class="card fixed block sender"> <h2>sender</h2> <p>Remote tracks: <span id="senderTotalRemoteTracks">0</span></p> <p>Local tracks: <span id="senderTotalLocalTracks">0</span></p> </div> <div class="card fixed block receiver"> <h2>receiver</h2> <p>Remote tracks: <span id="receiverTotalRemoteTracks">0</span></p> <p>Local tracks. <span id="receiverTotalLocalTracks">0</span></p> </div> </div> <div class="buttons"> <button class="block accent" id="btnOffer1">Offer 1 track / Receive 2 tracks</button> <button class="block" id="btnOffer2">Offer 2 tracks / Receive 1 track</button> </div> <button class="block" id="btnTryAgain">Try again</button> <script src="main.js" async></script> </body> </html>