feat(webRTC): enhance WebRTC connection management with improved state handling, add candidate queuing, and refine client stream visibility logic

Backend/public/mobile-sim.js

@@ -155,12 +155,17 @@ let peerSessionId = null;
 let peerTargetDeviceId = null;
 let remoteStreamWaitTimer = null;
 let frameRelayTimer = null;
+let frameRelayStartTimer = null;
 let frameCanvas = null;
 let frameContext = null;
 let activeMediaRecorder = null;
 let activeRecordingChunks = [];
 let activeRecordingStartedAt = null;
 let recordingModalUrl = null;
+let webrtcConnected = false;
+let hasWebrtcEverConnected = false;
+let lastPeerConnectionState = null;
+let pendingRemoteCandidates = [];
 const rtcConfig = {
   iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
 };
@@ -217,7 +222,14 @@ const startCameraPreview = async () => {
   }
 
   try {
-    localCameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
+    localCameraStream = await navigator.mediaDevices.getUserMedia({
+      video: {
+        width: { ideal: 640, max: 960 },
+        height: { ideal: 360, max: 540 },
+        frameRate: { ideal: 15, max: 24 },
+      },
+      audio: false,
+    });
     videoEl.srcObject = localCameraStream;
     videoEl.classList.remove('hidden');
     addActivity('Camera', 'Camera access granted');
@@ -241,19 +253,38 @@ const stopCameraPreview = () => {
   }
 };
 
-const setClientStreamVisibility = (isVisible) => {
+const setClientStreamPlaceholderText = (text) => {
+  const placeholderEl = $('clientStreamPlaceholder');
+  if (!placeholderEl) return;
+  const label = placeholderEl.querySelector('p');
+  if (label) {
+    label.textContent = text;
+  }
+};
+
+const setClientStreamMode = (mode) => {
   const videoEl = $('clientStreamVideo');
   const imageEl = $('clientStreamImage');
   const placeholderEl = $('clientStreamPlaceholder');
-  if (videoEl) {
-    videoEl.classList.toggle('hidden', !isVisible);
+
+  if (videoEl) videoEl.classList.toggle('hidden', mode !== 'video');
+  if (imageEl) imageEl.classList.toggle('hidden', mode !== 'image');
+
+  if (!placeholderEl) return;
+
+  if (mode === 'video' || mode === 'image') {
+    placeholderEl.classList.add('hidden');
+    return;
   }
-  if (imageEl) {
-    imageEl.classList.toggle('hidden', !isVisible);
-  }
-  if (placeholderEl) {
-    placeholderEl.classList.toggle('hidden', isVisible);
+
+  if (mode === 'unavailable') {
+    setClientStreamPlaceholderText('Stream unavailable');
+  } else if (mode === 'connecting') {
+    setClientStreamPlaceholderText('Connecting stream...');
+  } else {
+    setClientStreamPlaceholderText('Waiting for stream');
   }
+  placeholderEl.classList.remove('hidden');
 };
 
 const clearClientStream = () => {
@@ -273,7 +304,7 @@ const clearClientStream = () => {
   if (imageEl) {
     imageEl.src = '';
   }
-  setClientStreamVisibility(false);
+  setClientStreamMode('none');
 };
 
 const getCameraLabel = (cameraDeviceId) => `Camera ${cameraDeviceId?.substring(0, 6) ?? 'Unknown'}`;
@@ -341,6 +372,10 @@ const closeRecordingModal = () => {
 };
 
 const stopFrameRelay = () => {
+  if (frameRelayStartTimer) {
+    clearTimeout(frameRelayStartTimer);
+    frameRelayStartTimer = null;
+  }
   if (frameRelayTimer) {
     clearInterval(frameRelayTimer);
     frameRelayTimer = null;
@@ -349,6 +384,7 @@ const stopFrameRelay = () => {
 
 const startFrameRelay = async (streamSessionId, toDeviceId) => {
   if (!socket || !streamSessionId || !toDeviceId) return;
+  if (hasWebrtcEverConnected) return;
 
   const ready = await startCameraPreview();
   if (!ready) {
@@ -360,6 +396,7 @@ const startFrameRelay = async (streamSessionId, toDeviceId) => {
 
   stopFrameRelay();
   frameRelayTimer = setInterval(() => {
+    if (webrtcConnected || hasWebrtcEverConnected) return;
     if (!socket || cameraVideoEl.readyState < 2 || !cameraVideoEl.videoWidth || !cameraVideoEl.videoHeight) return;
 
     if (!frameCanvas) {
@@ -379,7 +416,7 @@ const startFrameRelay = async (streamSessionId, toDeviceId) => {
       frame,
       capturedAt: new Date().toISOString(),
     });
-  }, 300);
+  }, 600);
 };
 
 const getPreferredRecordingMimeType = () => {
@@ -459,6 +496,7 @@ const stopLocalRecording = async () => {
 };
 
 const teardownPeerConnection = () => {
+  const previousSessionId = peerSessionId;
   if (peerConnection) {
     peerConnection.onicecandidate = null;
     peerConnection.ontrack = null;
@@ -469,9 +507,46 @@ const teardownPeerConnection = () => {
   peerConnection = null;
   peerSessionId = null;
   peerTargetDeviceId = null;
+  lastPeerConnectionState = null;
+  webrtcConnected = false;
+  hasWebrtcEverConnected = false;
+  if (previousSessionId) {
+    pendingRemoteCandidates = pendingRemoteCandidates.filter((item) => item.streamSessionId !== previousSessionId);
+  }
   clearClientStream();
 };
 
+const queueRemoteCandidate = ({ streamSessionId, fromDeviceId, data }) => {
+  if (!streamSessionId || !fromDeviceId || !data) return;
+  pendingRemoteCandidates.push({ streamSessionId, fromDeviceId, data, createdAt: Date.now() });
+  const cutoff = Date.now() - 120000;
+  pendingRemoteCandidates = pendingRemoteCandidates
+    .filter((item) => item.createdAt >= cutoff)
+    .slice(-200);
+};
+
+const takeQueuedCandidates = (streamSessionId, fromDeviceId) => {
+  const queued = pendingRemoteCandidates.filter(
+    (item) => item.streamSessionId === streamSessionId && item.fromDeviceId === fromDeviceId
+  );
+  pendingRemoteCandidates = pendingRemoteCandidates.filter(
+    (item) => !(item.streamSessionId === streamSessionId && item.fromDeviceId === fromDeviceId)
+  );
+  return queued;
+};
+
+const applyQueuedCandidates = async (connection, streamSessionId, fromDeviceId) => {
+  if (!connection?.remoteDescription) return;
+  const queued = takeQueuedCandidates(streamSessionId, fromDeviceId);
+  for (const candidate of queued) {
+    try {
+      await connection.addIceCandidate(new RTCIceCandidate(candidate.data));
+    } catch (error) {
+      console.warn('Dropping queued ICE candidate', error);
+    }
+  }
+};
+
 const ensurePeerConnection = async ({
   streamSessionId,
   targetDeviceId,
@@ -499,8 +574,35 @@ const ensurePeerConnection = async ({
   };
 
   connection.onconnectionstatechange = () => {
+    if (connection.connectionState !== lastPeerConnectionState) {
+      if (connection.connectionState === 'connected') {
+        addActivity('WebRTC', 'Peer connected');
+      } else if (
+        connection.connectionState === 'failed' ||
+        connection.connectionState === 'disconnected' ||
+        connection.connectionState === 'closed'
+      ) {
         addActivity('WebRTC', `Peer ${connection.connectionState}`);
-    if (connection.connectionState === 'failed' || connection.connectionState === 'disconnected' || connection.connectionState === 'closed') {
+      }
+      lastPeerConnectionState = connection.connectionState;
+    }
+
+    if (connection.connectionState === 'connected') {
+      webrtcConnected = true;
+      hasWebrtcEverConnected = true;
+      stopFrameRelay();
+    }
+
+    if (connection.connectionState === 'disconnected') {
+      if (!hasWebrtcEverConnected) {
+        webrtcConnected = false;
+      }
+      return;
+    }
+
+    if (connection.connectionState === 'failed' || connection.connectionState === 'closed') {
+      webrtcConnected = false;
+      hasWebrtcEverConnected = false;
       if (store.get().device?.role === 'client') {
         clearClientStream();
       }
@@ -514,11 +616,14 @@ const ensurePeerConnection = async ({
     }
     const [stream] = event.streams;
     if (!stream) return;
+    webrtcConnected = true;
+    hasWebrtcEverConnected = true;
+    stopFrameRelay();
     remoteClientStream = stream;
     const videoEl = $('clientStreamVideo');
     if (videoEl) {
       videoEl.srcObject = stream;
-      setClientStreamVisibility(true);
+      setClientStreamMode('video');
       void videoEl.play().catch(() => {});
     }
   };
@@ -662,7 +767,11 @@ const connectSocket = () => {
         await startLocalRecording();
         if (payload.sourceDeviceId) {
           await startOfferToClient(streamId, payload.sourceDeviceId);
-          await startFrameRelay(streamId, payload.sourceDeviceId);
+          frameRelayStartTimer = setTimeout(() => {
+            if (!webrtcConnected && !hasWebrtcEverConnected) {
+              void startFrameRelay(streamId, payload.sourceDeviceId);
+            }
+          }, 2500);
         }
         addActivity('Stream', 'Accepted & Published');
         // Auto-stop after 15s for simulation
@@ -720,6 +829,7 @@ const connectSocket = () => {
   });
 
   socket.on('stream:frame', (payload) => {
+    if (webrtcConnected) return;
     if (!payload?.frame) return;
     if (remoteStreamWaitTimer) {
       clearTimeout(remoteStreamWaitTimer);
@@ -733,7 +843,7 @@ const connectSocket = () => {
     if (videoEl) {
       videoEl.classList.add('hidden');
     }
-    setClientStreamVisibility(true);
+    setClientStreamMode('image');
   });
 
   socket.on('stream:ended', (payload) => {
@@ -757,6 +867,7 @@ const connectSocket = () => {
           asCamera: false,
         });
         await connection.setRemoteDescription(new RTCSessionDescription(payload.data));
+        await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId);
         const answer = await connection.createAnswer();
         await connection.setLocalDescription(answer);
         socket.emit('webrtc:signal', {
@@ -771,15 +882,32 @@ const connectSocket = () => {
 
       if (payload.signalType === 'answer') {
         if (device.role !== 'camera' || !peerConnection) return;
+        if (peerSessionId !== payload.streamSessionId || peerTargetDeviceId !== payload.fromDeviceId) {
+          return;
+        }
+        if (peerConnection.signalingState !== 'have-local-offer') {
+          if (peerConnection.signalingState === 'stable' && peerConnection.remoteDescription?.type === 'answer') {
+            return;
+          }
+          return;
+        }
         await peerConnection.setRemoteDescription(new RTCSessionDescription(payload.data));
+        await applyQueuedCandidates(peerConnection, payload.streamSessionId, payload.fromDeviceId);
         addActivity('WebRTC', 'Answer received');
         return;
       }
 
       if (payload.signalType === 'candidate') {
-        if (!peerConnection || !payload.data) return;
+        if (!payload.data) return;
+        if (!peerConnection || peerSessionId !== payload.streamSessionId || peerTargetDeviceId !== payload.fromDeviceId) {
+          queueRemoteCandidate(payload);
+          return;
+        }
+        if (!peerConnection.remoteDescription) {
+          queueRemoteCandidate(payload);
+          return;
+        }
         await peerConnection.addIceCandidate(new RTCIceCandidate(payload.data));
-        addActivity('WebRTC', 'ICE candidate added');
         return;
       }
 
@@ -805,10 +933,14 @@ const startPolling = () => {
   if (pollInterval) clearInterval(pollInterval);
 
   const poller = async () => {
-    const { device, screen } = store.get();
+    const { device, screen, activeStreamSessionId } = store.get();
     if (!device) return;
 
     if (screen === 'home' && device.role === 'client') {
+      if (activeStreamSessionId) {
+        return;
+      }
+
       const recs = await API.ops.listRecordings().catch(() => ({ recordings: [] }));
       store.update({ recordings: recs.recordings || [] });
 
@@ -1128,10 +1260,22 @@ const render = (state) => {
         const videoEl = $('clientStreamVideo');
         if (videoEl && videoEl.srcObject !== remoteClientStream) {
           videoEl.srcObject = remoteClientStream;
-          setClientStreamVisibility(true);
+          setClientStreamMode('video');
           void videoEl.play().catch(() => {});
         }
       }
+
+      const imageEl = $('clientStreamImage');
+      if (imageEl && !imageEl.dataset.errorBound) {
+        imageEl.dataset.errorBound = '1';
+        imageEl.addEventListener('error', () => {
+          const videoEl = $('clientStreamVideo');
+          if (videoEl) {
+            videoEl.classList.add('hidden');
+          }
+          setClientStreamMode('unavailable');
+        });
+      }
     }
 
     const recList = $('recordingsList');