From d9f55ba66e83892a95639670648cc376daa0bf8f Mon Sep 17 00:00:00 2001 From: Matiss Jurevics Date: Sat, 7 Feb 2026 10:30:00 +0000 Subject: [PATCH] feat(webRTC): enhance WebRTC connection management with improved state handling, add candidate queuing, and refine client stream visibility logic --- Backend/public/mobile-sim.js | 184 +++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 20 deletions(-) diff --git a/Backend/public/mobile-sim.js b/Backend/public/mobile-sim.js index 198c274..994873c 100644 --- a/Backend/public/mobile-sim.js +++ b/Backend/public/mobile-sim.js @@ -155,12 +155,17 @@ let peerSessionId = null; let peerTargetDeviceId = null; let remoteStreamWaitTimer = null; let frameRelayTimer = null; +let frameRelayStartTimer = null; let frameCanvas = null; let frameContext = null; let activeMediaRecorder = null; let activeRecordingChunks = []; let activeRecordingStartedAt = null; let recordingModalUrl = null; +let webrtcConnected = false; +let hasWebrtcEverConnected = false; +let lastPeerConnectionState = null; +let pendingRemoteCandidates = []; const rtcConfig = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], }; @@ -217,7 +222,14 @@ const startCameraPreview = async () => { } try { - localCameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); + localCameraStream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { ideal: 640, max: 960 }, + height: { ideal: 360, max: 540 }, + frameRate: { ideal: 15, max: 24 }, + }, + audio: false, + }); videoEl.srcObject = localCameraStream; videoEl.classList.remove('hidden'); addActivity('Camera', 'Camera access granted'); @@ -241,19 +253,38 @@ const stopCameraPreview = () => { } }; -const setClientStreamVisibility = (isVisible) => { +const setClientStreamPlaceholderText = (text) => { + const placeholderEl = $('clientStreamPlaceholder'); + if (!placeholderEl) return; + const label = placeholderEl.querySelector('p'); + if (label) { + label.textContent = text; + } +}; + +const setClientStreamMode = (mode) => { const videoEl = $('clientStreamVideo'); const imageEl = $('clientStreamImage'); const placeholderEl = $('clientStreamPlaceholder'); - if (videoEl) { - videoEl.classList.toggle('hidden', !isVisible); + + if (videoEl) videoEl.classList.toggle('hidden', mode !== 'video'); + if (imageEl) imageEl.classList.toggle('hidden', mode !== 'image'); + + if (!placeholderEl) return; + + if (mode === 'video' || mode === 'image') { + placeholderEl.classList.add('hidden'); + return; } - if (imageEl) { - imageEl.classList.toggle('hidden', !isVisible); - } - if (placeholderEl) { - placeholderEl.classList.toggle('hidden', isVisible); + + if (mode === 'unavailable') { + setClientStreamPlaceholderText('Stream unavailable'); + } else if (mode === 'connecting') { + setClientStreamPlaceholderText('Connecting stream...'); + } else { + setClientStreamPlaceholderText('Waiting for stream'); } + placeholderEl.classList.remove('hidden'); }; const clearClientStream = () => { @@ -273,7 +304,7 @@ const clearClientStream = () => { if (imageEl) { imageEl.src = ''; } - setClientStreamVisibility(false); + setClientStreamMode('none'); }; const getCameraLabel = (cameraDeviceId) => `Camera ${cameraDeviceId?.substring(0, 6) ?? 'Unknown'}`; @@ -341,6 +372,10 @@ const closeRecordingModal = () => { }; const stopFrameRelay = () => { + if (frameRelayStartTimer) { + clearTimeout(frameRelayStartTimer); + frameRelayStartTimer = null; + } if (frameRelayTimer) { clearInterval(frameRelayTimer); frameRelayTimer = null; @@ -349,6 +384,7 @@ const stopFrameRelay = () => { const startFrameRelay = async (streamSessionId, toDeviceId) => { if (!socket || !streamSessionId || !toDeviceId) return; + if (hasWebrtcEverConnected) return; const ready = await startCameraPreview(); if (!ready) { @@ -360,6 +396,7 @@ const startFrameRelay = async (streamSessionId, toDeviceId) => { stopFrameRelay(); frameRelayTimer = setInterval(() => { + if (webrtcConnected || hasWebrtcEverConnected) return; if (!socket || cameraVideoEl.readyState < 2 || !cameraVideoEl.videoWidth || !cameraVideoEl.videoHeight) return; if (!frameCanvas) { @@ -379,7 +416,7 @@ const startFrameRelay = async (streamSessionId, toDeviceId) => { frame, capturedAt: new Date().toISOString(), }); - }, 300); + }, 600); }; const getPreferredRecordingMimeType = () => { @@ -459,6 +496,7 @@ const stopLocalRecording = async () => { }; const teardownPeerConnection = () => { + const previousSessionId = peerSessionId; if (peerConnection) { peerConnection.onicecandidate = null; peerConnection.ontrack = null; @@ -469,9 +507,46 @@ const teardownPeerConnection = () => { peerConnection = null; peerSessionId = null; peerTargetDeviceId = null; + lastPeerConnectionState = null; + webrtcConnected = false; + hasWebrtcEverConnected = false; + if (previousSessionId) { + pendingRemoteCandidates = pendingRemoteCandidates.filter((item) => item.streamSessionId !== previousSessionId); + } clearClientStream(); }; +const queueRemoteCandidate = ({ streamSessionId, fromDeviceId, data }) => { + if (!streamSessionId || !fromDeviceId || !data) return; + pendingRemoteCandidates.push({ streamSessionId, fromDeviceId, data, createdAt: Date.now() }); + const cutoff = Date.now() - 120000; + pendingRemoteCandidates = pendingRemoteCandidates + .filter((item) => item.createdAt >= cutoff) + .slice(-200); +}; + +const takeQueuedCandidates = (streamSessionId, fromDeviceId) => { + const queued = pendingRemoteCandidates.filter( + (item) => item.streamSessionId === streamSessionId && item.fromDeviceId === fromDeviceId + ); + pendingRemoteCandidates = pendingRemoteCandidates.filter( + (item) => !(item.streamSessionId === streamSessionId && item.fromDeviceId === fromDeviceId) + ); + return queued; +}; + +const applyQueuedCandidates = async (connection, streamSessionId, fromDeviceId) => { + if (!connection?.remoteDescription) return; + const queued = takeQueuedCandidates(streamSessionId, fromDeviceId); + for (const candidate of queued) { + try { + await connection.addIceCandidate(new RTCIceCandidate(candidate.data)); + } catch (error) { + console.warn('Dropping queued ICE candidate', error); + } + } +}; + const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, @@ -499,8 +574,35 @@ const ensurePeerConnection = async ({ }; connection.onconnectionstatechange = () => { - addActivity('WebRTC', `Peer ${connection.connectionState}`); - if (connection.connectionState === 'failed' || connection.connectionState === 'disconnected' || connection.connectionState === 'closed') { + if (connection.connectionState !== lastPeerConnectionState) { + if (connection.connectionState === 'connected') { + addActivity('WebRTC', 'Peer connected'); + } else if ( + connection.connectionState === 'failed' || + connection.connectionState === 'disconnected' || + connection.connectionState === 'closed' + ) { + addActivity('WebRTC', `Peer ${connection.connectionState}`); + } + lastPeerConnectionState = connection.connectionState; + } + + if (connection.connectionState === 'connected') { + webrtcConnected = true; + hasWebrtcEverConnected = true; + stopFrameRelay(); + } + + if (connection.connectionState === 'disconnected') { + if (!hasWebrtcEverConnected) { + webrtcConnected = false; + } + return; + } + + if (connection.connectionState === 'failed' || connection.connectionState === 'closed') { + webrtcConnected = false; + hasWebrtcEverConnected = false; if (store.get().device?.role === 'client') { clearClientStream(); } @@ -514,11 +616,14 @@ const ensurePeerConnection = async ({ } const [stream] = event.streams; if (!stream) return; + webrtcConnected = true; + hasWebrtcEverConnected = true; + stopFrameRelay(); remoteClientStream = stream; const videoEl = $('clientStreamVideo'); if (videoEl) { videoEl.srcObject = stream; - setClientStreamVisibility(true); + setClientStreamMode('video'); void videoEl.play().catch(() => {}); } }; @@ -662,7 +767,11 @@ const connectSocket = () => { await startLocalRecording(); if (payload.sourceDeviceId) { await startOfferToClient(streamId, payload.sourceDeviceId); - await startFrameRelay(streamId, payload.sourceDeviceId); + frameRelayStartTimer = setTimeout(() => { + if (!webrtcConnected && !hasWebrtcEverConnected) { + void startFrameRelay(streamId, payload.sourceDeviceId); + } + }, 2500); } addActivity('Stream', 'Accepted & Published'); // Auto-stop after 15s for simulation @@ -720,6 +829,7 @@ const connectSocket = () => { }); socket.on('stream:frame', (payload) => { + if (webrtcConnected) return; if (!payload?.frame) return; if (remoteStreamWaitTimer) { clearTimeout(remoteStreamWaitTimer); @@ -733,7 +843,7 @@ const connectSocket = () => { if (videoEl) { videoEl.classList.add('hidden'); } - setClientStreamVisibility(true); + setClientStreamMode('image'); }); socket.on('stream:ended', (payload) => { @@ -757,6 +867,7 @@ const connectSocket = () => { asCamera: false, }); await connection.setRemoteDescription(new RTCSessionDescription(payload.data)); + await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId); const answer = await connection.createAnswer(); await connection.setLocalDescription(answer); socket.emit('webrtc:signal', { @@ -771,15 +882,32 @@ const connectSocket = () => { if (payload.signalType === 'answer') { if (device.role !== 'camera' || !peerConnection) return; + if (peerSessionId !== payload.streamSessionId || peerTargetDeviceId !== payload.fromDeviceId) { + return; + } + if (peerConnection.signalingState !== 'have-local-offer') { + if (peerConnection.signalingState === 'stable' && peerConnection.remoteDescription?.type === 'answer') { + return; + } + return; + } await peerConnection.setRemoteDescription(new RTCSessionDescription(payload.data)); + await applyQueuedCandidates(peerConnection, payload.streamSessionId, payload.fromDeviceId); addActivity('WebRTC', 'Answer received'); return; } if (payload.signalType === 'candidate') { - if (!peerConnection || !payload.data) return; + if (!payload.data) return; + if (!peerConnection || peerSessionId !== payload.streamSessionId || peerTargetDeviceId !== payload.fromDeviceId) { + queueRemoteCandidate(payload); + return; + } + if (!peerConnection.remoteDescription) { + queueRemoteCandidate(payload); + return; + } await peerConnection.addIceCandidate(new RTCIceCandidate(payload.data)); - addActivity('WebRTC', 'ICE candidate added'); return; } @@ -805,10 +933,14 @@ const startPolling = () => { if (pollInterval) clearInterval(pollInterval); const poller = async () => { - const { device, screen } = store.get(); + const { device, screen, activeStreamSessionId } = store.get(); if (!device) return; if (screen === 'home' && device.role === 'client') { + if (activeStreamSessionId) { + return; + } + const recs = await API.ops.listRecordings().catch(() => ({ recordings: [] })); store.update({ recordings: recs.recordings || [] }); @@ -1128,10 +1260,22 @@ const render = (state) => { const videoEl = $('clientStreamVideo'); if (videoEl && videoEl.srcObject !== remoteClientStream) { videoEl.srcObject = remoteClientStream; - setClientStreamVisibility(true); + setClientStreamMode('video'); void videoEl.play().catch(() => {}); } } + + const imageEl = $('clientStreamImage'); + if (imageEl && !imageEl.dataset.errorBound) { + imageEl.dataset.errorBound = '1'; + imageEl.addEventListener('error', () => { + const videoEl = $('clientStreamVideo'); + if (videoEl) { + videoEl.classList.add('hidden'); + } + setClientStreamMode('unavailable'); + }); + } } const recList = $('recordingsList');