From 531fd87197b3cfdbdb2184ad296dd5e99a9bfd55 Mon Sep 17 00:00:00 2001 From: Matiss Jurevics Date: Mon, 13 Apr 2026 10:30:00 +0100 Subject: [PATCH] fix(webapp): bind client streams earlier and show diagnostics --- WebApp/src/lib/app/controller.js | 210 ++++++++++++++++++++++++-- WebApp/src/lib/app/store.js | 1 + WebApp/src/routes/client/+page.svelte | 49 ++++++ 3 files changed, 246 insertions(+), 14 deletions(-) diff --git a/WebApp/src/lib/app/controller.js b/WebApp/src/lib/app/controller.js index b751985..e2530ac 100644 --- a/WebApp/src/lib/app/controller.js +++ b/WebApp/src/lib/app/controller.js @@ -73,6 +73,8 @@ const DEFAULT_CAMERA_CONSTRAINTS = { frameRate: { ideal: 15, max: 24 } }; const SOCKET_HEARTBEAT_INTERVAL_MS = 10_000; +const MAX_STREAM_DIAGNOSTIC_SESSIONS = 12; +const MAX_STREAM_DIAGNOSTIC_ENTRIES = 24; const rtcConfig = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] @@ -444,6 +446,96 @@ const addActivity = (type, message) => { })); }; +const pruneStreamDiagnostics = (diagnostics) => { + const entries = Object.entries(diagnostics || {}); + if (entries.length <= MAX_STREAM_DIAGNOSTIC_SESSIONS) { + return diagnostics; + } + + return Object.fromEntries( + entries + .sort( + (left, right) => + new Date(right[1]?.updatedAt || 0).getTime() - new Date(left[1]?.updatedAt || 0).getTime() + ) + .slice(0, MAX_STREAM_DIAGNOSTIC_SESSIONS) + ); +}; + +const pushStreamDiagnostic = (streamSessionId, stage, message, level = 'info', meta = {}) => { + if (!streamSessionId || !stage || !message) { + return; + } + + const createdAt = new Date().toISOString(); + patchAppState((state) => { + const current = state.streamDiagnostics?.[streamSessionId] ?? { + streamSessionId, + cameraDeviceId: null, + updatedAt: createdAt, + entries: [] + }; + const next = { + ...current, + ...meta, + streamSessionId, + updatedAt: createdAt, + entries: [ + { + id: makeId(), + stage, + message, + level, + createdAt + }, + ...(current.entries || []) + ].slice(0, MAX_STREAM_DIAGNOSTIC_ENTRIES) + }; + + return { + streamDiagnostics: pruneStreamDiagnostics({ + ...(state.streamDiagnostics || {}), + [streamSessionId]: next + }) + }; + }); +}; + +const bindClientStreamSession = (cameraDeviceId, streamSessionId, reason = '') => { + if (!cameraDeviceId || !streamSessionId) { + return; + } + + const currentState = getAppState(); + const nextCameraSessions = { + ...(currentState.cameraSessions || {}), + [cameraDeviceId]: streamSessionId + }; + + const nextState = { + cameraSessions: nextCameraSessions + }; + + if (currentState.activeCameraDeviceId === cameraDeviceId) { + nextState.activeStreamSessionId = streamSessionId; + } + + setAppState(nextState); + + if (reason) { + pushStreamDiagnostic(streamSessionId, 'session', reason, 'info', { cameraDeviceId }); + } + + if (currentState.activeCameraDeviceId === cameraDeviceId) { + if (remoteStreams.has(streamSessionId)) { + attachClientStreamToElement(); + setClientStreamMode('video'); + } else if (getAppState().clientStreamMode !== 'video') { + setClientStreamMode('connecting'); + } + } +}; + const setClientStreamMode = (mode) => { let clientPlaceholderText = 'Select a camera to view'; if (mode === 'connecting') clientPlaceholderText = 'Connecting stream...'; @@ -532,7 +624,14 @@ const attachClientStreamToElement = () => { clientVideoElement.srcObject = stream; } clientVideoElement.muted = true; + pushStreamDiagnostic(activeStreamSessionId, 'viewer', 'Attached remote stream to client video element'); void clientVideoElement.play().catch((error) => { + pushStreamDiagnostic( + activeStreamSessionId, + 'viewer', + `Autoplay blocked: ${error?.name || 'play_failed'}`, + 'error' + ); addActivity('Stream', `Autoplay blocked for ${activeStreamSessionId}: ${error?.name || 'play_failed'}`); }); }; @@ -561,6 +660,7 @@ const primeClientStreamPlayback = (streamSessionId, stream) => { if (hiddenVideoElement.srcObject !== stream) { hiddenVideoElement.srcObject = stream; } + pushStreamDiagnostic(streamSessionId, 'viewer', 'Primed remote stream in hidden video element'); void hiddenVideoElement.play().catch(() => {}); }; @@ -794,11 +894,16 @@ const clearClientStream = () => { clientVideoElement.srcObject = null; } + if (activeStreamSessionId) { + pushStreamDiagnostic(activeStreamSessionId, 'viewer', 'Cleared active client stream viewer'); + } + setClientStreamMode('none'); }; const endClientStreamSession = async (streamSessionId, { teardown = true } = {}) => { if (!streamSessionId) return; + pushStreamDiagnostic(streamSessionId, 'session', 'Ending client stream session'); try { await api.streams.end(streamSessionId); @@ -1163,6 +1268,7 @@ const teardownPeerConnection = (streamSessionId) => { return; } + pushStreamDiagnostic(streamSessionId, 'peer', 'Tearing down peer connection'); if (peerConnections.has(streamSessionId)) { peerConnections.get(streamSessionId)?.close(); peerConnections.delete(streamSessionId); @@ -1215,10 +1321,14 @@ const takeQueuedCandidates = (streamSessionId, fromDeviceId) => { const applyQueuedCandidates = async (connection, streamSessionId, fromDeviceId) => { if (!connection?.remoteDescription) return; const queued = takeQueuedCandidates(streamSessionId, fromDeviceId); + if (queued.length > 0) { + pushStreamDiagnostic(streamSessionId, 'signal', `Applying ${queued.length} queued ICE candidate(s)`); + } for (const candidate of queued) { try { await connection.addIceCandidate(new RTCIceCandidate(candidate.data)); } catch (error) { + pushStreamDiagnostic(streamSessionId, 'error', 'Dropped queued ICE candidate', 'error'); console.warn('Dropping queued ICE candidate', error); } } @@ -1231,9 +1341,15 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera const connection = new RTCPeerConnection(rtcConfig); peerConnections.set(streamSessionId, connection); + pushStreamDiagnostic( + streamSessionId, + 'peer', + asCamera ? 'Created camera-side RTCPeerConnection' : 'Created client-side RTCPeerConnection' + ); connection.onicecandidate = (event) => { if (!socket || !event.candidate) return; + pushStreamDiagnostic(streamSessionId, 'signal', 'Sent ICE candidate'); socket.emit('webrtc:signal', { toDeviceId: targetDeviceId, streamSessionId, @@ -1243,6 +1359,12 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera }; connection.onconnectionstatechange = () => { + pushStreamDiagnostic( + streamSessionId, + 'peer', + `Connection state changed to ${connection.connectionState}`, + connection.connectionState === 'failed' ? 'error' : 'info' + ); if (connection.connectionState === 'connected') { addActivity('WebRTC', `Peer connected for ${streamSessionId}`); connectedPeers.add(streamSessionId); @@ -1263,6 +1385,15 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera } }; + connection.oniceconnectionstatechange = () => { + pushStreamDiagnostic( + streamSessionId, + 'peer', + `ICE connection state changed to ${connection.iceConnectionState}`, + connection.iceConnectionState === 'failed' ? 'error' : 'info' + ); + }; + connection.ontrack = (event) => { if (streamTimers.has(streamSessionId)) { clearTimeout(streamTimers.get(streamSessionId)); @@ -1271,6 +1402,7 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera const [stream] = event.streams; if (!stream) return; + pushStreamDiagnostic(streamSessionId, 'media', 'Received remote media track'); connectedPeers.add(streamSessionId); setConnectedStreamSessionIds(); @@ -1307,6 +1439,7 @@ const startOfferToClient = async (streamSessionId, requesterDeviceId) => { const offer = await connection.createOffer(); await connection.setLocalDescription(offer); + pushStreamDiagnostic(streamSessionId, 'signal', 'Created and sent WebRTC offer'); socket.emit('webrtc:signal', { toDeviceId: requesterDeviceId, streamSessionId, @@ -1626,6 +1759,27 @@ const connectSocket = () => { void stopLocalRecording(); teardownPeerConnection(); setAppState({ activeCameraDeviceId: null, activeStreamSessionId: null }); + patchAppState((state) => ({ + streamDiagnostics: Object.fromEntries( + Object.entries(state.streamDiagnostics || {}).map(([streamSessionId, diagnostic]) => [ + streamSessionId, + { + ...diagnostic, + updatedAt: new Date().toISOString(), + entries: [ + { + id: makeId(), + stage: 'realtime', + message: 'Realtime socket disconnected', + level: 'error', + createdAt: new Date().toISOString() + }, + ...(diagnostic.entries || []) + ].slice(0, MAX_STREAM_DIAGNOSTIC_ENTRIES) + } + ]) + ) + })); applyMotionDetectionReadiness(); }); @@ -1661,6 +1815,15 @@ const connectSocket = () => { }); socket.on('stream:requested', async (payload) => { + if (getAppState().device?.role === 'client') { + bindClientStreamSession( + payload.cameraDeviceId, + payload.streamSessionId, + 'Requester received stream session creation event' + ); + return; + } + if (getAppState().device?.role !== 'camera') return; try { @@ -1688,20 +1851,9 @@ const connectSocket = () => { socket.on('stream:started', async (payload) => { addActivity('Stream', 'Stream is live, connecting...'); + pushStreamDiagnostic(payload.streamSessionId, 'session', 'Received stream:started from realtime gateway'); - const currentState = getAppState(); - const cameraSessions = { ...currentState.cameraSessions, [payload.cameraDeviceId]: payload.streamSessionId }; - setAppState({ cameraSessions }); - - if (payload.cameraDeviceId === currentState.activeCameraDeviceId) { - setAppState({ activeStreamSessionId: payload.streamSessionId }); - if (remoteStreams.has(payload.streamSessionId)) { - attachClientStreamToElement(); - setClientStreamMode('video'); - } else { - setClientStreamMode('connecting'); - } - } + bindClientStreamSession(payload.cameraDeviceId, payload.streamSessionId); streamTimers.set( payload.streamSessionId, @@ -1719,6 +1871,7 @@ const connectSocket = () => { socket.on('stream:ended', async (payload) => { if (!payload?.streamSessionId) return; const streamSessionId = payload.streamSessionId; + pushStreamDiagnostic(streamSessionId, 'session', 'Received stream:ended event'); teardownPeerConnection(streamSessionId); if (streamSessionId === getAppState().activeStreamSessionId) { @@ -1747,6 +1900,7 @@ const connectSocket = () => { try { if (payload.signalType === 'offer') { if (device.role !== 'client') return; + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received WebRTC offer'); addActivity('WebRTC', 'Offer received'); const connection = await ensurePeerConnection({ streamSessionId: payload.streamSessionId, @@ -1754,15 +1908,18 @@ const connectSocket = () => { asCamera: false }); await connection.setRemoteDescription(new RTCSessionDescription(payload.data)); + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote offer'); await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId); const answer = await connection.createAnswer(); await connection.setLocalDescription(answer); + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Created local answer'); socket.emit('webrtc:signal', { toDeviceId: payload.fromDeviceId, streamSessionId: payload.streamSessionId, signalType: 'answer', data: answer }); + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Sent WebRTC answer'); addActivity('WebRTC', 'Answer sent'); return; } @@ -1770,6 +1927,7 @@ const connectSocket = () => { if (payload.signalType === 'answer') { const connection = peerConnections.get(payload.streamSessionId); if (device.role !== 'camera' || !connection) return; + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received WebRTC answer'); if (connection.signalingState !== 'have-local-offer') { if (connection.signalingState === 'stable' && connection.remoteDescription?.type === 'answer') { @@ -1778,6 +1936,7 @@ const connectSocket = () => { return; } await connection.setRemoteDescription(new RTCSessionDescription(payload.data)); + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote answer'); await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId); addActivity('WebRTC', 'Answer received and applied'); return; @@ -1787,14 +1946,17 @@ const connectSocket = () => { if (!payload.data) return; const connection = peerConnections.get(payload.streamSessionId); if (!connection || !connection.remoteDescription) { + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Queued remote ICE candidate'); queueRemoteCandidate(payload); return; } await connection.addIceCandidate(new RTCIceCandidate(payload.data)); + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote ICE candidate'); return; } if (payload.signalType === 'hangup') { + pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received remote hangup'); teardownPeerConnection(payload.streamSessionId); if (getAppState().activeStreamSessionId === payload.streamSessionId) { setAppState({ activeStreamSessionId: null }); @@ -1802,6 +1964,12 @@ const connectSocket = () => { addActivity('Stream', 'Remote stream ended'); } } catch (error) { + pushStreamDiagnostic( + payload.streamSessionId, + 'error', + error?.message || 'Failed handling WebRTC signal', + 'error' + ); console.error('Failed handling WebRTC signal', error); pushToast('WebRTC negotiation failed', 'error'); } @@ -1809,6 +1977,9 @@ const connectSocket = () => { socket.on('error:webrtc_signal', (payload) => { const message = payload?.message || 'WebRTC signaling error'; + if (payload?.streamSessionId) { + pushStreamDiagnostic(payload.streamSessionId, 'error', message, 'error'); + } addActivity('WebRTC', message); pushToast(message, 'error'); }); @@ -2261,7 +2432,17 @@ const actions = { try { requestedStreams.add(cameraDeviceId); - await api.streams.request(cameraDeviceId); + const result = await api.streams.request(cameraDeviceId); + requestedStreams.delete(cameraDeviceId); + const streamSessionId = result?.streamSession?.id; + if (streamSessionId) { + bindClientStreamSession( + cameraDeviceId, + streamSessionId, + 'Bound requested stream session from HTTP response' + ); + pushStreamDiagnostic(streamSessionId, 'request', 'Backend accepted stream request'); + } } catch (error) { requestedStreams.delete(cameraDeviceId); pushToast(error.message || 'Failed to request stream', 'error'); @@ -2292,6 +2473,7 @@ const actions = { } if (reusableSessionId) { + pushStreamDiagnostic(reusableSessionId, 'viewer', 'Reusing existing stream session for selected camera'); attachClientStreamToElement(); setClientStreamMode(remoteStreams.has(reusableSessionId) ? 'video' : 'connecting'); return; diff --git a/WebApp/src/lib/app/store.js b/WebApp/src/lib/app/store.js index 080f9f3..6f04072 100644 --- a/WebApp/src/lib/app/store.js +++ b/WebApp/src/lib/app/store.js @@ -23,6 +23,7 @@ export const createInitialState = () => ({ activityLog: [], cameraSessions: {}, connectedStreamSessionIds: [], + streamDiagnostics: {}, loading: true, isRegistering: false, authForm: { diff --git a/WebApp/src/routes/client/+page.svelte b/WebApp/src/routes/client/+page.svelte index 240507a..9a7fbd3 100644 --- a/WebApp/src/routes/client/+page.svelte +++ b/WebApp/src/routes/client/+page.svelte @@ -29,6 +29,30 @@ const linked = $appState.linkedCameras.find((camera) => camera.cameraDeviceId === $appState.activeCameraDeviceId); return linked?.cameraName || linked?.cameraDeviceId || 'Live Feed Viewer'; }; + + const activeStreamSessionId = () => { + if ($appState.activeStreamSessionId) { + return $appState.activeStreamSessionId; + } + if (!$appState.activeCameraDeviceId) { + return null; + } + return $appState.cameraSessions?.[$appState.activeCameraDeviceId] ?? null; + }; + + const activeStreamDiagnostics = () => { + const streamSessionId = activeStreamSessionId(); + if (!streamSessionId) { + return null; + } + return $appState.streamDiagnostics?.[streamSessionId] ?? null; + }; + + const diagnosticLevelClass = (level: string) => { + if (level === 'error') return 'border-red-500/20 bg-red-500/10 text-red-200'; + if (level === 'success') return 'border-emerald-500/20 bg-emerald-500/10 text-emerald-200'; + return 'border-white/10 bg-white/5 text-gray-300'; + }; @@ -240,6 +264,31 @@ + + {#if activeStreamDiagnostics()} +
+
+ Live Diagnostics + + {activeStreamSessionId()?.slice(0, 8)} + + {#if isCameraLive($appState.activeCameraDeviceId)} + Peer connected + {/if} +
+
+ {#each activeStreamDiagnostics().entries.slice(0, 6) as entry (entry.id)} +
+
+ {entry.stage} + {new Date(entry.createdAt).toLocaleTimeString()} +
+

{entry.message}

+
+ {/each} +
+
+ {/if}