Matiss/Final-Year-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webapp): remove frame fallback from runtime stream path

Browse Source
This commit is contained in:
Matiss Jurevics
2026-03-05 16:10:00 +00:00
parent 19baf76169
commit d03b22a99f
5 changed files with 69 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
WebApp/src/lib/app/api.js
View File

@@ -50,9 +50,7 @@ export const api = {
body: JSON.stringify({ cameraDeviceId, reason: 'on_demand' }) body: JSON.stringify({ cameraDeviceId, reason: 'on_demand' })
}), }),
accept: (id) => request(`/streams/${id}/accept`, { method: 'POST', body: JSON.stringify({}) }), accept: (id) => request(`/streams/${id}/accept`, { method: 'POST', body: JSON.stringify({}) }),
end: (id) => request(`/streams/${id}/end`, { method: 'POST', body: JSON.stringify({ reason: 'completed' }) }), end: (id) => request(`/streams/${id}/end`, { method: 'POST', body: JSON.stringify({ reason: 'completed' }) })
getPublishCreds: (id) => request(`/streams/${id}/publish-credentials`),
getSubscribeCreds: (id) => request(`/streams/${id}/subscribe-credentials`)
}, },
events: { events: {
startMotion: () => startMotion: () =>
@@ -69,4 +67,3 @@ export const api = {
listNotifications: () => request('/push-notifications/me') listNotifications: () => request('/push-notifications/me')
} }
}; };

181
WebApp/src/lib/app/controller.js
View File

@@ -39,12 +39,6 @@ let activeRecordingChunks = [];
let activeRecordingStartedAt = null; let activeRecordingStartedAt = null;
let activeRecordingStreamSessionId = null; let activeRecordingStreamSessionId = null;
let lastMotionEventId = null; let lastMotionEventId = null;
let frameRelayTimer = null;
let frameRelayStartTimer = null;
let frameCanvas = null;
let frameContext = null;
let hasWebrtcEverConnected = false;
let webrtcConnected = false;
let cameraVideoElement = null; let cameraVideoElement = null;
let clientVideoElement = null; let clientVideoElement = null;
@@ -153,10 +147,9 @@ const setClientStreamMode = (mode) => {
if (mode === 'unavailable') clientPlaceholderText = 'Stream unavailable'; if (mode === 'unavailable') clientPlaceholderText = 'Stream unavailable';
if (mode === 'none') clientPlaceholderText = 'Select a camera to view'; if (mode === 'none') clientPlaceholderText = 'Select a camera to view';
patchAppState((state) => ({ patchAppState(() => ({
clientStreamMode: mode, clientStreamMode: mode,
clientPlaceholderText, clientPlaceholderText
clientFallbackFrame: mode === 'image' ? state.clientFallbackFrame : ''
})); }));
}; };
@@ -417,60 +410,6 @@ const closeRecordingModal = () => {
}); });
}; };
const stopFrameRelay = () => {
if (frameRelayStartTimer) {
clearTimeout(frameRelayStartTimer);
frameRelayStartTimer = null;
}
if (frameRelayTimer) {
clearInterval(frameRelayTimer);
frameRelayTimer = null;
}
};
const startFrameRelay = async (streamSessionId, toDeviceId) => {
if (!socket || !streamSessionId || !toDeviceId) return;
if (hasWebrtcEverConnected) return;
const ready = await startCameraPreview();
if (!ready) {
throw new Error('Camera permission is required before streaming');
}
if (!cameraVideoElement) return;
stopFrameRelay();
frameRelayTimer = setInterval(() => {
if (webrtcConnected || hasWebrtcEverConnected) return;
if (
!socket ||
cameraVideoElement.readyState < 2 ||
!cameraVideoElement.videoWidth ||
!cameraVideoElement.videoHeight
) {
return;
}
if (!frameCanvas) {
frameCanvas = document.createElement('canvas');
frameContext = frameCanvas.getContext('2d');
}
if (!frameCanvas || !frameContext) return;
frameCanvas.width = cameraVideoElement.videoWidth;
frameCanvas.height = cameraVideoElement.videoHeight;
frameContext.drawImage(cameraVideoElement, 0, 0, frameCanvas.width, frameCanvas.height);
const frame = frameCanvas.toDataURL('image/jpeg', 0.6);
socket.emit('stream:frame', {
toDeviceId,
streamSessionId,
frame,
capturedAt: new Date().toISOString()
});
}, 600);
};
const getPreferredRecordingMimeType = () => { const getPreferredRecordingMimeType = () => {
if (typeof MediaRecorder === 'undefined') return ''; if (typeof MediaRecorder === 'undefined') return '';
const preferredTypes = ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm']; const preferredTypes = ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm'];
@@ -672,8 +611,6 @@ const teardownPeerConnection = (streamSessionId) => {
pendingCandidatesMap.clear(); pendingCandidatesMap.clear();
connectedPeers.clear(); connectedPeers.clear();
setConnectedStreamSessionIds(); setConnectedStreamSessionIds();
webrtcConnected = false;
hasWebrtcEverConnected = false;
clearClientStream(); clearClientStream();
return; return;
} }
@@ -752,11 +689,6 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
addActivity('WebRTC', `Peer connected for ${streamSessionId}`); addActivity('WebRTC', `Peer connected for ${streamSessionId}`);
connectedPeers.add(streamSessionId); connectedPeers.add(streamSessionId);
setConnectedStreamSessionIds(); setConnectedStreamSessionIds();
if (asCamera) {
webrtcConnected = true;
hasWebrtcEverConnected = true;
stopFrameRelay();
}
} else if ( } else if (
connection.connectionState === 'failed' || connection.connectionState === 'failed' ||
connection.connectionState === 'disconnected' || connection.connectionState === 'disconnected' ||
@@ -765,15 +697,9 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
addActivity('WebRTC', `Peer ${connection.connectionState} for ${streamSessionId}`); addActivity('WebRTC', `Peer ${connection.connectionState} for ${streamSessionId}`);
connectedPeers.delete(streamSessionId); connectedPeers.delete(streamSessionId);
setConnectedStreamSessionIds(); setConnectedStreamSessionIds();
if (asCamera) {
if (!hasWebrtcEverConnected) webrtcConnected = false;
if (connection.connectionState === 'failed' || connection.connectionState === 'closed') {
hasWebrtcEverConnected = false;
}
}
if (getAppState().device?.role === 'client' && getAppState().activeStreamSessionId === streamSessionId) { if (getAppState().device?.role === 'client' && getAppState().activeStreamSessionId === streamSessionId) {
if (connection.connectionState === 'failed' || connection.connectionState === 'closed') { if (connection.connectionState === 'failed' || connection.connectionState === 'closed') {
clearClientStream(); setClientStreamMode('unavailable');
} }
} }
} }
@@ -940,6 +866,23 @@ const uploadStandaloneMotionRecording = async (captureResult) => {
} }
}; };
const handleCameraStreamRequest = async ({ streamId, requesterDeviceId }) => {
if (!streamId || !requesterDeviceId) {
throw new Error('Missing stream request context');
}
const ready = await startCameraPreview();
if (!ready) {
throw new Error('Camera permission is required before streaming');
}
activeRecordingStreamSessionId = streamId;
await api.streams.accept(streamId);
await startLocalRecording();
await startOfferToClient(streamId, requesterDeviceId);
addActivity('Stream', 'Accepted stream request and started WebRTC offer');
};
const connectSocket = () => { const connectSocket = () => {
const { deviceToken } = getAppState(); const { deviceToken } = getAppState();
if (!deviceToken) return; if (!deviceToken) return;
@@ -957,7 +900,6 @@ const connectSocket = () => {
socket.on('disconnect', () => { socket.on('disconnect', () => {
setAppState({ socketConnected: false }); setAppState({ socketConnected: false });
stopFrameRelay();
void stopLocalRecording(); void stopLocalRecording();
teardownPeerConnection(); teardownPeerConnection();
setAppState({ activeCameraDeviceId: null, activeStreamSessionId: null }); setAppState({ activeCameraDeviceId: null, activeStreamSessionId: null });
@@ -968,25 +910,10 @@ const connectSocket = () => {
try { try {
if (payload.commandType === 'start_stream') { if (payload.commandType === 'start_stream') {
const streamId = payload.payload.streamSessionId; await handleCameraStreamRequest({
const ready = await startCameraPreview(); streamId: payload.payload.streamSessionId,
if (!ready) { requesterDeviceId: payload.sourceDeviceId
throw new Error('Camera permission is required before streaming'); });
}
activeRecordingStreamSessionId = streamId;
await api.streams.accept(streamId);
await api.streams.getPublishCreds(streamId);
await startLocalRecording();
if (payload.sourceDeviceId) {
await startOfferToClient(streamId, payload.sourceDeviceId);
frameRelayStartTimer = setTimeout(() => {
if (!webrtcConnected && !hasWebrtcEverConnected) {
void startFrameRelay(streamId, payload.sourceDeviceId);
}
}, 2500);
addActivity('Stream', 'Accepted & Published');
}
socket.emit('command:ack', { commandId: payload.commandId, status: 'acknowledged' }); socket.emit('command:ack', { commandId: payload.commandId, status: 'acknowledged' });
} }
} catch (error) { } catch (error) {
@@ -994,6 +921,20 @@ const connectSocket = () => {
} }
}); });
socket.on('stream:requested', async (payload) => {
if (getAppState().device?.role !== 'camera') return;
try {
await handleCameraStreamRequest({
streamId: payload.streamSessionId,
requesterDeviceId: payload.requesterDeviceId
});
} catch (error) {
console.error('Failed handling direct stream request', error);
pushToast(error.message || 'Failed to accept stream request', 'error');
}
});
socket.on('motion:detected', (payload) => { socket.on('motion:detected', (payload) => {
const cameraDeviceId = payload.cameraDeviceId || payload.deviceId; const cameraDeviceId = payload.cameraDeviceId || payload.deviceId;
addActivity('Motion', `${getCameraLabel(cameraDeviceId)} has detected movement`); addActivity('Motion', `${getCameraLabel(cameraDeviceId)} has detected movement`);
@@ -1018,37 +959,17 @@ const connectSocket = () => {
setClientStreamMode('connecting'); setClientStreamMode('connecting');
} }
try { streamTimers.set(
await api.streams.getSubscribeCreds(payload.streamSessionId); payload.streamSessionId,
streamTimers.set( setTimeout(() => {
payload.streamSessionId, if (!remoteStreams.has(payload.streamSessionId)) {
setTimeout(() => { addActivity('Stream', `No remote video track received for ${payload.streamSessionId}`);
if (!remoteStreams.has(payload.streamSessionId)) { if (getAppState().activeStreamSessionId === payload.streamSessionId) {
addActivity('Stream', `No remote video track received for ${payload.streamSessionId}`); setClientStreamMode('unavailable');
if (getAppState().activeStreamSessionId === payload.streamSessionId) {
setClientStreamMode('unavailable');
}
} }
}, 6000) }
); }, 6000)
} catch (error) { );
console.error('Stream connect failed', error);
}
});
socket.on('stream:frame', (payload) => {
if (connectedPeers.has(payload.streamSessionId)) return;
if (!payload?.frame) return;
if (streamTimers.has(payload.streamSessionId)) {
clearTimeout(streamTimers.get(payload.streamSessionId));
streamTimers.delete(payload.streamSessionId);
}
if (payload.streamSessionId === getAppState().activeStreamSessionId) {
setAppState({ clientFallbackFrame: payload.frame });
setClientStreamMode('image');
}
}); });
socket.on('stream:ended', async (payload) => { socket.on('stream:ended', async (payload) => {
@@ -1204,7 +1125,6 @@ const startPolling = () => {
const cleanupConnectionState = async () => { const cleanupConnectionState = async () => {
stopPolling(); stopPolling();
stopFrameRelay();
await stopLocalRecording(); await stopLocalRecording();
teardownPeerConnection(); teardownPeerConnection();
stopCameraPreview(); stopCameraPreview();
@@ -1603,10 +1523,7 @@ const actions = {
openLinkedCameraMenuId: null openLinkedCameraMenuId: null
}); });
if (!requestedStreams.has(cameraDeviceId)) { void actions.requestStream(cameraDeviceId);
requestedStreams.add(cameraDeviceId);
void actions.requestStream(cameraDeviceId);
}
attachClientStreamToElement(); attachClientStreamToElement();
if (!getAppState().activeStreamSessionId) { if (!getAppState().activeStreamSessionId) {

1
WebApp/src/lib/app/store.js
View File

@@ -41,7 +41,6 @@ export const createInitialState = () => ({
url: '' url: ''
}, },
clientStreamMode: 'none', clientStreamMode: 'none',
clientFallbackFrame: '',
clientPlaceholderText: 'Select a camera to view', clientPlaceholderText: 'Select a camera to view',
lastError: null lastError: null
}); });

44
WebApp/src/lib/sim/ui/AppChrome.svelte
View File

@@ -30,7 +30,25 @@
id="bottomNav" id="bottomNav"
class="w-20 lg:w-64 glass-panel border-r border-white/5 flex-col justify-between h-full {isAuthenticated() ? 'flex' : 'hidden'}" class="w-20 lg:w-64 glass-panel border-r border-white/5 flex-col justify-between h-full {isAuthenticated() ? 'flex' : 'hidden'}"
> >
<div class="p-6 flex items-center justify-center lg:justify-start gap-3 border-b border-white/5"></div> <div class="p-4 lg:p-6 border-b border-white/5">
<div id="authStatusBadge" class="flex items-center justify-center lg:justify-start gap-3 text-sm text-gray-300">
<div
class="w-10 h-10 rounded-full bg-gray-800 flex items-center justify-center text-xs font-bold border border-white/10 shrink-0"
>
{userInitial()}
</div>
<div class="hidden lg:block min-w-0">
<p class="font-semibold text-white truncate" title={userEmail()}>{userName()}</p>
<p class="text-xs text-gray-500 truncate" title={userEmail()}>{userEmail()}</p>
</div>
</div>
<div id="connectionStatus" class="mt-3 flex items-center justify-center lg:justify-start gap-2">
<span class="status-dot {$appState.socketConnected ? 'status-online' : 'status-offline'} transition-colors duration-300"></span>
<span class="hidden lg:inline text-[11px] text-gray-400 font-medium tracking-wide uppercase">
{$appState.socketConnected ? 'ONLINE' : 'OFFLINE'}
</span>
</div>
</div>
<nav class="flex-1 py-6 px-3 space-y-2"> <nav class="flex-1 py-6 px-3 space-y-2">
<button <button
@@ -112,30 +130,6 @@
</aside> </aside>
<main class="flex-1 flex flex-col h-full overflow-hidden relative"> <main class="flex-1 flex flex-col h-full overflow-hidden relative">
<header
class="h-16 shrink-0 border-b border-white/5 flex items-center justify-end px-6 relative z-10 glass-panel"
>
<div class="flex items-center gap-6">
<div id="connectionStatus" class="flex items-center gap-2">
<span class="status-dot {$appState.socketConnected ? 'status-online' : 'status-offline'} transition-colors duration-300"></span>
<span class="text-xs text-gray-400 font-medium tracking-wide uppercase">
{$appState.socketConnected ? 'ONLINE' : 'OFFLINE'}
</span>
</div>
<div class="h-6 w-px bg-white/10"></div>
<div id="authStatusBadge" class="flex items-center gap-2 text-sm text-gray-400">
<div
class="w-8 h-8 rounded-full bg-gray-800 flex items-center justify-center text-xs font-bold border border-white/10"
>
{userInitial()}
</div>
<span class="hidden sm:inline" title={userEmail()}>{userName()}</span>
</div>
</div>
</header>
<div class="flex-1 overflow-y-auto p-4 md:p-8 lg:p-10 relative"> <div class="flex-1 overflow-y-auto p-4 md:p-8 lg:p-10 relative">
{@render children()} {@render children()}
</div> </div>

8
WebApp/src/routes/client/+page.svelte
View File

@@ -185,14 +185,6 @@
playsinline playsinline
bind:this={clientVideoElement} bind:this={clientVideoElement}
></video> ></video>
{#if $appState.clientFallbackFrame}
<img
id="clientStreamImage"
class="absolute inset-0 w-full h-full object-contain {$appState.clientStreamMode === 'image' ? '' : 'hidden'}"
src={$appState.clientFallbackFrame}
alt="Live stream frame"
/>
{/if}
<div id="clientStreamPlaceholder" class="flex flex-col items-center gap-4 animate-pulse {$appState.clientStreamMode === 'none' || $appState.clientStreamMode === 'connecting' || $appState.clientStreamMode === 'unavailable' ? '' : 'hidden'}"> <div id="clientStreamPlaceholder" class="flex flex-col items-center gap-4 animate-pulse {$appState.clientStreamMode === 'none' || $appState.clientStreamMode === 'connecting' || $appState.clientStreamMode === 'unavailable' ? '' : 'hidden'}">
<svg xmlns="http://www.w3.org/2000/svg" class="h-12 w-12 text-gray-700" fill="none" viewBox="0 0 24 24" stroke="currentColor"> <svg xmlns="http://www.w3.org/2000/svg" class="h-12 w-12 text-gray-700" fill="none" viewBox="0 0 24 24" stroke="currentColor">