Matiss/Final-Year-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webapp): remove frame fallback from runtime stream path

Browse Source
This commit is contained in:
Matiss Jurevics
2026-03-05 16:10:00 +00:00
parent 19baf76169
commit d03b22a99f
5 changed files with 69 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
WebApp/src/lib/app/api.js
View File

@@ -50,9 +50,7 @@ export const api = {
body: JSON.stringify({ cameraDeviceId, reason: 'on_demand' })
}),
accept: (id) => request(`/streams/${id}/accept`, { method: 'POST', body: JSON.stringify({}) }),
end: (id) => request(`/streams/${id}/end`, { method: 'POST', body: JSON.stringify({ reason: 'completed' }) }),
getPublishCreds: (id) => request(`/streams/${id}/publish-credentials`),
getSubscribeCreds: (id) => request(`/streams/${id}/subscribe-credentials`)
end: (id) => request(`/streams/${id}/end`, { method: 'POST', body: JSON.stringify({ reason: 'completed' }) })
},
events: {
startMotion: () =>
@@ -69,4 +67,3 @@ export const api = {
listNotifications: () => request('/push-notifications/me')
}
};

181
WebApp/src/lib/app/controller.js
View File

@@ -39,12 +39,6 @@ let activeRecordingChunks = [];
let activeRecordingStartedAt = null;
let activeRecordingStreamSessionId = null;
let lastMotionEventId = null;
let frameRelayTimer = null;
let frameRelayStartTimer = null;
let frameCanvas = null;
let frameContext = null;
let hasWebrtcEverConnected = false;
let webrtcConnected = false;
let cameraVideoElement = null;
let clientVideoElement = null;
@@ -153,10 +147,9 @@ const setClientStreamMode = (mode) => {
if (mode === 'unavailable') clientPlaceholderText = 'Stream unavailable';
if (mode === 'none') clientPlaceholderText = 'Select a camera to view';
patchAppState((state) => ({
patchAppState(() => ({
clientStreamMode: mode,
clientPlaceholderText,
clientFallbackFrame: mode === 'image' ? state.clientFallbackFrame : ''
clientPlaceholderText
}));
};
@@ -417,60 +410,6 @@ const closeRecordingModal = () => {
});
};
const stopFrameRelay = () => {
if (frameRelayStartTimer) {
clearTimeout(frameRelayStartTimer);
frameRelayStartTimer = null;
}
if (frameRelayTimer) {
clearInterval(frameRelayTimer);
frameRelayTimer = null;
}
};
const startFrameRelay = async (streamSessionId, toDeviceId) => {
if (!socket || !streamSessionId || !toDeviceId) return;
if (hasWebrtcEverConnected) return;
const ready = await startCameraPreview();
if (!ready) {
throw new Error('Camera permission is required before streaming');
}
if (!cameraVideoElement) return;
stopFrameRelay();
frameRelayTimer = setInterval(() => {
if (webrtcConnected || hasWebrtcEverConnected) return;
if (
!socket ||
cameraVideoElement.readyState < 2 ||
!cameraVideoElement.videoWidth ||
!cameraVideoElement.videoHeight
) {
return;
}
if (!frameCanvas) {
frameCanvas = document.createElement('canvas');
frameContext = frameCanvas.getContext('2d');
}
if (!frameCanvas || !frameContext) return;
frameCanvas.width = cameraVideoElement.videoWidth;
frameCanvas.height = cameraVideoElement.videoHeight;
frameContext.drawImage(cameraVideoElement, 0, 0, frameCanvas.width, frameCanvas.height);
const frame = frameCanvas.toDataURL('image/jpeg', 0.6);
socket.emit('stream:frame', {
toDeviceId,
streamSessionId,
frame,
capturedAt: new Date().toISOString()
});
}, 600);
};
const getPreferredRecordingMimeType = () => {
if (typeof MediaRecorder === 'undefined') return '';
const preferredTypes = ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm'];
@@ -672,8 +611,6 @@ const teardownPeerConnection = (streamSessionId) => {
pendingCandidatesMap.clear();
connectedPeers.clear();
setConnectedStreamSessionIds();
webrtcConnected = false;
hasWebrtcEverConnected = false;
clearClientStream();
return;
}
@@ -752,11 +689,6 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
addActivity('WebRTC', `Peer connected for ${streamSessionId}`);
connectedPeers.add(streamSessionId);
setConnectedStreamSessionIds();
if (asCamera) {
webrtcConnected = true;
hasWebrtcEverConnected = true;
stopFrameRelay();
}
} else if (
connection.connectionState === 'failed' ||
connection.connectionState === 'disconnected' ||
@@ -765,15 +697,9 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
addActivity('WebRTC', `Peer ${connection.connectionState} for ${streamSessionId}`);
connectedPeers.delete(streamSessionId);
setConnectedStreamSessionIds();
if (asCamera) {
if (!hasWebrtcEverConnected) webrtcConnected = false;
if (connection.connectionState === 'failed' || connection.connectionState === 'closed') {
hasWebrtcEverConnected = false;
}
}
if (getAppState().device?.role === 'client' && getAppState().activeStreamSessionId === streamSessionId) {
if (connection.connectionState === 'failed' || connection.connectionState === 'closed') {
clearClientStream();
setClientStreamMode('unavailable');
}
}
}
@@ -940,6 +866,23 @@ const uploadStandaloneMotionRecording = async (captureResult) => {
}
};
const handleCameraStreamRequest = async ({ streamId, requesterDeviceId }) => {
if (!streamId || !requesterDeviceId) {
throw new Error('Missing stream request context');
}
const ready = await startCameraPreview();
if (!ready) {
throw new Error('Camera permission is required before streaming');
}
activeRecordingStreamSessionId = streamId;
await api.streams.accept(streamId);
await startLocalRecording();
await startOfferToClient(streamId, requesterDeviceId);
addActivity('Stream', 'Accepted stream request and started WebRTC offer');
};
const connectSocket = () => {
const { deviceToken } = getAppState();
if (!deviceToken) return;
@@ -957,7 +900,6 @@ const connectSocket = () => {
socket.on('disconnect', () => {
setAppState({ socketConnected: false });
stopFrameRelay();
void stopLocalRecording();
teardownPeerConnection();
setAppState({ activeCameraDeviceId: null, activeStreamSessionId: null });
@@ -968,25 +910,10 @@ const connectSocket = () => {
try {
if (payload.commandType === 'start_stream') {
const streamId = payload.payload.streamSessionId;
const ready = await startCameraPreview();
if (!ready) {
throw new Error('Camera permission is required before streaming');
}
activeRecordingStreamSessionId = streamId;
await api.streams.accept(streamId);
await api.streams.getPublishCreds(streamId);
await startLocalRecording();
if (payload.sourceDeviceId) {
await startOfferToClient(streamId, payload.sourceDeviceId);
frameRelayStartTimer = setTimeout(() => {
if (!webrtcConnected && !hasWebrtcEverConnected) {
void startFrameRelay(streamId, payload.sourceDeviceId);
}
}, 2500);
addActivity('Stream', 'Accepted & Published');
}
await handleCameraStreamRequest({
streamId: payload.payload.streamSessionId,
requesterDeviceId: payload.sourceDeviceId
});
socket.emit('command:ack', { commandId: payload.commandId, status: 'acknowledged' });
}
} catch (error) {
@@ -994,6 +921,20 @@ const connectSocket = () => {
}
});
socket.on('stream:requested', async (payload) => {
if (getAppState().device?.role !== 'camera') return;
try {
await handleCameraStreamRequest({
streamId: payload.streamSessionId,
requesterDeviceId: payload.requesterDeviceId
});
} catch (error) {
console.error('Failed handling direct stream request', error);
pushToast(error.message || 'Failed to accept stream request', 'error');
}
});
socket.on('motion:detected', (payload) => {
const cameraDeviceId = payload.cameraDeviceId || payload.deviceId;
addActivity('Motion', `${getCameraLabel(cameraDeviceId)} has detected movement`);
@@ -1018,37 +959,17 @@ const connectSocket = () => {
setClientStreamMode('connecting');
}
try {
await api.streams.getSubscribeCreds(payload.streamSessionId);
streamTimers.set(
payload.streamSessionId,
setTimeout(() => {
if (!remoteStreams.has(payload.streamSessionId)) {
addActivity('Stream', `No remote video track received for ${payload.streamSessionId}`);
if (getAppState().activeStreamSessionId === payload.streamSessionId) {
setClientStreamMode('unavailable');
}
streamTimers.set(
payload.streamSessionId,
setTimeout(() => {
if (!remoteStreams.has(payload.streamSessionId)) {
addActivity('Stream', `No remote video track received for ${payload.streamSessionId}`);
if (getAppState().activeStreamSessionId === payload.streamSessionId) {
setClientStreamMode('unavailable');
}
}, 6000)
);
} catch (error) {
console.error('Stream connect failed', error);
}
});
socket.on('stream:frame', (payload) => {
if (connectedPeers.has(payload.streamSessionId)) return;
if (!payload?.frame) return;
if (streamTimers.has(payload.streamSessionId)) {
clearTimeout(streamTimers.get(payload.streamSessionId));
streamTimers.delete(payload.streamSessionId);
}
if (payload.streamSessionId === getAppState().activeStreamSessionId) {
setAppState({ clientFallbackFrame: payload.frame });
setClientStreamMode('image');
}
}
}, 6000)
);
});
socket.on('stream:ended', async (payload) => {
@@ -1204,7 +1125,6 @@ const startPolling = () => {
const cleanupConnectionState = async () => {
stopPolling();
stopFrameRelay();
await stopLocalRecording();
teardownPeerConnection();
stopCameraPreview();
@@ -1603,10 +1523,7 @@ const actions = {
openLinkedCameraMenuId: null
});
if (!requestedStreams.has(cameraDeviceId)) {
requestedStreams.add(cameraDeviceId);
void actions.requestStream(cameraDeviceId);
}
void actions.requestStream(cameraDeviceId);
attachClientStreamToElement();
if (!getAppState().activeStreamSessionId) {

1
WebApp/src/lib/app/store.js
View File

@@ -41,7 +41,6 @@ export const createInitialState = () => ({
url: ''
},
clientStreamMode: 'none',
clientFallbackFrame: '',
clientPlaceholderText: 'Select a camera to view',
lastError: null
});

44
WebApp/src/lib/sim/ui/AppChrome.svelte
View File

@@ -30,7 +30,25 @@
id="bottomNav"
class="w-20 lg:w-64 glass-panel border-r border-white/5 flex-col justify-between h-full {isAuthenticated() ? 'flex' : 'hidden'}"
>
<div class="p-6 flex items-center justify-center lg:justify-start gap-3 border-b border-white/5"></div>
<div class="p-4 lg:p-6 border-b border-white/5">
<div id="authStatusBadge" class="flex items-center justify-center lg:justify-start gap-3 text-sm text-gray-300">
<div
class="w-10 h-10 rounded-full bg-gray-800 flex items-center justify-center text-xs font-bold border border-white/10 shrink-0"
>
{userInitial()}
</div>
<div class="hidden lg:block min-w-0">
<p class="font-semibold text-white truncate" title={userEmail()}>{userName()}</p>
<p class="text-xs text-gray-500 truncate" title={userEmail()}>{userEmail()}</p>
</div>
</div>
<div id="connectionStatus" class="mt-3 flex items-center justify-center lg:justify-start gap-2">
<span class="status-dot {$appState.socketConnected ? 'status-online' : 'status-offline'} transition-colors duration-300"></span>
<span class="hidden lg:inline text-[11px] text-gray-400 font-medium tracking-wide uppercase">
{$appState.socketConnected ? 'ONLINE' : 'OFFLINE'}
</span>
</div>
</div>
<nav class="flex-1 py-6 px-3 space-y-2">
<button
@@ -112,30 +130,6 @@
</aside>
<main class="flex-1 flex flex-col h-full overflow-hidden relative">
<header
class="h-16 shrink-0 border-b border-white/5 flex items-center justify-end px-6 relative z-10 glass-panel"
>
<div class="flex items-center gap-6">
<div id="connectionStatus" class="flex items-center gap-2">
<span class="status-dot {$appState.socketConnected ? 'status-online' : 'status-offline'} transition-colors duration-300"></span>
<span class="text-xs text-gray-400 font-medium tracking-wide uppercase">
{$appState.socketConnected ? 'ONLINE' : 'OFFLINE'}
</span>
</div>
<div class="h-6 w-px bg-white/10"></div>
<div id="authStatusBadge" class="flex items-center gap-2 text-sm text-gray-400">
<div
class="w-8 h-8 rounded-full bg-gray-800 flex items-center justify-center text-xs font-bold border border-white/10"
>
{userInitial()}
</div>
<span class="hidden sm:inline" title={userEmail()}>{userName()}</span>
</div>
</div>
</header>
<div class="flex-1 overflow-y-auto p-4 md:p-8 lg:p-10 relative">
{@render children()}
</div>

8
WebApp/src/routes/client/+page.svelte
View File

@@ -185,14 +185,6 @@
playsinline
bind:this={clientVideoElement}
></video>
{#if $appState.clientFallbackFrame}
<img
id="clientStreamImage"
class="absolute inset-0 w-full h-full object-contain {$appState.clientStreamMode === 'image' ? '' : 'hidden'}"
src={$appState.clientFallbackFrame}
alt="Live stream frame"
/>
{/if}
<div id="clientStreamPlaceholder" class="flex flex-col items-center gap-4 animate-pulse {$appState.clientStreamMode === 'none' || $appState.clientStreamMode === 'connecting' || $appState.clientStreamMode === 'unavailable' ? '' : 'hidden'}">
<svg xmlns="http://www.w3.org/2000/svg" class="h-12 w-12 text-gray-700" fill="none" viewBox="0 0 24 24" stroke="currentColor">