Matiss/Final-Year-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(webapp): bind client streams earlier and show diagnostics

Browse Source
This commit is contained in:
Matiss Jurevics
2026-04-13 10:30:00 +01:00
parent f7c3eafb54
commit 531fd87197
3 changed files with 246 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
WebApp/src/lib/app/controller.js
View File

@@ -73,6 +73,8 @@ const DEFAULT_CAMERA_CONSTRAINTS = {
frameRate: { ideal: 15, max: 24 }
};
const SOCKET_HEARTBEAT_INTERVAL_MS = 10_000;
const MAX_STREAM_DIAGNOSTIC_SESSIONS = 12;
const MAX_STREAM_DIAGNOSTIC_ENTRIES = 24;
const rtcConfig = {
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
@@ -444,6 +446,96 @@ const addActivity = (type, message) => {
}));
};
const pruneStreamDiagnostics = (diagnostics) => {
const entries = Object.entries(diagnostics || {});
if (entries.length <= MAX_STREAM_DIAGNOSTIC_SESSIONS) {
return diagnostics;
}
return Object.fromEntries(
entries
.sort(
(left, right) =>
new Date(right[1]?.updatedAt || 0).getTime() - new Date(left[1]?.updatedAt || 0).getTime()
)
.slice(0, MAX_STREAM_DIAGNOSTIC_SESSIONS)
);
};
const pushStreamDiagnostic = (streamSessionId, stage, message, level = 'info', meta = {}) => {
if (!streamSessionId || !stage || !message) {
return;
}
const createdAt = new Date().toISOString();
patchAppState((state) => {
const current = state.streamDiagnostics?.[streamSessionId] ?? {
streamSessionId,
cameraDeviceId: null,
updatedAt: createdAt,
entries: []
};
const next = {
...current,
...meta,
streamSessionId,
updatedAt: createdAt,
entries: [
{
id: makeId(),
stage,
message,
level,
createdAt
},
...(current.entries || [])
].slice(0, MAX_STREAM_DIAGNOSTIC_ENTRIES)
};
return {
streamDiagnostics: pruneStreamDiagnostics({
...(state.streamDiagnostics || {}),
[streamSessionId]: next
})
};
});
};
const bindClientStreamSession = (cameraDeviceId, streamSessionId, reason = '') => {
if (!cameraDeviceId || !streamSessionId) {
return;
}
const currentState = getAppState();
const nextCameraSessions = {
...(currentState.cameraSessions || {}),
[cameraDeviceId]: streamSessionId
};
const nextState = {
cameraSessions: nextCameraSessions
};
if (currentState.activeCameraDeviceId === cameraDeviceId) {
nextState.activeStreamSessionId = streamSessionId;
}
setAppState(nextState);
if (reason) {
pushStreamDiagnostic(streamSessionId, 'session', reason, 'info', { cameraDeviceId });
}
if (currentState.activeCameraDeviceId === cameraDeviceId) {
if (remoteStreams.has(streamSessionId)) {
attachClientStreamToElement();
setClientStreamMode('video');
} else if (getAppState().clientStreamMode !== 'video') {
setClientStreamMode('connecting');
}
}
};
const setClientStreamMode = (mode) => {
let clientPlaceholderText = 'Select a camera to view';
if (mode === 'connecting') clientPlaceholderText = 'Connecting stream...';
@@ -532,7 +624,14 @@ const attachClientStreamToElement = () => {
clientVideoElement.srcObject = stream;
}
clientVideoElement.muted = true;
pushStreamDiagnostic(activeStreamSessionId, 'viewer', 'Attached remote stream to client video element');
void clientVideoElement.play().catch((error) => {
pushStreamDiagnostic(
activeStreamSessionId,
'viewer',
`Autoplay blocked: ${error?.name || 'play_failed'}`,
'error'
);
addActivity('Stream', `Autoplay blocked for ${activeStreamSessionId}: ${error?.name || 'play_failed'}`);
});
};
@@ -561,6 +660,7 @@ const primeClientStreamPlayback = (streamSessionId, stream) => {
if (hiddenVideoElement.srcObject !== stream) {
hiddenVideoElement.srcObject = stream;
}
pushStreamDiagnostic(streamSessionId, 'viewer', 'Primed remote stream in hidden video element');
void hiddenVideoElement.play().catch(() => {});
};
@@ -794,11 +894,16 @@ const clearClientStream = () => {
clientVideoElement.srcObject = null;
}
if (activeStreamSessionId) {
pushStreamDiagnostic(activeStreamSessionId, 'viewer', 'Cleared active client stream viewer');
}
setClientStreamMode('none');
};
const endClientStreamSession = async (streamSessionId, { teardown = true } = {}) => {
if (!streamSessionId) return;
pushStreamDiagnostic(streamSessionId, 'session', 'Ending client stream session');
try {
await api.streams.end(streamSessionId);
@@ -1163,6 +1268,7 @@ const teardownPeerConnection = (streamSessionId) => {
return;
}
pushStreamDiagnostic(streamSessionId, 'peer', 'Tearing down peer connection');
if (peerConnections.has(streamSessionId)) {
peerConnections.get(streamSessionId)?.close();
peerConnections.delete(streamSessionId);
@@ -1215,10 +1321,14 @@ const takeQueuedCandidates = (streamSessionId, fromDeviceId) => {
const applyQueuedCandidates = async (connection, streamSessionId, fromDeviceId) => {
if (!connection?.remoteDescription) return;
const queued = takeQueuedCandidates(streamSessionId, fromDeviceId);
if (queued.length > 0) {
pushStreamDiagnostic(streamSessionId, 'signal', `Applying ${queued.length} queued ICE candidate(s)`);
}
for (const candidate of queued) {
try {
await connection.addIceCandidate(new RTCIceCandidate(candidate.data));
} catch (error) {
pushStreamDiagnostic(streamSessionId, 'error', 'Dropped queued ICE candidate', 'error');
console.warn('Dropping queued ICE candidate', error);
}
}
@@ -1231,9 +1341,15 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
const connection = new RTCPeerConnection(rtcConfig);
peerConnections.set(streamSessionId, connection);
pushStreamDiagnostic(
streamSessionId,
'peer',
asCamera ? 'Created camera-side RTCPeerConnection' : 'Created client-side RTCPeerConnection'
);
connection.onicecandidate = (event) => {
if (!socket || !event.candidate) return;
pushStreamDiagnostic(streamSessionId, 'signal', 'Sent ICE candidate');
socket.emit('webrtc:signal', {
toDeviceId: targetDeviceId,
streamSessionId,
@@ -1243,6 +1359,12 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
};
connection.onconnectionstatechange = () => {
pushStreamDiagnostic(
streamSessionId,
'peer',
`Connection state changed to ${connection.connectionState}`,
connection.connectionState === 'failed' ? 'error' : 'info'
);
if (connection.connectionState === 'connected') {
addActivity('WebRTC', `Peer connected for ${streamSessionId}`);
connectedPeers.add(streamSessionId);
@@ -1263,6 +1385,15 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
}
};
connection.oniceconnectionstatechange = () => {
pushStreamDiagnostic(
streamSessionId,
'peer',
`ICE connection state changed to ${connection.iceConnectionState}`,
connection.iceConnectionState === 'failed' ? 'error' : 'info'
);
};
connection.ontrack = (event) => {
if (streamTimers.has(streamSessionId)) {
clearTimeout(streamTimers.get(streamSessionId));
@@ -1271,6 +1402,7 @@ const ensurePeerConnection = async ({ streamSessionId, targetDeviceId, asCamera
const [stream] = event.streams;
if (!stream) return;
pushStreamDiagnostic(streamSessionId, 'media', 'Received remote media track');
connectedPeers.add(streamSessionId);
setConnectedStreamSessionIds();
@@ -1307,6 +1439,7 @@ const startOfferToClient = async (streamSessionId, requesterDeviceId) => {
const offer = await connection.createOffer();
await connection.setLocalDescription(offer);
pushStreamDiagnostic(streamSessionId, 'signal', 'Created and sent WebRTC offer');
socket.emit('webrtc:signal', {
toDeviceId: requesterDeviceId,
streamSessionId,
@@ -1626,6 +1759,27 @@ const connectSocket = () => {
void stopLocalRecording();
teardownPeerConnection();
setAppState({ activeCameraDeviceId: null, activeStreamSessionId: null });
patchAppState((state) => ({
streamDiagnostics: Object.fromEntries(
Object.entries(state.streamDiagnostics || {}).map(([streamSessionId, diagnostic]) => [
streamSessionId,
{
...diagnostic,
updatedAt: new Date().toISOString(),
entries: [
{
id: makeId(),
stage: 'realtime',
message: 'Realtime socket disconnected',
level: 'error',
createdAt: new Date().toISOString()
},
...(diagnostic.entries || [])
].slice(0, MAX_STREAM_DIAGNOSTIC_ENTRIES)
}
])
)
}));
applyMotionDetectionReadiness();
});
@@ -1661,6 +1815,15 @@ const connectSocket = () => {
});
socket.on('stream:requested', async (payload) => {
if (getAppState().device?.role === 'client') {
bindClientStreamSession(
payload.cameraDeviceId,
payload.streamSessionId,
'Requester received stream session creation event'
);
return;
}
if (getAppState().device?.role !== 'camera') return;
try {
@@ -1688,20 +1851,9 @@ const connectSocket = () => {
socket.on('stream:started', async (payload) => {
addActivity('Stream', 'Stream is live, connecting...');
pushStreamDiagnostic(payload.streamSessionId, 'session', 'Received stream:started from realtime gateway');
const currentState = getAppState();
const cameraSessions = { ...currentState.cameraSessions, [payload.cameraDeviceId]: payload.streamSessionId };
setAppState({ cameraSessions });
if (payload.cameraDeviceId === currentState.activeCameraDeviceId) {
setAppState({ activeStreamSessionId: payload.streamSessionId });
if (remoteStreams.has(payload.streamSessionId)) {
attachClientStreamToElement();
setClientStreamMode('video');
} else {
setClientStreamMode('connecting');
}
}
bindClientStreamSession(payload.cameraDeviceId, payload.streamSessionId);
streamTimers.set(
payload.streamSessionId,
@@ -1719,6 +1871,7 @@ const connectSocket = () => {
socket.on('stream:ended', async (payload) => {
if (!payload?.streamSessionId) return;
const streamSessionId = payload.streamSessionId;
pushStreamDiagnostic(streamSessionId, 'session', 'Received stream:ended event');
teardownPeerConnection(streamSessionId);
if (streamSessionId === getAppState().activeStreamSessionId) {
@@ -1747,6 +1900,7 @@ const connectSocket = () => {
try {
if (payload.signalType === 'offer') {
if (device.role !== 'client') return;
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received WebRTC offer');
addActivity('WebRTC', 'Offer received');
const connection = await ensurePeerConnection({
streamSessionId: payload.streamSessionId,
@@ -1754,15 +1908,18 @@ const connectSocket = () => {
asCamera: false
});
await connection.setRemoteDescription(new RTCSessionDescription(payload.data));
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote offer');
await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId);
const answer = await connection.createAnswer();
await connection.setLocalDescription(answer);
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Created local answer');
socket.emit('webrtc:signal', {
toDeviceId: payload.fromDeviceId,
streamSessionId: payload.streamSessionId,
signalType: 'answer',
data: answer
});
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Sent WebRTC answer');
addActivity('WebRTC', 'Answer sent');
return;
}
@@ -1770,6 +1927,7 @@ const connectSocket = () => {
if (payload.signalType === 'answer') {
const connection = peerConnections.get(payload.streamSessionId);
if (device.role !== 'camera' || !connection) return;
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received WebRTC answer');
if (connection.signalingState !== 'have-local-offer') {
if (connection.signalingState === 'stable' && connection.remoteDescription?.type === 'answer') {
@@ -1778,6 +1936,7 @@ const connectSocket = () => {
return;
}
await connection.setRemoteDescription(new RTCSessionDescription(payload.data));
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote answer');
await applyQueuedCandidates(connection, payload.streamSessionId, payload.fromDeviceId);
addActivity('WebRTC', 'Answer received and applied');
return;
@@ -1787,14 +1946,17 @@ const connectSocket = () => {
if (!payload.data) return;
const connection = peerConnections.get(payload.streamSessionId);
if (!connection || !connection.remoteDescription) {
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Queued remote ICE candidate');
queueRemoteCandidate(payload);
return;
}
await connection.addIceCandidate(new RTCIceCandidate(payload.data));
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Applied remote ICE candidate');
return;
}
if (payload.signalType === 'hangup') {
pushStreamDiagnostic(payload.streamSessionId, 'signal', 'Received remote hangup');
teardownPeerConnection(payload.streamSessionId);
if (getAppState().activeStreamSessionId === payload.streamSessionId) {
setAppState({ activeStreamSessionId: null });
@@ -1802,6 +1964,12 @@ const connectSocket = () => {
addActivity('Stream', 'Remote stream ended');
}
} catch (error) {
pushStreamDiagnostic(
payload.streamSessionId,
'error',
error?.message || 'Failed handling WebRTC signal',
'error'
);
console.error('Failed handling WebRTC signal', error);
pushToast('WebRTC negotiation failed', 'error');
}
@@ -1809,6 +1977,9 @@ const connectSocket = () => {
socket.on('error:webrtc_signal', (payload) => {
const message = payload?.message || 'WebRTC signaling error';
if (payload?.streamSessionId) {
pushStreamDiagnostic(payload.streamSessionId, 'error', message, 'error');
}
addActivity('WebRTC', message);
pushToast(message, 'error');
});
@@ -2261,7 +2432,17 @@ const actions = {
try {
requestedStreams.add(cameraDeviceId);
await api.streams.request(cameraDeviceId);
const result = await api.streams.request(cameraDeviceId);
requestedStreams.delete(cameraDeviceId);
const streamSessionId = result?.streamSession?.id;
if (streamSessionId) {
bindClientStreamSession(
cameraDeviceId,
streamSessionId,
'Bound requested stream session from HTTP response'
);
pushStreamDiagnostic(streamSessionId, 'request', 'Backend accepted stream request');
}
} catch (error) {
requestedStreams.delete(cameraDeviceId);
pushToast(error.message || 'Failed to request stream', 'error');
@@ -2292,6 +2473,7 @@ const actions = {
}
if (reusableSessionId) {
pushStreamDiagnostic(reusableSessionId, 'viewer', 'Reusing existing stream session for selected camera');
attachClientStreamToElement();
setClientStreamMode(remoteStreams.has(reusableSessionId) ? 'video' : 'connecting');
return;

1
WebApp/src/lib/app/store.js
View File

@@ -23,6 +23,7 @@ export const createInitialState = () => ({
activityLog: [],
cameraSessions: {},
connectedStreamSessionIds: [],
streamDiagnostics: {},
loading: true,
isRegistering: false,
authForm: {