diff --git a/Backend/public/mobile-sim.js b/Backend/public/mobile-sim.js index 52f1da4..d25352d 100644 --- a/Backend/public/mobile-sim.js +++ b/Backend/public/mobile-sim.js @@ -151,6 +151,9 @@ let peerConnection = null; let peerSessionId = null; let peerTargetDeviceId = null; let remoteStreamWaitTimer = null; +let frameRelayTimer = null; +let frameCanvas = null; +let frameContext = null; const rtcConfig = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], }; @@ -233,10 +236,14 @@ const stopCameraPreview = () => { const setClientStreamVisibility = (isVisible) => { const videoEl = $('clientStreamVideo'); + const imageEl = $('clientStreamImage'); const placeholderEl = $('clientStreamPlaceholder'); if (videoEl) { videoEl.classList.toggle('hidden', !isVisible); } + if (imageEl) { + imageEl.classList.toggle('hidden', !isVisible); + } if (placeholderEl) { placeholderEl.classList.toggle('hidden', isVisible); } @@ -248,6 +255,7 @@ const clearClientStream = () => { remoteStreamWaitTimer = null; } const videoEl = $('clientStreamVideo'); + const imageEl = $('clientStreamImage'); if (remoteClientStream) { remoteClientStream.getTracks().forEach((track) => track.stop()); remoteClientStream = null; @@ -255,9 +263,54 @@ const clearClientStream = () => { if (videoEl) { videoEl.srcObject = null; } + if (imageEl) { + imageEl.src = ''; + } setClientStreamVisibility(false); }; +const stopFrameRelay = () => { + if (frameRelayTimer) { + clearInterval(frameRelayTimer); + frameRelayTimer = null; + } +}; + +const startFrameRelay = async (streamSessionId, toDeviceId) => { + if (!socket || !streamSessionId || !toDeviceId) return; + + const ready = await startCameraPreview(); + if (!ready) { + throw new Error('Camera permission is required before streaming'); + } + + const cameraVideoEl = $('cameraVideo'); + if (!cameraVideoEl) return; + + stopFrameRelay(); + frameRelayTimer = setInterval(() => { + if (!socket || cameraVideoEl.readyState < 2 || !cameraVideoEl.videoWidth || !cameraVideoEl.videoHeight) return; + + if (!frameCanvas) { + frameCanvas = document.createElement('canvas'); + frameContext = frameCanvas.getContext('2d'); + } + if (!frameCanvas || !frameContext) return; + + frameCanvas.width = cameraVideoEl.videoWidth; + frameCanvas.height = cameraVideoEl.videoHeight; + frameContext.drawImage(cameraVideoEl, 0, 0, frameCanvas.width, frameCanvas.height); + const frame = frameCanvas.toDataURL('image/jpeg', 0.6); + + socket.emit('stream:frame', { + toDeviceId, + streamSessionId, + frame, + capturedAt: new Date().toISOString(), + }); + }, 300); +}; + const teardownPeerConnection = () => { if (peerConnection) { peerConnection.onicecandidate = null; @@ -373,6 +426,7 @@ const connectSocket = () => { socket.on('disconnect', () => { store.update({ socketConnected: false }); + stopFrameRelay(); teardownPeerConnection(); }); @@ -391,11 +445,13 @@ const connectSocket = () => { await API.streams.getPublishCreds(streamId); if (payload.sourceDeviceId) { await startOfferToClient(streamId, payload.sourceDeviceId); + await startFrameRelay(streamId, payload.sourceDeviceId); } addActivity('Stream', 'Accepted & Published'); // Auto-stop after 15s for simulation setTimeout(async () => { await API.streams.end(streamId); + stopFrameRelay(); if (socket && payload.sourceDeviceId) { socket.emit('webrtc:signal', { toDeviceId: payload.sourceDeviceId, @@ -438,6 +494,23 @@ const connectSocket = () => { } }); + socket.on('stream:frame', (payload) => { + if (!payload?.frame) return; + if (remoteStreamWaitTimer) { + clearTimeout(remoteStreamWaitTimer); + remoteStreamWaitTimer = null; + } + const imageEl = $('clientStreamImage'); + if (!imageEl) return; + imageEl.src = payload.frame; + imageEl.classList.remove('hidden'); + const videoEl = $('clientStreamVideo'); + if (videoEl) { + videoEl.classList.add('hidden'); + } + setClientStreamVisibility(true); + }); + socket.on('webrtc:signal', async (payload) => { const device = store.get().device; if (!device || !payload?.streamSessionId || !payload?.signalType || !payload?.fromDeviceId) return; @@ -614,6 +687,7 @@ const Actions = { await API.auth.signOut(); store.update({ session: null, screen: 'auth', device: null, deviceToken: null, socketConnected: false }); if (socket) socket.disconnect(); + stopFrameRelay(); teardownPeerConnection(); stopCameraPreview(); localStorage.removeItem('mobileSimDevice'); @@ -873,6 +947,7 @@ store.subscribe(render); init(); window.addEventListener('beforeunload', () => { + stopFrameRelay(); teardownPeerConnection(); stopCameraPreview(); }); diff --git a/Backend/realtime/gateway.ts b/Backend/realtime/gateway.ts index 569a2cd..f648d6b 100644 --- a/Backend/realtime/gateway.ts +++ b/Backend/realtime/gateway.ts @@ -26,6 +26,13 @@ const webrtcSignalSchema = z.object({ data: z.record(z.string(), z.unknown()).nullable().optional(), }); +const streamFrameSchema = z.object({ + toDeviceId: z.string().uuid(), + streamSessionId: z.string().uuid(), + frame: z.string().min(32), + capturedAt: z.string().optional(), +}); + const roomForDevice = (deviceId: string): string => `device:${deviceId}`; let io: SocketIOServer | null = null; @@ -233,6 +240,7 @@ export const setupRealtimeGateway = (server: HttpServer): SocketIOServer => { io.on('connection', async (socket) => { const auth = socket.data.deviceAuth as { userId: string; deviceId: string; role: 'camera' | 'client' }; const deviceRoom = roomForDevice(auth.deviceId); + const verifiedRelayTargets = new Set(); socket.join(deviceRoom); await markDevicePresence(auth.deviceId, 'online'); @@ -321,6 +329,38 @@ export const setupRealtimeGateway = (server: HttpServer): SocketIOServer => { }); }); + socket.on('stream:frame', async (input) => { + const parsed = streamFrameSchema.safeParse(input); + + if (!parsed.success) { + socket.emit('error:stream_frame', { + message: 'Invalid stream frame payload', + errors: parsed.error.flatten(), + }); + return; + } + + if (!verifiedRelayTargets.has(parsed.data.toDeviceId)) { + const targetDevice = await db.query.devices.findFirst({ + where: and(eq(devices.id, parsed.data.toDeviceId), eq(devices.userId, auth.userId)), + }); + + if (!targetDevice) { + socket.emit('error:stream_frame', { message: 'Target device not found for this account' }); + return; + } + + verifiedRelayTargets.add(parsed.data.toDeviceId); + } + + io?.to(roomForDevice(parsed.data.toDeviceId)).emit('stream:frame', { + fromDeviceId: auth.deviceId, + streamSessionId: parsed.data.streamSessionId, + frame: parsed.data.frame, + capturedAt: parsed.data.capturedAt ?? new Date().toISOString(), + }); + }); + socket.on('disconnect', async () => { // Small delay allows fast reconnects to reuse presence without flapping. setTimeout(async () => {