feat(streaming): implement frame relay functionality for real-time video streaming and enhance client stream visibility

Backend/public/mobile-sim.js

@@ -151,6 +151,9 @@ let peerConnection = null;
 let peerSessionId = null;
 let peerTargetDeviceId = null;
 let remoteStreamWaitTimer = null;
+let frameRelayTimer = null;
+let frameCanvas = null;
+let frameContext = null;
 const rtcConfig = {
   iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
 };
@@ -233,10 +236,14 @@ const stopCameraPreview = () => {
 
 const setClientStreamVisibility = (isVisible) => {
   const videoEl = $('clientStreamVideo');
+  const imageEl = $('clientStreamImage');
   const placeholderEl = $('clientStreamPlaceholder');
   if (videoEl) {
     videoEl.classList.toggle('hidden', !isVisible);
   }
+  if (imageEl) {
+    imageEl.classList.toggle('hidden', !isVisible);
+  }
   if (placeholderEl) {
     placeholderEl.classList.toggle('hidden', isVisible);
   }
@@ -248,6 +255,7 @@ const clearClientStream = () => {
     remoteStreamWaitTimer = null;
   }
   const videoEl = $('clientStreamVideo');
+  const imageEl = $('clientStreamImage');
   if (remoteClientStream) {
     remoteClientStream.getTracks().forEach((track) => track.stop());
     remoteClientStream = null;
@@ -255,9 +263,54 @@ const clearClientStream = () => {
   if (videoEl) {
     videoEl.srcObject = null;
   }
+  if (imageEl) {
+    imageEl.src = '';
+  }
   setClientStreamVisibility(false);
 };
 
+const stopFrameRelay = () => {
+  if (frameRelayTimer) {
+    clearInterval(frameRelayTimer);
+    frameRelayTimer = null;
+  }
+};
+
+const startFrameRelay = async (streamSessionId, toDeviceId) => {
+  if (!socket || !streamSessionId || !toDeviceId) return;
+
+  const ready = await startCameraPreview();
+  if (!ready) {
+    throw new Error('Camera permission is required before streaming');
+  }
+
+  const cameraVideoEl = $('cameraVideo');
+  if (!cameraVideoEl) return;
+
+  stopFrameRelay();
+  frameRelayTimer = setInterval(() => {
+    if (!socket || cameraVideoEl.readyState < 2 || !cameraVideoEl.videoWidth || !cameraVideoEl.videoHeight) return;
+
+    if (!frameCanvas) {
+      frameCanvas = document.createElement('canvas');
+      frameContext = frameCanvas.getContext('2d');
+    }
+    if (!frameCanvas || !frameContext) return;
+
+    frameCanvas.width = cameraVideoEl.videoWidth;
+    frameCanvas.height = cameraVideoEl.videoHeight;
+    frameContext.drawImage(cameraVideoEl, 0, 0, frameCanvas.width, frameCanvas.height);
+    const frame = frameCanvas.toDataURL('image/jpeg', 0.6);
+
+    socket.emit('stream:frame', {
+      toDeviceId,
+      streamSessionId,
+      frame,
+      capturedAt: new Date().toISOString(),
+    });
+  }, 300);
+};
+
 const teardownPeerConnection = () => {
   if (peerConnection) {
     peerConnection.onicecandidate = null;
@@ -373,6 +426,7 @@ const connectSocket = () => {
 
   socket.on('disconnect', () => {
     store.update({ socketConnected: false });
+    stopFrameRelay();
     teardownPeerConnection();
   });
 
@@ -391,11 +445,13 @@ const connectSocket = () => {
         await API.streams.getPublishCreds(streamId);
         if (payload.sourceDeviceId) {
           await startOfferToClient(streamId, payload.sourceDeviceId);
+          await startFrameRelay(streamId, payload.sourceDeviceId);
         }
         addActivity('Stream', 'Accepted & Published');
         // Auto-stop after 15s for simulation
         setTimeout(async () => {
           await API.streams.end(streamId);
+          stopFrameRelay();
           if (socket && payload.sourceDeviceId) {
             socket.emit('webrtc:signal', {
               toDeviceId: payload.sourceDeviceId,
@@ -438,6 +494,23 @@ const connectSocket = () => {
     }
   });
 
+  socket.on('stream:frame', (payload) => {
+    if (!payload?.frame) return;
+    if (remoteStreamWaitTimer) {
+      clearTimeout(remoteStreamWaitTimer);
+      remoteStreamWaitTimer = null;
+    }
+    const imageEl = $('clientStreamImage');
+    if (!imageEl) return;
+    imageEl.src = payload.frame;
+    imageEl.classList.remove('hidden');
+    const videoEl = $('clientStreamVideo');
+    if (videoEl) {
+      videoEl.classList.add('hidden');
+    }
+    setClientStreamVisibility(true);
+  });
+
   socket.on('webrtc:signal', async (payload) => {
     const device = store.get().device;
     if (!device || !payload?.streamSessionId || !payload?.signalType || !payload?.fromDeviceId) return;
@@ -614,6 +687,7 @@ const Actions = {
     await API.auth.signOut();
     store.update({ session: null, screen: 'auth', device: null, deviceToken: null, socketConnected: false });
     if (socket) socket.disconnect();
+    stopFrameRelay();
     teardownPeerConnection();
     stopCameraPreview();
     localStorage.removeItem('mobileSimDevice');
@@ -873,6 +947,7 @@ store.subscribe(render);
 init();
 
 window.addEventListener('beforeunload', () => {
+  stopFrameRelay();
   teardownPeerConnection();
   stopCameraPreview();
 });

Backend/realtime/gateway.ts

@@ -26,6 +26,13 @@ const webrtcSignalSchema = z.object({
   data: z.record(z.string(), z.unknown()).nullable().optional(),
 });
 
+const streamFrameSchema = z.object({
+  toDeviceId: z.string().uuid(),
+  streamSessionId: z.string().uuid(),
+  frame: z.string().min(32),
+  capturedAt: z.string().optional(),
+});
+
 const roomForDevice = (deviceId: string): string => `device:${deviceId}`;
 
 let io: SocketIOServer | null = null;
@@ -233,6 +240,7 @@ export const setupRealtimeGateway = (server: HttpServer): SocketIOServer => {
   io.on('connection', async (socket) => {
     const auth = socket.data.deviceAuth as { userId: string; deviceId: string; role: 'camera' | 'client' };
     const deviceRoom = roomForDevice(auth.deviceId);
+    const verifiedRelayTargets = new Set<string>();
 
     socket.join(deviceRoom);
     await markDevicePresence(auth.deviceId, 'online');
@@ -321,6 +329,38 @@ export const setupRealtimeGateway = (server: HttpServer): SocketIOServer => {
       });
     });
 
+    socket.on('stream:frame', async (input) => {
+      const parsed = streamFrameSchema.safeParse(input);
+
+      if (!parsed.success) {
+        socket.emit('error:stream_frame', {
+          message: 'Invalid stream frame payload',
+          errors: parsed.error.flatten(),
+        });
+        return;
+      }
+
+      if (!verifiedRelayTargets.has(parsed.data.toDeviceId)) {
+        const targetDevice = await db.query.devices.findFirst({
+          where: and(eq(devices.id, parsed.data.toDeviceId), eq(devices.userId, auth.userId)),
+        });
+
+        if (!targetDevice) {
+          socket.emit('error:stream_frame', { message: 'Target device not found for this account' });
+          return;
+        }
+
+        verifiedRelayTargets.add(parsed.data.toDeviceId);
+      }
+
+      io?.to(roomForDevice(parsed.data.toDeviceId)).emit('stream:frame', {
+        fromDeviceId: auth.deviceId,
+        streamSessionId: parsed.data.streamSessionId,
+        frame: parsed.data.frame,
+        capturedAt: parsed.data.capturedAt ?? new Date().toISOString(),
+      });
+    });
+
     socket.on('disconnect', async () => {
       // Small delay allows fast reconnects to reuse presence without flapping.
       setTimeout(async () => {