feat(sim): render SFU media via mediasoup-client in single_server_sfu mode

Backend/public/mobile-sim.js

@@ -145,6 +145,7 @@ const API = {
   },
 
   sfu: {
+    getRouterRtpCapabilities: (id) => API.request(`/streams/${id}/sfu/router-rtp-capabilities`),
     getSession: (id) => API.request(`/streams/${id}/sfu/session`),
     createPublishTransport: (id) => API.request(`/streams/${id}/sfu/publish-transport`, { method: 'POST', body: JSON.stringify({ role: 'camera' }) }),
     connectPublishTransport: (id, payload) => API.request(`/streams/${id}/sfu/publish-transport/connect`, { method: 'POST', body: JSON.stringify(payload) }),
@@ -178,6 +179,13 @@ let hasWebrtcEverConnected = false;
 let lastPeerConnectionState = null;
 let pendingRemoteCandidates = [];
 let mediaMode = 'legacy';
+let mediasoupClientModulePromise = null;
+let sfuDevice = null;
+let sfuDeviceStreamSessionId = null;
+let sfuSendTransport = null;
+let sfuRecvTransport = null;
+let sfuPublishedProducerId = null;
+let sfuConsumedTrack = null;
 const rtcConfig = {
   iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
 };
@@ -197,6 +205,67 @@ const detectMediaMode = async () => {
   }
 };
 
+const loadMediasoupClientModule = async () => {
+  if (!mediasoupClientModulePromise) {
+    mediasoupClientModulePromise = import('https://cdn.jsdelivr.net/npm/mediasoup-client@3/+esm');
+  }
+  return await mediasoupClientModulePromise;
+};
+
+const closeSfuTransports = () => {
+  try {
+    sfuSendTransport?.close?.();
+  } catch { }
+  try {
+    sfuRecvTransport?.close?.();
+  } catch { }
+
+  if (sfuConsumedTrack) {
+    try {
+      sfuConsumedTrack.stop();
+    } catch { }
+  }
+
+  sfuSendTransport = null;
+  sfuRecvTransport = null;
+  sfuPublishedProducerId = null;
+  sfuConsumedTrack = null;
+};
+
+const resetSfuRuntime = () => {
+  closeSfuTransports();
+  sfuDevice = null;
+  sfuDeviceStreamSessionId = null;
+};
+
+const getMediasoupDeviceClass = async () => {
+  const module = await loadMediasoupClientModule();
+  return module.Device || module.default?.Device || module.default;
+};
+
+const ensureSfuDevice = async (streamSessionId) => {
+  if (sfuDevice && sfuDeviceStreamSessionId === streamSessionId) {
+    return sfuDevice;
+  }
+
+  resetSfuRuntime();
+
+  const DeviceClass = await getMediasoupDeviceClass();
+  if (!DeviceClass) {
+    throw new Error('mediasoup-client Device class unavailable');
+  }
+  const device = new DeviceClass();
+  const caps = await API.sfu.getRouterRtpCapabilities(streamSessionId);
+  if (!caps?.routerRtpCapabilities) {
+    throw new Error('Router RTP capabilities not available');
+  }
+
+  await device.load({ routerRtpCapabilities: caps.routerRtpCapabilities });
+  sfuDevice = device;
+  sfuDeviceStreamSessionId = streamSessionId;
+  return device;
+};
+
 const init = async () => {
   // Load local storage
   const saved = localStorage.getItem('mobileSimDevice');
@@ -541,6 +610,7 @@ const teardownPeerConnection = () => {
   if (previousSessionId) {
     pendingRemoteCandidates = pendingRemoteCandidates.filter((item) => item.streamSessionId !== previousSessionId);
   }
+  closeSfuTransports();
   clearClientStream();
 };
 
@@ -694,29 +764,55 @@ const startSfuPublishHandshake = async (streamSessionId) => {
     throw new Error('Camera stream unavailable for SFU publish');
   }
 
+  const device = await ensureSfuDevice(streamSessionId);
   const publishTransport = await API.sfu.createPublishTransport(streamSessionId);
-  const transportId = publishTransport?.transport?.transportId;
-  if (!transportId) {
-    throw new Error('Missing SFU publish transport id');
+  const transportMeta = publishTransport?.transport;
+  const transportId = transportMeta?.transportId;
+  const transportOptions = transportMeta?.transportOptions;
+  if (!transportId || !transportOptions) {
+    throw new Error('Missing SFU publish transport parameters');
   }
 
-  await API.sfu.connectPublishTransport(streamSessionId, {
-    transportId,
-    dtlsParameters: { role: 'auto' },
+  closeSfuTransports();
+  const sendTransport = device.createSendTransport(transportOptions);
+  sfuSendTransport = sendTransport;
+
+  sendTransport.on('connect', async ({ dtlsParameters }, callback, errback) => {
+    try {
+      await API.sfu.connectPublishTransport(streamSessionId, {
+        transportId,
+        dtlsParameters,
+      });
+      callback();
+    } catch (error) {
+      errback(error);
+    }
+  });
+
+  sendTransport.on('produce', async ({ kind, rtpParameters }, callback, errback) => {
+    try {
+      const produced = await API.sfu.produce(streamSessionId, {
+        transportId,
+        kind,
+        rtpParameters,
+      });
+      const producerId = produced?.producer?.producerId;
+      if (!producerId) {
+        throw new Error('SFU producer id missing');
+      }
+      sfuPublishedProducerId = producerId;
+      callback({ id: producerId });
+    } catch (error) {
+      errback(error);
+    }
   });
 
   const track = localCameraStream.getVideoTracks()[0];
-  await API.sfu.produce(streamSessionId, {
-    transportId,
-    kind: 'video',
-    rtpParameters: {
-      codec: 'VP8',
-      source: 'mobile-sim-camera',
-      trackSettings: track?.getSettings?.() ?? {},
-    },
-  });
-
-  addActivity('SFU', 'Publish handshake completed');
+  if (!track) {
+    throw new Error('No local video track available for SFU publish');
+  }
+  await sendTransport.produce({ track });
+  addActivity('SFU', 'Publish handshake completed (mediasoup)');
 };
 
 const setClientSfuConnectedState = () => {
@@ -724,20 +820,41 @@ const setClientSfuConnectedState = () => {
     clearTimeout(remoteStreamWaitTimer);
     remoteStreamWaitTimer = null;
   }
-  setClientStreamPlaceholderText('Connected via SFU (simulated media path)');
-  setClientStreamMode('connecting');
+  if (!remoteClientStream) {
+    setClientStreamPlaceholderText('Connected via SFU');
+    setClientStreamMode('connecting');
+  }
 };
 
 const startSfuSubscribeHandshake = async (streamSessionId) => {
+  const device = await ensureSfuDevice(streamSessionId);
   const subscribeTransport = await API.sfu.createSubscribeTransport(streamSessionId);
-  const transportId = subscribeTransport?.transport?.transportId;
-  if (!transportId) {
-    throw new Error('Missing SFU subscribe transport id');
+  const transportMeta = subscribeTransport?.transport;
+  const transportId = transportMeta?.transportId;
+  const transportOptions = transportMeta?.transportOptions;
+  if (!transportId || !transportOptions) {
+    throw new Error('Missing SFU subscribe transport parameters');
   }
 
-  await API.sfu.connectSubscribeTransport(streamSessionId, {
-    transportId,
-    dtlsParameters: { role: 'auto' },
+  if (sfuRecvTransport) {
+    try {
+      sfuRecvTransport.close?.();
+    } catch { }
+  }
+
+  const recvTransport = device.createRecvTransport(transportOptions);
+  sfuRecvTransport = recvTransport;
+
+  recvTransport.on('connect', async ({ dtlsParameters }, callback, errback) => {
+    try {
+      await API.sfu.connectSubscribeTransport(streamSessionId, {
+        transportId,
+        dtlsParameters,
+      });
+      callback();
+    } catch (error) {
+      errback(error);
+    }
   });
 
   let consumeResult = null;
@@ -745,7 +862,7 @@ const startSfuSubscribeHandshake = async (streamSessionId) => {
     try {
       consumeResult = await API.sfu.consume(streamSessionId, {
         transportId,
-        rtpCapabilities: { browser: 'mobile-sim' },
+        rtpCapabilities: device.rtpCapabilities,
       });
       if (consumeResult?.consumer?.consumerId) {
         break;
@@ -760,8 +877,25 @@ const startSfuSubscribeHandshake = async (streamSessionId) => {
     throw new Error('SFU consumer was not created');
   }
 
+  const consumerMeta = consumeResult.consumer;
+  const consumer = await recvTransport.consume({
+    id: consumerMeta.consumerId,
+    producerId: consumerMeta.producerId,
+    kind: consumerMeta.kind,
+    rtpParameters: consumerMeta.rtpParameters,
+  });
+  sfuConsumedTrack = consumer.track;
+  const stream = new MediaStream([consumer.track]);
+  remoteClientStream = stream;
+  const videoEl = $('clientStreamVideo');
+  if (videoEl) {
+    videoEl.srcObject = stream;
+    setClientStreamMode('video');
+    void videoEl.play().catch(() => {});
+  }
+
   setClientSfuConnectedState();
-  addActivity('SFU', `Subscribed to producer ${consumeResult.consumer.producerId}`);
+  addActivity('SFU', `Subscribed to producer ${consumeResult.consumer.producerId} (mediasoup)`);
 };
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -853,6 +987,7 @@ const connectSocket = () => {
     stopFrameRelay();
     void stopLocalRecording();
     teardownPeerConnection();
+    resetSfuRuntime();
     store.update({ activeCameraDeviceId: null, activeStreamSessionId: null });
   });
 
@@ -962,6 +1097,7 @@ const connectSocket = () => {
   socket.on('stream:ended', (payload) => {
     if (payload?.streamSessionId && payload.streamSessionId === store.get().activeStreamSessionId) {
       clearClientStream();
+      closeSfuTransports();
       store.update({ activeCameraDeviceId: null, activeStreamSessionId: null });
     }
   });
@@ -1171,6 +1307,7 @@ const Actions = {
     stopFrameRelay();
     await stopLocalRecording();
     teardownPeerConnection();
+    resetSfuRuntime();
     stopCameraPreview();
     localStorage.removeItem('mobileSimDevice');
     Toast.show('Signed Out', 'info');