From 53b040a7f06bfe30a4b029ed316af79f1f88e637 Mon Sep 17 00:00:00 2001 From: Matiss Jurevics Date: Fri, 13 Feb 2026 13:20:00 +0000 Subject: [PATCH] feat(sim): render SFU media via mediasoup-client in single_server_sfu mode --- Backend/public/mobile-sim.js | 191 ++++++++++++++++++++++++++++++----- 1 file changed, 164 insertions(+), 27 deletions(-) diff --git a/Backend/public/mobile-sim.js b/Backend/public/mobile-sim.js index ff73f47..893a854 100644 --- a/Backend/public/mobile-sim.js +++ b/Backend/public/mobile-sim.js @@ -145,6 +145,7 @@ const API = { }, sfu: { + getRouterRtpCapabilities: (id) => API.request(`/streams/${id}/sfu/router-rtp-capabilities`), getSession: (id) => API.request(`/streams/${id}/sfu/session`), createPublishTransport: (id) => API.request(`/streams/${id}/sfu/publish-transport`, { method: 'POST', body: JSON.stringify({ role: 'camera' }) }), connectPublishTransport: (id, payload) => API.request(`/streams/${id}/sfu/publish-transport/connect`, { method: 'POST', body: JSON.stringify(payload) }), @@ -178,6 +179,13 @@ let hasWebrtcEverConnected = false; let lastPeerConnectionState = null; let pendingRemoteCandidates = []; let mediaMode = 'legacy'; +let mediasoupClientModulePromise = null; +let sfuDevice = null; +let sfuDeviceStreamSessionId = null; +let sfuSendTransport = null; +let sfuRecvTransport = null; +let sfuPublishedProducerId = null; +let sfuConsumedTrack = null; const rtcConfig = { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], }; @@ -197,6 +205,67 @@ const detectMediaMode = async () => { } }; +const loadMediasoupClientModule = async () => { + if (!mediasoupClientModulePromise) { + mediasoupClientModulePromise = import('https://cdn.jsdelivr.net/npm/mediasoup-client@3/+esm'); + } + return await mediasoupClientModulePromise; +}; + +const closeSfuTransports = () => { + try { + sfuSendTransport?.close?.(); + } catch { } + try { + sfuRecvTransport?.close?.(); + } catch { } + + if (sfuConsumedTrack) { + try { + sfuConsumedTrack.stop(); + } catch { } + } + + sfuSendTransport = null; + sfuRecvTransport = null; + sfuPublishedProducerId = null; + sfuConsumedTrack = null; +}; + +const resetSfuRuntime = () => { + closeSfuTransports(); + sfuDevice = null; + sfuDeviceStreamSessionId = null; +}; + +const getMediasoupDeviceClass = async () => { + const module = await loadMediasoupClientModule(); + return module.Device || module.default?.Device || module.default; +}; + +const ensureSfuDevice = async (streamSessionId) => { + if (sfuDevice && sfuDeviceStreamSessionId === streamSessionId) { + return sfuDevice; + } + + resetSfuRuntime(); + + const DeviceClass = await getMediasoupDeviceClass(); + if (!DeviceClass) { + throw new Error('mediasoup-client Device class unavailable'); + } + const device = new DeviceClass(); + const caps = await API.sfu.getRouterRtpCapabilities(streamSessionId); + if (!caps?.routerRtpCapabilities) { + throw new Error('Router RTP capabilities not available'); + } + + await device.load({ routerRtpCapabilities: caps.routerRtpCapabilities }); + sfuDevice = device; + sfuDeviceStreamSessionId = streamSessionId; + return device; +}; + const init = async () => { // Load local storage const saved = localStorage.getItem('mobileSimDevice'); @@ -541,6 +610,7 @@ const teardownPeerConnection = () => { if (previousSessionId) { pendingRemoteCandidates = pendingRemoteCandidates.filter((item) => item.streamSessionId !== previousSessionId); } + closeSfuTransports(); clearClientStream(); }; @@ -694,29 +764,55 @@ const startSfuPublishHandshake = async (streamSessionId) => { throw new Error('Camera stream unavailable for SFU publish'); } + const device = await ensureSfuDevice(streamSessionId); const publishTransport = await API.sfu.createPublishTransport(streamSessionId); - const transportId = publishTransport?.transport?.transportId; - if (!transportId) { - throw new Error('Missing SFU publish transport id'); + const transportMeta = publishTransport?.transport; + const transportId = transportMeta?.transportId; + const transportOptions = transportMeta?.transportOptions; + if (!transportId || !transportOptions) { + throw new Error('Missing SFU publish transport parameters'); } - await API.sfu.connectPublishTransport(streamSessionId, { - transportId, - dtlsParameters: { role: 'auto' }, + closeSfuTransports(); + const sendTransport = device.createSendTransport(transportOptions); + sfuSendTransport = sendTransport; + + sendTransport.on('connect', async ({ dtlsParameters }, callback, errback) => { + try { + await API.sfu.connectPublishTransport(streamSessionId, { + transportId, + dtlsParameters, + }); + callback(); + } catch (error) { + errback(error); + } + }); + + sendTransport.on('produce', async ({ kind, rtpParameters }, callback, errback) => { + try { + const produced = await API.sfu.produce(streamSessionId, { + transportId, + kind, + rtpParameters, + }); + const producerId = produced?.producer?.producerId; + if (!producerId) { + throw new Error('SFU producer id missing'); + } + sfuPublishedProducerId = producerId; + callback({ id: producerId }); + } catch (error) { + errback(error); + } }); const track = localCameraStream.getVideoTracks()[0]; - await API.sfu.produce(streamSessionId, { - transportId, - kind: 'video', - rtpParameters: { - codec: 'VP8', - source: 'mobile-sim-camera', - trackSettings: track?.getSettings?.() ?? {}, - }, - }); - - addActivity('SFU', 'Publish handshake completed'); + if (!track) { + throw new Error('No local video track available for SFU publish'); + } + await sendTransport.produce({ track }); + addActivity('SFU', 'Publish handshake completed (mediasoup)'); }; const setClientSfuConnectedState = () => { @@ -724,20 +820,41 @@ const setClientSfuConnectedState = () => { clearTimeout(remoteStreamWaitTimer); remoteStreamWaitTimer = null; } - setClientStreamPlaceholderText('Connected via SFU (simulated media path)'); - setClientStreamMode('connecting'); + if (!remoteClientStream) { + setClientStreamPlaceholderText('Connected via SFU'); + setClientStreamMode('connecting'); + } }; const startSfuSubscribeHandshake = async (streamSessionId) => { + const device = await ensureSfuDevice(streamSessionId); const subscribeTransport = await API.sfu.createSubscribeTransport(streamSessionId); - const transportId = subscribeTransport?.transport?.transportId; - if (!transportId) { - throw new Error('Missing SFU subscribe transport id'); + const transportMeta = subscribeTransport?.transport; + const transportId = transportMeta?.transportId; + const transportOptions = transportMeta?.transportOptions; + if (!transportId || !transportOptions) { + throw new Error('Missing SFU subscribe transport parameters'); } - await API.sfu.connectSubscribeTransport(streamSessionId, { - transportId, - dtlsParameters: { role: 'auto' }, + if (sfuRecvTransport) { + try { + sfuRecvTransport.close?.(); + } catch { } + } + + const recvTransport = device.createRecvTransport(transportOptions); + sfuRecvTransport = recvTransport; + + recvTransport.on('connect', async ({ dtlsParameters }, callback, errback) => { + try { + await API.sfu.connectSubscribeTransport(streamSessionId, { + transportId, + dtlsParameters, + }); + callback(); + } catch (error) { + errback(error); + } }); let consumeResult = null; @@ -745,7 +862,7 @@ const startSfuSubscribeHandshake = async (streamSessionId) => { try { consumeResult = await API.sfu.consume(streamSessionId, { transportId, - rtpCapabilities: { browser: 'mobile-sim' }, + rtpCapabilities: device.rtpCapabilities, }); if (consumeResult?.consumer?.consumerId) { break; @@ -760,8 +877,25 @@ const startSfuSubscribeHandshake = async (streamSessionId) => { throw new Error('SFU consumer was not created'); } + const consumerMeta = consumeResult.consumer; + const consumer = await recvTransport.consume({ + id: consumerMeta.consumerId, + producerId: consumerMeta.producerId, + kind: consumerMeta.kind, + rtpParameters: consumerMeta.rtpParameters, + }); + sfuConsumedTrack = consumer.track; + const stream = new MediaStream([consumer.track]); + remoteClientStream = stream; + const videoEl = $('clientStreamVideo'); + if (videoEl) { + videoEl.srcObject = stream; + setClientStreamMode('video'); + void videoEl.play().catch(() => {}); + } + setClientSfuConnectedState(); - addActivity('SFU', `Subscribed to producer ${consumeResult.consumer.producerId}`); + addActivity('SFU', `Subscribed to producer ${consumeResult.consumer.producerId} (mediasoup)`); }; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -853,6 +987,7 @@ const connectSocket = () => { stopFrameRelay(); void stopLocalRecording(); teardownPeerConnection(); + resetSfuRuntime(); store.update({ activeCameraDeviceId: null, activeStreamSessionId: null }); }); @@ -962,6 +1097,7 @@ const connectSocket = () => { socket.on('stream:ended', (payload) => { if (payload?.streamSessionId && payload.streamSessionId === store.get().activeStreamSessionId) { clearClientStream(); + closeSfuTransports(); store.update({ activeCameraDeviceId: null, activeStreamSessionId: null }); } }); @@ -1171,6 +1307,7 @@ const Actions = { stopFrameRelay(); await stopLocalRecording(); teardownPeerConnection(); + resetSfuRuntime(); stopCameraPreview(); localStorage.removeItem('mobileSimDevice'); Toast.show('Signed Out', 'info');