From fdad261c2892c0aee830424454b50b3e2657ad78 Mon Sep 17 00:00:00 2001 From: Matiss Jurevics Date: Wed, 11 Feb 2026 16:45:00 +0000 Subject: [PATCH] feat(sfu): add mediasoup runtime and router capabilities endpoint --- Backend/bun.lock | 40 +++- Backend/media/sfu/mediasoup.ts | 367 +++++++++++++++++++++++++++++++++ Backend/media/sfu/noop.ts | 19 ++ Backend/media/sfu/service.ts | 10 +- Backend/media/sfu/types.ts | 11 + Backend/package.json | 6 +- Backend/routes/streams.ts | 38 ++++ 7 files changed, 486 insertions(+), 5 deletions(-) create mode 100644 Backend/media/sfu/mediasoup.ts diff --git a/Backend/bun.lock b/Backend/bun.lock index 64f6059..6de0373 100644 --- a/Backend/bun.lock +++ b/Backend/bun.lock @@ -15,6 +15,7 @@ "drizzle-orm": "^0.44.0", "express": "^5.2.1", "helmet": "^8.1.0", + "mediasoup": "^3.15.6", "minio": "^8.0.6", "openai": "^6.18.0", "pg": "^8.18.0", @@ -39,6 +40,9 @@ }, }, }, + "trustedDependencies": [ + "mediasoup", + ], "packages": { "@asteasolutions/zod-to-openapi": ["@asteasolutions/zod-to-openapi@8.4.0", "", { "dependencies": { "openapi3-ts": "^4.1.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-Ckp971tmTw4pnv+o7iK85ldBHBKk6gxMaoNyLn3c2Th/fKoTG8G3jdYuOanpdGqwlDB0z01FOjry2d32lfTqrA=="], @@ -110,6 +114,8 @@ "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="], + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], @@ -232,6 +238,8 @@ "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], + "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], @@ -246,6 +254,8 @@ "create-require": ["create-require@1.1.1", "", {}, "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ=="], + "data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "decode-uri-component": ["decode-uri-component@0.2.2", "", {}, "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ=="], @@ -306,16 +316,22 @@ "fast-xml-parser": ["fast-xml-parser@4.5.3", "", { "dependencies": { "strnum": "^1.1.1" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig=="], + "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], + "filter-obj": ["filter-obj@1.1.0", "", {}, "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ=="], "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], + "flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="], + "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], "for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="], "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], + "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], + "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], @@ -346,6 +362,8 @@ "gtoken": ["gtoken@5.3.2", "", { "dependencies": { "gaxios": "^4.0.0", "google-p12-pem": "^3.1.3", "jws": "^4.0.0" } }, "sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ=="], + "h264-profile-level-id": ["h264-profile-level-id@2.3.2", "", { "dependencies": { "debug": "^4.4.3" } }, "sha512-hnq1UDlw7WGJV6GCr/g7wnkHYUjdAY2bis9rgn2JqSdQS2WfVvnt1ZE9g8nTguracodf5LLKZOwURsDN49YtBQ=="], + "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], @@ -400,6 +418,8 @@ "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + "mediasoup": ["mediasoup@3.19.17", "", { "dependencies": { "debug": "^4.4.3", "flatbuffers": "^25.9.23", "h264-profile-level-id": "^2.3.2", "node-fetch": "^3.3.2", "supports-color": "^10.2.2", "tar": "^7.5.7" } }, "sha512-wnmp/0dd56GBR5LzP+DXnDSAggykl9RncPIoUsZJffW/ggByyTqUjhC78lPGOPta+xmYLR0bKsogGQLi26S+9g=="], + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], @@ -408,6 +428,10 @@ "minio": ["minio@8.0.6", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.4.1", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-sOeh2/b/XprRmEtYsnNRFtOqNRTPDvYtMWh+spWlfsuCV/+IdxNeKVUMKLqI7b5Dr07ZqCPuaRGU/rB9pZYVdQ=="], + "minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="], + + "minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "nanostores": ["nanostores@1.1.0", "", {}, "sha512-yJBmDJr18xy47dbNVlHcgdPrulSn1nhSE6Ns9vTG+Nx9VPT6iV1MD6aQFp/t52zpf82FhLLTXAXr30NuCnxvwA=="], @@ -416,7 +440,9 @@ "node-addon-api": ["node-addon-api@8.5.0", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="], - "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], + + "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], "node-forge": ["node-forge@1.3.3", "", {}, "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg=="], @@ -536,10 +562,14 @@ "strnum": ["strnum@1.1.2", "", {}, "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA=="], + "supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], + "swagger-ui-dist": ["swagger-ui-dist@5.31.0", "", { "dependencies": { "@scarf/scarf": "=1.4.0" } }, "sha512-zSUTIck02fSga6rc0RZP3b7J7wgHXwLea8ZjgLA3Vgnb8QeOl3Wou2/j5QkzSGeoz6HusP/coYuJl33aQxQZpg=="], "swagger-ui-express": ["swagger-ui-express@5.0.1", "", { "dependencies": { "swagger-ui-dist": ">=5.0.0" }, "peerDependencies": { "express": ">=4.0.0 || >=5.0.0-beta" } }, "sha512-SrNU3RiBGTLLmFU8GIJdOdanJTl4TOmT27tt3bWWHppqYmAZ6IDuEuBvMU6nZq0zLEe6b/1rACXCgLZqO6ZfrA=="], + "tar": ["tar@7.5.7", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ=="], + "through2": ["through2@4.0.2", "", { "dependencies": { "readable-stream": "3" } }, "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], @@ -570,6 +600,8 @@ "web-encoding": ["web-encoding@1.1.5", "", { "dependencies": { "util": "^0.12.3" }, "optionalDependencies": { "@zxing/text-encoding": "0.9.0" } }, "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA=="], + "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], + "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], @@ -586,7 +618,7 @@ "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], - "yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], + "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], @@ -606,8 +638,12 @@ "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "gaxios/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], + "googleapis-common/uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="], + "lru-cache/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], + "minio/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], "proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], diff --git a/Backend/media/sfu/mediasoup.ts b/Backend/media/sfu/mediasoup.ts new file mode 100644 index 0000000..bbc78a4 --- /dev/null +++ b/Backend/media/sfu/mediasoup.ts @@ -0,0 +1,367 @@ +import { randomUUID } from 'crypto'; + +import { mediaConfig } from '../config'; +import type { + SfuConnectTransportInput, + SfuConsumeInput, + SfuConsumerDescriptor, + SfuProduceInput, + SfuProducerDescriptor, + SfuPublishTransportRequest, + SfuPublishTransportResult, + SfuService, + SfuSessionDescriptor, + SfuSessionStartInput, + SfuSubscribeTransportRequest, + SfuSubscribeTransportResult, + SfuTransportDescriptor, + SfuTransportOptions, +} from './types'; + +type DynamicMediasoup = { + createWorker: (opts?: Record) => Promise; +}; + +type MediaSessionRuntime = { + session: SfuSessionDescriptor; + router: any; + transports: Map; + transportDescriptors: Map; + producers: Map; + producerDescriptors: Map; + consumers: Map; + consumerDescriptors: Map; +}; + +const parsePort = (value: string | undefined, fallback: number): number => { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + return parsed; +}; + +const toTransportOptions = (transport: any): SfuTransportOptions => ({ + id: transport.id, + iceParameters: transport.iceParameters ?? {}, + iceCandidates: transport.iceCandidates ?? [], + dtlsParameters: transport.dtlsParameters ?? {}, + ...(transport.sctpParameters ? { sctpParameters: transport.sctpParameters } : {}), +}); + +const mediasoupCodecs = [ + { + kind: 'audio', + mimeType: 'audio/opus', + clockRate: 48000, + channels: 2, + }, + { + kind: 'video', + mimeType: 'video/VP8', + clockRate: 90000, + parameters: {}, + }, +]; + +export class MediasoupSfuService implements SfuService { + mode: 'single_server_sfu' = 'single_server_sfu'; + private mediasoupPromise: Promise | null = null; + private workerPromise: Promise | null = null; + private readonly sessions = new Map(); + + private async getMediasoup(): Promise { + if (!this.mediasoupPromise) { + this.mediasoupPromise = (new Function('return import("mediasoup")')() as Promise).catch((error) => { + throw new Error(`mediasoup package is required for MEDIA_SFU_ENGINE=mediasoup: ${error instanceof Error ? error.message : 'load failed'}`); + }); + } + return await this.mediasoupPromise; + } + + private async getWorker(): Promise { + if (!this.workerPromise) { + this.workerPromise = (async () => { + const mediasoup = await this.getMediasoup(); + const worker = await mediasoup.createWorker({ + logLevel: (process.env.MEDIA_SFU_LOG_LEVEL ?? 'warn') as 'debug' | 'warn' | 'error' | 'none', + rtcMinPort: parsePort(process.env.MEDIA_RTC_MIN_PORT, 40000), + rtcMaxPort: parsePort(process.env.MEDIA_RTC_MAX_PORT, 49999), + }); + worker.on?.('died', () => { + console.error('mediasoup worker died; clearing worker handle'); + this.workerPromise = null; + }); + return worker; + })(); + } + return await this.workerPromise; + } + + private getRuntime(streamSessionId: string): MediaSessionRuntime { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) { + throw new Error('SFU session not initialized'); + } + return runtime; + } + + private async createWebRtcTransport(router: any): Promise { + const listenIp = process.env.MEDIA_WEBRTC_LISTEN_IP ?? '0.0.0.0'; + const announcedAddress = process.env.MEDIA_WEBRTC_ANNOUNCED_IP?.trim(); + return await router.createWebRtcTransport({ + listenInfos: [ + { + protocol: 'udp', + ip: listenIp, + ...(announcedAddress ? { announcedAddress } : {}), + }, + { + protocol: 'tcp', + ip: listenIp, + ...(announcedAddress ? { announcedAddress } : {}), + }, + ], + enableUdp: true, + enableTcp: true, + preferUdp: true, + }); + } + + async startSession(input: SfuSessionStartInput): Promise { + const existing = this.sessions.get(input.streamSessionId); + if (existing) { + return existing.session; + } + + const worker = await this.getWorker(); + const router = await worker.createRouter({ mediaCodecs: mediasoupCodecs }); + const session: SfuSessionDescriptor = { + streamSessionId: input.streamSessionId, + ownerUserId: input.ownerUserId, + cameraDeviceId: input.cameraDeviceId, + requesterDeviceId: input.requesterDeviceId, + state: 'starting', + createdAt: new Date().toISOString(), + }; + + this.sessions.set(input.streamSessionId, { + session, + router, + transports: new Map(), + transportDescriptors: new Map(), + producers: new Map(), + producerDescriptors: new Map(), + consumers: new Map(), + consumerDescriptors: new Map(), + }); + return session; + } + + async setSessionState(streamSessionId: string, state: SfuSessionDescriptor['state']): Promise { + const runtime = this.getRuntime(streamSessionId); + runtime.session = { ...runtime.session, state }; + } + + async endSession(streamSessionId: string): Promise { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) return; + + runtime.session = { ...runtime.session, state: 'ending' }; + for (const consumer of runtime.consumers.values()) { + consumer.close?.(); + } + for (const producer of runtime.producers.values()) { + producer.close?.(); + } + for (const transport of runtime.transports.values()) { + transport.close?.(); + } + runtime.router.close?.(); + runtime.session = { ...runtime.session, state: 'ended' }; + this.sessions.delete(streamSessionId); + } + + async getSession(streamSessionId: string): Promise { + return this.sessions.get(streamSessionId)?.session ?? null; + } + + async getRouterRtpCapabilities(streamSessionId: string): Promise | null> { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) return null; + return runtime.router.rtpCapabilities ?? null; + } + + async listSessions(): Promise { + return Array.from(this.sessions.values()).map((runtime) => runtime.session); + } + + async listTransports(streamSessionId: string): Promise { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) return []; + return Array.from(runtime.transportDescriptors.values()); + } + + async listProducers(streamSessionId: string): Promise { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) return []; + return Array.from(runtime.producerDescriptors.values()); + } + + async listConsumers(streamSessionId: string): Promise { + const runtime = this.sessions.get(streamSessionId); + if (!runtime) return []; + return Array.from(runtime.consumerDescriptors.values()); + } + + async createPublishTransport(input: SfuPublishTransportRequest): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = await this.createWebRtcTransport(runtime.router); + const descriptor: SfuTransportDescriptor = { + transportId: transport.id, + streamSessionId: input.streamSessionId, + ownerDeviceId: input.cameraDeviceId, + direction: 'publish', + state: 'new', + createdAt: new Date().toISOString(), + }; + runtime.transports.set(transport.id, transport); + runtime.transportDescriptors.set(transport.id, descriptor); + return { + transportId: transport.id, + iceServers: mediaConfig.turn.urls.map((urls) => ({ + urls, + ...(mediaConfig.turn.username ? { username: mediaConfig.turn.username } : {}), + ...(mediaConfig.turn.credential ? { credential: mediaConfig.turn.credential } : {}), + })), + transportOptions: toTransportOptions(transport), + }; + } + + async createSubscribeTransport(input: SfuSubscribeTransportRequest): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = await this.createWebRtcTransport(runtime.router); + const descriptor: SfuTransportDescriptor = { + transportId: transport.id, + streamSessionId: input.streamSessionId, + ownerDeviceId: input.viewerDeviceId, + direction: 'subscribe', + state: 'new', + createdAt: new Date().toISOString(), + }; + runtime.transports.set(transport.id, transport); + runtime.transportDescriptors.set(transport.id, descriptor); + return { + transportId: transport.id, + iceServers: mediaConfig.turn.urls.map((urls) => ({ + urls, + ...(mediaConfig.turn.username ? { username: mediaConfig.turn.username } : {}), + ...(mediaConfig.turn.credential ? { credential: mediaConfig.turn.credential } : {}), + })), + transportOptions: toTransportOptions(transport), + }; + } + + async connectPublishTransport(input: SfuConnectTransportInput): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = runtime.transports.get(input.transportId); + const descriptor = runtime.transportDescriptors.get(input.transportId); + if (!transport || !descriptor) throw new Error('Publish transport not found'); + if (descriptor.direction !== 'publish') throw new Error('Transport is not a publish transport'); + if (descriptor.ownerDeviceId !== input.deviceId) throw new Error('Device does not own this publish transport'); + + await transport.connect({ dtlsParameters: input.dtlsParameters }); + const next = { ...descriptor, state: 'connected' as const }; + runtime.transportDescriptors.set(descriptor.transportId, next); + return next; + } + + async connectSubscribeTransport(input: SfuConnectTransportInput): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = runtime.transports.get(input.transportId); + const descriptor = runtime.transportDescriptors.get(input.transportId); + if (!transport || !descriptor) throw new Error('Subscribe transport not found'); + if (descriptor.direction !== 'subscribe') throw new Error('Transport is not a subscribe transport'); + if (descriptor.ownerDeviceId !== input.deviceId) throw new Error('Device does not own this subscribe transport'); + + await transport.connect({ dtlsParameters: input.dtlsParameters }); + const next = { ...descriptor, state: 'connected' as const }; + runtime.transportDescriptors.set(descriptor.transportId, next); + return next; + } + + async produce(input: SfuProduceInput): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = runtime.transports.get(input.transportId); + const descriptor = runtime.transportDescriptors.get(input.transportId); + if (!transport || !descriptor) throw new Error('Publish transport not found'); + if (descriptor.direction !== 'publish') throw new Error('Transport is not a publish transport'); + if (descriptor.ownerDeviceId !== input.cameraDeviceId) throw new Error('Device does not own this publish transport'); + if (descriptor.state !== 'connected') throw new Error('Publish transport must be connected before producing'); + + const producer = await transport.produce({ + kind: input.kind, + rtpParameters: input.rtpParameters, + appData: { cameraDeviceId: input.cameraDeviceId }, + }); + const producerId = producer.id ?? `prod_${randomUUID()}`; + const producerDescriptor: SfuProducerDescriptor = { + producerId, + streamSessionId: input.streamSessionId, + transportId: input.transportId, + cameraDeviceId: input.cameraDeviceId, + kind: input.kind, + rtpParameters: input.rtpParameters, + createdAt: new Date().toISOString(), + }; + runtime.producers.set(producerId, producer); + runtime.producerDescriptors.set(producerId, producerDescriptor); + return producerDescriptor; + } + + async consume(input: SfuConsumeInput): Promise { + const runtime = this.getRuntime(input.streamSessionId); + const transport = runtime.transports.get(input.transportId); + const descriptor = runtime.transportDescriptors.get(input.transportId); + if (!transport || !descriptor) throw new Error('Subscribe transport not found'); + if (descriptor.direction !== 'subscribe') throw new Error('Transport is not a subscribe transport'); + if (descriptor.ownerDeviceId !== input.viewerDeviceId) throw new Error('Device does not own this subscribe transport'); + if (descriptor.state !== 'connected') throw new Error('Subscribe transport must be connected before consuming'); + + const producerId = + input.producerId ?? + Array.from(runtime.producerDescriptors.values()) + .slice() + .reverse() + .find((producer) => producer.kind === 'video')?.producerId; + if (!producerId) throw new Error('No producer available for consume'); + + const producer = runtime.producers.get(producerId); + const producerDescriptor = runtime.producerDescriptors.get(producerId); + if (!producer || !producerDescriptor) throw new Error('Producer not found'); + if (!runtime.router.canConsume({ producerId: producer.id ?? producerId, rtpCapabilities: input.rtpCapabilities ?? {} })) { + throw new Error('Router cannot consume with provided RTP capabilities'); + } + + const consumer = await transport.consume({ + producerId: producer.id ?? producerId, + rtpCapabilities: input.rtpCapabilities ?? {}, + paused: false, + appData: { viewerDeviceId: input.viewerDeviceId }, + }); + + const consumerId = consumer.id ?? `cons_${randomUUID()}`; + const consumerDescriptor: SfuConsumerDescriptor = { + consumerId, + streamSessionId: input.streamSessionId, + transportId: input.transportId, + viewerDeviceId: input.viewerDeviceId, + producerId, + kind: producerDescriptor.kind, + rtpParameters: consumer.rtpParameters ?? producerDescriptor.rtpParameters, + createdAt: new Date().toISOString(), + }; + runtime.consumers.set(consumerId, consumer); + runtime.consumerDescriptors.set(consumerId, consumerDescriptor); + return consumerDescriptor; + } +} + diff --git a/Backend/media/sfu/noop.ts b/Backend/media/sfu/noop.ts index 71eb696..8643f12 100644 --- a/Backend/media/sfu/noop.ts +++ b/Backend/media/sfu/noop.ts @@ -64,6 +64,13 @@ export class NoopSfuService implements SfuService { return this.registry.get(streamSessionId); } + async getRouterRtpCapabilities(_streamSessionId: string): Promise | null> { + return { + codecs: [{ mimeType: 'video/VP8', clockRate: 90000, kind: 'video' }], + headerExtensions: [], + }; + } + async listSessions(): Promise { return this.registry.list(); } @@ -91,6 +98,12 @@ export class NoopSfuService implements SfuService { return { transportId, iceServers: toIceServers(), + transportOptions: { + id: transportId, + iceParameters: {}, + iceCandidates: [], + dtlsParameters: {}, + }, }; } @@ -105,6 +118,12 @@ export class NoopSfuService implements SfuService { return { transportId, iceServers: toIceServers(), + transportOptions: { + id: transportId, + iceParameters: {}, + iceCandidates: [], + dtlsParameters: {}, + }, }; } diff --git a/Backend/media/sfu/service.ts b/Backend/media/sfu/service.ts index 2dec3c6..0652638 100644 --- a/Backend/media/sfu/service.ts +++ b/Backend/media/sfu/service.ts @@ -1,14 +1,20 @@ import { mediaMode } from '../config'; +import { MediasoupSfuService } from './mediasoup'; import { NoopSfuService } from './noop'; import type { SfuService } from './types'; +const sfuEngine = (process.env.MEDIA_SFU_ENGINE ?? 'mediasoup').trim().toLowerCase(); + const createSfuService = (): SfuService | null => { if (mediaMode !== 'single_server_sfu') { return null; } - return new NoopSfuService(); + if (sfuEngine === 'noop') { + return new NoopSfuService(); + } + + return new MediasoupSfuService(); }; export const sfuService = createSfuService(); - diff --git a/Backend/media/sfu/types.ts b/Backend/media/sfu/types.ts index 74fa5f6..70575bf 100644 --- a/Backend/media/sfu/types.ts +++ b/Backend/media/sfu/types.ts @@ -9,6 +9,14 @@ export type SfuIceServer = { credential?: string; }; +export type SfuTransportOptions = { + id: string; + iceParameters: Record; + iceCandidates: Array>; + dtlsParameters: Record; + sctpParameters?: Record; +}; + export type SfuSessionDescriptor = { streamSessionId: string; ownerUserId: string; @@ -33,6 +41,7 @@ export type SfuPublishTransportRequest = { export type SfuPublishTransportResult = { transportId: string; iceServers: SfuIceServer[]; + transportOptions?: SfuTransportOptions; }; export type SfuSubscribeTransportRequest = { @@ -43,6 +52,7 @@ export type SfuSubscribeTransportRequest = { export type SfuSubscribeTransportResult = { transportId: string; iceServers: SfuIceServer[]; + transportOptions?: SfuTransportOptions; }; export type SfuTransportDescriptor = { @@ -104,6 +114,7 @@ export interface SfuService { setSessionState(streamSessionId: string, state: SfuSessionState): Promise; endSession(streamSessionId: string): Promise; getSession(streamSessionId: string): Promise; + getRouterRtpCapabilities(streamSessionId: string): Promise | null>; listSessions(): Promise; listTransports(streamSessionId: string): Promise; listProducers(streamSessionId: string): Promise; diff --git a/Backend/package.json b/Backend/package.json index 1928bb5..4fdea20 100644 --- a/Backend/package.json +++ b/Backend/package.json @@ -28,6 +28,7 @@ "drizzle-orm": "^0.44.0", "express": "^5.2.1", "helmet": "^8.1.0", + "mediasoup": "^3.15.6", "minio": "^8.0.6", "openai": "^6.18.0", "pg": "^8.18.0", @@ -46,5 +47,8 @@ "db:push": "drizzle-kit push", "db:studio": "drizzle-kit studio", "auth:migrate": "bun run scripts/migrate-better-auth.ts" - } + }, + "trustedDependencies": [ + "mediasoup" + ] } diff --git a/Backend/routes/streams.ts b/Backend/routes/streams.ts index a196ae2..81f5971 100644 --- a/Backend/routes/streams.ts +++ b/Backend/routes/streams.ts @@ -503,6 +503,44 @@ router.get('/:streamSessionId/sfu/session', requireDeviceAuth, async (req, res) }); }); +router.get('/:streamSessionId/sfu/router-rtp-capabilities', requireDeviceAuth, async (req, res) => { + const parsedParams = streamParamSchema.safeParse(req.params); + if (!parsedParams.success) { + res.status(400).json({ message: 'Invalid streamSessionId', errors: parsedParams.error.flatten() }); + return; + } + + const deviceAuth = ensureDeviceAuth(req, res); + if (!deviceAuth) return; + + const sfu = ensureSfuEnabled(res); + if (!sfu) return; + + const session = await getOwnedStreamSession(parsedParams.data.streamSessionId, deviceAuth.userId); + if (!session) { + res.status(404).json({ message: 'Stream session not found' }); + return; + } + + const isParticipant = session.requesterDeviceId === deviceAuth.deviceId || session.cameraDeviceId === deviceAuth.deviceId; + if (!isParticipant) { + res.status(403).json({ message: 'Device cannot access SFU router capabilities for this stream' }); + return; + } + + const routerRtpCapabilities = await sfu.getRouterRtpCapabilities(session.id); + if (!routerRtpCapabilities) { + res.status(409).json({ message: 'SFU session is not initialized for this stream' }); + return; + } + + res.json({ + streamSessionId: session.id, + mediaMode, + routerRtpCapabilities, + }); +}); + router.post('/:streamSessionId/sfu/publish-transport', requireDeviceAuth, async (req, res) => { const parsedParams = streamParamSchema.safeParse(req.params); if (!parsedParams.success) {