feat(observability): add phase9 health, readiness, metrics, request tracing, and simulator ops checks

Backend/index.ts

@@ -17,7 +17,9 @@ import streamsRoutes from './routes/streams';
 import recordingsRoutes from './routes/recordings';
 import pushNotificationsRoutes from './routes/push-notifications';
 import auditRoutes from './routes/audit';
+import opsRoutes from './routes/ops';
 import { rateLimit } from './middleware/security';
+import { requestContext } from './middleware/observability';
 import { setupRealtimeGateway } from './realtime/gateway';
 import { ensureMinioBucket } from './utils/minio';
 import { startRecordingsWorker } from './workers/recordings';
@@ -49,6 +51,7 @@ app.use(
   }),
 );
 app.use(rateLimit({ keyPrefix: 'global', windowMs: 60_000, max: 400 }));
+app.use(requestContext);
 app.use(express.json());
 app.use('/sim', express.static('public'));
 app.use('/videos', videosRoutes);
@@ -61,6 +64,7 @@ app.use('/streams', rateLimit({ keyPrefix: 'streams', windowMs: 60_000, max: 120
 app.use('/recordings', rateLimit({ keyPrefix: 'recordings', windowMs: 60_000, max: 120 }), recordingsRoutes);
 app.use('/push-notifications', pushNotificationsRoutes);
 app.use('/audit', auditRoutes);
+app.use('/ops', opsRoutes);
 
 app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
   console.error(err);

Backend/middleware/observability.ts

@@ -0,0 +1,31 @@
+import { randomUUID } from 'crypto';
+
+import type { NextFunction, Request, Response } from 'express';
+
+import { incrementMetric } from '../observability/metrics';
+
+export const requestContext = (req: Request, res: Response, next: NextFunction): void => {
+  const requestId = req.headers['x-request-id']?.toString() ?? randomUUID();
+  const start = Date.now();
+
+  res.setHeader('x-request-id', requestId);
+  incrementMetric('http.requests.total');
+
+  res.on('finish', () => {
+    const durationMs = Date.now() - start;
+    incrementMetric(`http.status.${res.statusCode}`);
+
+    console.log(
+      JSON.stringify({
+        requestId,
+        method: req.method,
+        path: req.path,
+        statusCode: res.statusCode,
+        durationMs,
+        ip: req.ip,
+      }),
+    );
+  });
+
+  next();
+};

Backend/observability/metrics.ts

@@ -0,0 +1,15 @@
+const counters = new Map<string, number>();
+
+export const incrementMetric = (name: string, value = 1): void => {
+  counters.set(name, (counters.get(name) ?? 0) + value);
+};
+
+export const getAllMetrics = (): Record<string, number> => {
+  const result: Record<string, number> = {};
+
+  for (const [key, value] of counters.entries()) {
+    result[key] = value;
+  }
+
+  return result;
+};

Backend/public/mobile-sim.html

@@ -705,6 +705,46 @@
         }
       });
 
+      const opsPanel = document.createElement('section');
+      opsPanel.className = 'panel';
+      opsPanel.style.marginTop = '16px';
+      opsPanel.innerHTML = `
+        <h2>Ops Checks</h2>
+        <div class="row">
+          <button id="checkLiveBtn" class="alt">GET /ops/live</button>
+          <button id="checkReadyBtn" class="alt">GET /ops/ready</button>
+        </div>
+        <button id="checkMetricsBtn" class="alt">GET /ops/metrics</button>
+      `;
+      document.querySelector('.page').appendChild(opsPanel);
+
+      $('checkLiveBtn').addEventListener('click', async () => {
+        try {
+          const payload = await authFetch('/ops/live');
+          log('ops live', payload);
+        } catch (error) {
+          log('ops live failed', { error: error.message });
+        }
+      });
+
+      $('checkReadyBtn').addEventListener('click', async () => {
+        try {
+          const payload = await authFetch('/ops/ready');
+          log('ops ready', payload);
+        } catch (error) {
+          log('ops ready failed', { error: error.message });
+        }
+      });
+
+      $('checkMetricsBtn').addEventListener('click', async () => {
+        try {
+          const payload = await authFetch('/ops/metrics');
+          log('ops metrics', payload);
+        } catch (error) {
+          log('ops metrics failed', { error: error.message });
+        }
+      });
+
       render();
     </script>
   </body>

Backend/routes/ops.ts

@@ -0,0 +1,44 @@
+import { Router } from 'express';
+
+import { db } from '../db/client';
+import { mediaProvider } from '../media/service';
+import { getAllMetrics } from '../observability/metrics';
+import { minioBucket, minioClient } from '../utils/minio';
+
+const router = Router();
+
+router.get('/live', (_req, res) => {
+  res.json({ status: 'ok', service: 'backend', timestamp: new Date().toISOString() });
+});
+
+router.get('/ready', async (_req, res) => {
+  try {
+    await db.execute('select 1');
+    await minioClient.bucketExists(minioBucket);
+
+    res.json({
+      status: 'ready',
+      checks: {
+        database: 'ok',
+        minio: 'ok',
+        mediaProvider: mediaProvider.name,
+      },
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    res.status(503).json({
+      status: 'not_ready',
+      error: error instanceof Error ? error.message : 'unknown error',
+      timestamp: new Date().toISOString(),
+    });
+  }
+});
+
+router.get('/metrics', (_req, res) => {
+  res.json({
+    timestamp: new Date().toISOString(),
+    metrics: getAllMetrics(),
+  });
+});
+
+export default router;