feat: Implement analysis job tracking with progress timeline and enhanced data source status management.

app/api/analyze-manual/route.ts

@@ -1,5 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server'
-import { isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { fetchMutation } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { analyzeFromText } from '@/lib/scraper'
 import { performDeepAnalysis } from '@/lib/analysis-pipeline'
@@ -7,10 +9,18 @@ import { performDeepAnalysis } from '@/lib/analysis-pipeline'
 const bodySchema = z.object({
   productName: z.string().min(1),
   description: z.string().min(1),
-  features: z.string()
+  features: z.string(),
+  jobId: z.optional(z.string())
 })
 
 export async function POST(request: NextRequest) {
+  let jobId: string | undefined
+  let timeline: {
+    key: string
+    label: string
+    status: "pending" | "running" | "completed" | "failed"
+    detail?: string
+  }[] = []
   try {
     if (!(await isAuthenticatedNextjs())) {
       const redirectUrl = new URL("/auth", request.url);
@@ -21,9 +31,69 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { productName, description, features } = bodySchema.parse(body)
+    const parsed = bodySchema.parse(body)
+    const { productName, description, features } = parsed
+    jobId = parsed.jobId
+
+    const token = await convexAuthNextjsToken();
+    timeline = [
+      { key: "scrape", label: "Prepare input", status: "pending" },
+      { key: "features", label: "Pass 1: Features", status: "pending" },
+      { key: "competitors", label: "Pass 2: Competitors", status: "pending" },
+      { key: "keywords", label: "Pass 3: Keywords", status: "pending" },
+      { key: "problems", label: "Pass 4: Problems & Personas", status: "pending" },
+      { key: "useCases", label: "Pass 5: Use cases", status: "pending" },
+      { key: "dorkQueries", label: "Pass 6: Dork queries", status: "pending" },
+      { key: "finalize", label: "Finalize analysis", status: "pending" },
+    ]
+    const updateTimeline = async ({
+      key,
+      status,
+      detail,
+      progress,
+      finalStatus,
+    }: {
+      key: string
+      status: "pending" | "running" | "completed" | "failed"
+      detail?: string
+      progress?: number
+      finalStatus?: "running" | "completed" | "failed"
+    }) => {
+      if (!jobId) return
+      timeline = timeline.map((item) =>
+        item.key === key ? { ...item, status, detail: detail ?? item.detail } : item
+      )
+      await fetchMutation(
+        api.analysisJobs.update,
+        {
+          jobId: jobId as any,
+          status: finalStatus || "running",
+          progress,
+          stage: key,
+          timeline,
+        },
+        { token }
+      )
+    }
+    if (jobId) {
+      await updateTimeline({ key: "scrape", status: "running", progress: 10 })
+    }
 
     if (!process.env.OPENAI_API_KEY) {
+      if (jobId) {
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: "OpenAI API key not configured",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      }
       return NextResponse.json(
         { error: 'OpenAI API key not configured' },
         { status: 500 }
@@ -32,10 +102,49 @@ export async function POST(request: NextRequest) {
 
     console.log('📝 Creating content from manual input...')
     const scrapedContent = await analyzeFromText(productName, description, features)
+    if (jobId) {
+      await updateTimeline({
+        key: "scrape",
+        status: "completed",
+        detail: "Manual input prepared",
+        progress: 20,
+      })
+    }
 
     console.log('🤖 Starting enhanced analysis...')
-    const analysis = await performDeepAnalysis(scrapedContent)
+    const progressMap: Record<string, number> = {
+      features: 35,
+      competitors: 50,
+      keywords: 65,
+      problems: 78,
+      useCases: 88,
+      dorkQueries: 95,
+    }
+    const analysis = await performDeepAnalysis(scrapedContent, async (update) => {
+      await updateTimeline({
+        key: update.key,
+        status: update.status,
+        detail: update.detail,
+        progress: progressMap[update.key] ?? 80,
+      })
+    })
     console.log(`   ✓ Analysis complete: ${analysis.features.length} features, ${analysis.keywords.length} keywords`)
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "running",
+        progress: 98,
+      })
+    }
+
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "completed",
+        progress: 100,
+        finalStatus: "completed",
+      })
+    }
 
     return NextResponse.json({
       success: true,
@@ -52,6 +161,26 @@ export async function POST(request: NextRequest) {
 
   } catch (error: any) {
     console.error('❌ Manual analysis error:', error)
+
+    if (jobId) {
+      try {
+        const token = await convexAuthNextjsToken();
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: error.message || "Manual analysis failed",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      } catch {
+        // Best-effort job update only.
+      }
+    }
     
     if (error.name === 'ZodError') {
       return NextResponse.json(