feat: Implement analysis job tracking with progress timeline and enhanced data source status management.

app/onboarding/page.tsx

@@ -11,8 +11,9 @@ import { Skeleton } from '@/components/ui/skeleton'
 import { Alert, AlertDescription } from '@/components/ui/alert'
 import { ArrowRight, Globe, Loader2, Sparkles, AlertCircle, ArrowLeft } from 'lucide-react'
 import type { EnhancedProductAnalysis, Keyword } from '@/lib/types'
-import { useMutation } from 'convex/react'
+import { useMutation, useQuery } from 'convex/react'
 import { api } from '@/convex/_generated/api'
+import { AnalysisTimeline } from '@/components/analysis-timeline'
 
 const examples = [
   { name: 'Notion', url: 'https://notion.so' },
@@ -26,6 +27,7 @@ export default function OnboardingPage() {
   const addDataSource = useMutation(api.dataSources.addDataSource)
   const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
   const createAnalysis = useMutation(api.analyses.createAnalysis)
+  const createAnalysisJob = useMutation(api.analysisJobs.create)
   const [url, setUrl] = useState('')
   const [loading, setLoading] = useState(false)
   const [progress, setProgress] = useState('')
@@ -36,33 +38,56 @@ export default function OnboardingPage() {
   const [manualProductName, setManualProductName] = useState('')
   const [manualDescription, setManualDescription] = useState('')
   const [manualFeatures, setManualFeatures] = useState('')
+  const [pendingSourceId, setPendingSourceId] = useState<string | null>(null)
+  const [pendingProjectId, setPendingProjectId] = useState<string | null>(null)
+  const [pendingJobId, setPendingJobId] = useState<string | null>(null)
+  const analysisJob = useQuery(
+    api.analysisJobs.getById,
+    pendingJobId ? { jobId: pendingJobId as any } : "skip"
+  )
 
   const persistAnalysis = async ({
     analysis,
     sourceUrl,
     sourceName,
+    projectId,
+    dataSourceId,
   }: {
     analysis: EnhancedProductAnalysis
     sourceUrl: string
     sourceName: string
+    projectId?: string
+    dataSourceId?: string
   }) => {
-    const { sourceId, projectId } = await addDataSource({
-      url: sourceUrl,
-      name: sourceName,
-      type: 'website',
-    })
+    const resolved = projectId && dataSourceId
+      ? { projectId, sourceId: dataSourceId }
+      : await addDataSource({
+          url: sourceUrl,
+          name: sourceName,
+          type: 'website',
+        })
 
-    await createAnalysis({
-      projectId,
-      dataSourceId: sourceId,
-      analysis,
-    })
+    try {
+      await createAnalysis({
+        projectId: resolved.projectId,
+        dataSourceId: resolved.sourceId,
+        analysis,
+      })
 
-    await updateDataSourceStatus({
-      dataSourceId: sourceId,
-      analysisStatus: 'completed',
-      lastAnalyzedAt: Date.now(),
-    })
+      await updateDataSourceStatus({
+        dataSourceId: resolved.sourceId,
+        analysisStatus: 'completed',
+        lastAnalyzedAt: Date.now(),
+      })
+    } catch (err: any) {
+      await updateDataSourceStatus({
+        dataSourceId: resolved.sourceId,
+        analysisStatus: 'failed',
+        lastError: err?.message || 'Failed to save analysis',
+        lastAnalyzedAt: Date.now(),
+      })
+      throw err
+    }
   }
 
   async function analyzeWebsite() {
@@ -71,12 +96,35 @@ export default function OnboardingPage() {
     setLoading(true)
     setError('')
     setProgress('Scraping website...')
+    let manualFallback = false
 
     try {
+      const { sourceId, projectId } = await addDataSource({
+        url,
+        name: url.replace(/^https?:\/\//, '').replace(/\/$/, ''),
+        type: 'website',
+      })
+
+      await updateDataSourceStatus({
+        dataSourceId: sourceId,
+        analysisStatus: 'pending',
+        lastError: undefined,
+        lastAnalyzedAt: undefined,
+      })
+
+      setPendingSourceId(sourceId)
+      setPendingProjectId(projectId)
+
+      const jobId = await createAnalysisJob({
+        projectId,
+        dataSourceId: sourceId,
+      })
+      setPendingJobId(jobId)
+
       const response = await fetch('/api/analyze', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ url }),
+        body: JSON.stringify({ url, jobId }),
       })
 
       if (response.redirected) {
@@ -90,6 +138,7 @@ export default function OnboardingPage() {
         if (data.needsManualInput) {
           setShowManualInput(true)
           setManualProductName(url.replace(/^https?:\/\//, '').replace(/\/$/, ''))
+          manualFallback = true
           throw new Error(data.error)
         }
         throw new Error(data.error || 'Failed to analyze')
@@ -106,8 +155,14 @@ export default function OnboardingPage() {
         analysis: data.data,
         sourceUrl: url,
         sourceName: data.data.productName,
+        projectId,
+        dataSourceId: sourceId,
       })
 
+      setPendingSourceId(null)
+      setPendingProjectId(null)
+      setPendingJobId(null)
+
       setProgress('Redirecting to dashboard...')
 
       // Redirect to dashboard with product name in query
@@ -116,6 +171,14 @@ export default function OnboardingPage() {
     } catch (err: any) {
       console.error('Analysis error:', err)
       setError(err.message || 'Failed to analyze website')
+      if (pendingSourceId && !manualFallback) {
+        await updateDataSourceStatus({
+          dataSourceId: pendingSourceId,
+          analysisStatus: 'failed',
+          lastError: err?.message || 'Failed to analyze',
+          lastAnalyzedAt: Date.now(),
+        })
+      }
     } finally {
       setLoading(false)
     }
@@ -172,13 +235,38 @@ export default function OnboardingPage() {
       }
 
       // Send to API to enhance with AI
+      let resolvedProjectId = pendingProjectId
+      let resolvedSourceId = pendingSourceId
+      let resolvedJobId = pendingJobId
+
+      if (!resolvedProjectId || !resolvedSourceId) {
+        const created = await addDataSource({
+          url: `manual:${manualProductName}`,
+          name: manualProductName,
+          type: 'website',
+        })
+        resolvedProjectId = created.projectId
+        resolvedSourceId = created.sourceId
+        setPendingProjectId(created.projectId)
+        setPendingSourceId(created.sourceId)
+      }
+
+      if (!resolvedJobId && resolvedProjectId && resolvedSourceId) {
+        resolvedJobId = await createAnalysisJob({
+          projectId: resolvedProjectId,
+          dataSourceId: resolvedSourceId,
+        })
+        setPendingJobId(resolvedJobId)
+      }
+
       const response = await fetch('/api/analyze-manual', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({
           productName: manualProductName,
           description: manualDescription,
-          features: manualFeatures
+          features: manualFeatures,
+          jobId: resolvedJobId || undefined,
         }),
       })
 
@@ -206,18 +294,36 @@ export default function OnboardingPage() {
       }))
 
       setProgress('Saving analysis...')
+      const manualSourceUrl = pendingSourceId
+        ? 'manual-input'
+        : `manual:${finalAnalysis.productName}`
+
       await persistAnalysis({
         analysis: finalAnalysis,
-        sourceUrl: 'manual-input',
+        sourceUrl: manualSourceUrl,
         sourceName: finalAnalysis.productName,
+        projectId: resolvedProjectId || undefined,
+        dataSourceId: resolvedSourceId || undefined,
       })
 
+      setPendingSourceId(null)
+      setPendingProjectId(null)
+      setPendingJobId(null)
+
       // Redirect to dashboard
       const params = new URLSearchParams({ product: finalAnalysis.productName })
       router.push(`/dashboard?${params.toString()}`)
     } catch (err: any) {
       console.error('Manual analysis error:', err)
       setError(err.message || 'Failed to analyze')
+      if (pendingSourceId) {
+        await updateDataSourceStatus({
+          dataSourceId: pendingSourceId,
+          analysisStatus: 'failed',
+          lastError: err?.message || 'Failed to analyze',
+          lastAnalyzedAt: Date.now(),
+        })
+      }
     } finally {
       setLoading(false)
     }
@@ -294,10 +400,14 @@ export default function OnboardingPage() {
                       <Loader2 className="h-4 w-4 animate-spin" />
                       {progress}
                     </div>
-                    <div className="space-y-2">
-                      <Skeleton className="h-4 w-full" />
-                      <Skeleton className="h-4 w-4/5" />
-                    </div>
+                    {analysisJob?.timeline?.length ? (
+                      <AnalysisTimeline items={analysisJob.timeline} />
+                    ) : (
+                      <div className="space-y-2">
+                        <Skeleton className="h-4 w-full" />
+                        <Skeleton className="h-4 w-4/5" />
+                      </div>
+                    )}
                   </div>
                 )}
 
@@ -400,11 +510,15 @@ export default function OnboardingPage() {
                     <Loader2 className="h-4 w-4 animate-spin" />
                     {progress}
                   </div>
-                  <div className="space-y-2">
-                    <Skeleton className="h-4 w-full" />
-                    <Skeleton className="h-4 w-4/5" />
-                    <Skeleton className="h-4 w-3/5" />
-                  </div>
+                  {analysisJob?.timeline?.length ? (
+                    <AnalysisTimeline items={analysisJob.timeline} />
+                  ) : (
+                    <div className="space-y-2">
+                      <Skeleton className="h-4 w-full" />
+                      <Skeleton className="h-4 w-4/5" />
+                      <Skeleton className="h-4 w-3/5" />
+                    </div>
+                  )}
                 </div>
               )}