feat: Implement data source management and analysis flow, allowing users to add and analyze websites for project opportunities.

app/(app)/dashboard/page.tsx

@@ -1,20 +1,36 @@
 "use client"
 
 import { useQuery } from "convex/react"
+import { useState } from "react"
 import { api } from "@/convex/_generated/api"
 import { useProject } from "@/components/project-context"
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
 import { Badge } from "@/components/ui/badge"
+import { Alert, AlertDescription } from "@/components/ui/alert"
+import { Button } from "@/components/ui/button"
+import { useMutation } from "convex/react"
 
 export default function Page() {
     const { selectedProjectId } = useProject()
     const projects = useQuery(api.projects.getProjects)
+    const dataSources = useQuery(
+        api.dataSources.getProjectDataSources,
+        selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+    )
+    const searchContext = useQuery(
+        api.projects.getSearchContext,
+        selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+    )
+    const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
+    const createAnalysis = useMutation(api.analyses.createAnalysis)
+    const [reanalyzingId, setReanalyzingId] = useState<string | null>(null)
     const analysis = useQuery(
         api.analyses.getLatestByProject,
         selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
     )
 
     const selectedProject = projects?.find((project) => project._id === selectedProjectId)
+    const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || []
     const isLoading = selectedProjectId && analysis === undefined
 
     if (!selectedProjectId && projects && projects.length === 0) {
@@ -47,6 +63,53 @@ export default function Page() {
         )
     }
 
+    const handleReanalyze = async (source: any) => {
+        if (!selectedProjectId) return
+
+        setReanalyzingId(source._id)
+        await updateDataSourceStatus({
+            dataSourceId: source._id,
+            analysisStatus: "pending",
+            lastError: undefined,
+            lastAnalyzedAt: undefined,
+        })
+
+        try {
+            const response = await fetch("/api/analyze", {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ url: source.url }),
+            })
+
+            const data = await response.json()
+
+            if (!response.ok) {
+                await updateDataSourceStatus({
+                    dataSourceId: source._id,
+                    analysisStatus: "failed",
+                    lastError: data.error || "Analysis failed",
+                    lastAnalyzedAt: Date.now(),
+                })
+                return
+            }
+
+            await createAnalysis({
+                projectId: selectedProjectId as any,
+                dataSourceId: source._id,
+                analysis: data.data,
+            })
+
+            await updateDataSourceStatus({
+                dataSourceId: source._id,
+                analysisStatus: "completed",
+                lastError: undefined,
+                lastAnalyzedAt: Date.now(),
+            })
+        } finally {
+            setReanalyzingId(null)
+        }
+    }
+
     return (
         <div className="flex flex-1 flex-col gap-6 p-4 lg:p-8">
             <div className="space-y-2">
@@ -60,6 +123,14 @@ export default function Page() {
                 <p className="max-w-3xl text-sm text-muted-foreground">{analysis.description}</p>
             </div>
 
+            {searchContext?.missingSources?.length > 0 && (
+                <Alert>
+                    <AlertDescription>
+                        Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+                    </AlertDescription>
+                </Alert>
+            )}
+
             <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-5">
                 <Card>
                     <CardHeader className="pb-2">
@@ -93,6 +164,53 @@ export default function Page() {
                 </Card>
             </div>
 
+            <Card>
+                <CardHeader>
+                    <CardTitle className="text-base">Data Sources</CardTitle>
+                </CardHeader>
+                <CardContent className="space-y-2 text-sm text-muted-foreground">
+                    {dataSources && dataSources.length > 0 ? (
+                        dataSources.map((source: any) => (
+                            <div key={source._id} className="flex items-center justify-between gap-3">
+                                <span className="truncate">{source.name || source.url}</span>
+                                <div className="flex items-center gap-2">
+                                    <Badge variant={selectedSourceIds.includes(source._id) ? "secondary" : "outline"}>
+                                        {selectedSourceIds.includes(source._id) ? "active" : "inactive"}
+                                    </Badge>
+                                    <Badge variant={source.analysisStatus === "completed" ? "secondary" : "outline"}>
+                                        {source.analysisStatus}
+                                    </Badge>
+                                    <Button
+                                        size="sm"
+                                        variant="outline"
+                                        onClick={() => handleReanalyze(source)}
+                                        disabled={reanalyzingId === source._id}
+                                    >
+                                        {reanalyzingId === source._id ? "Analyzing..." : "Re-analyze"}
+                                    </Button>
+                                </div>
+                            </div>
+                        ))
+                    ) : (
+                        <p>No data sources yet. Add a source during onboarding.</p>
+                    )}
+                </CardContent>
+            </Card>
+
+            {searchContext?.context && (
+                <Card>
+                    <CardHeader>
+                        <CardTitle className="text-base">Aggregated Context</CardTitle>
+                    </CardHeader>
+                    <CardContent className="grid gap-3 text-sm text-muted-foreground md:grid-cols-2">
+                        <div>Keywords: <span className="text-foreground font-medium">{searchContext.context.keywords.length}</span></div>
+                        <div>Problems: <span className="text-foreground font-medium">{searchContext.context.problemsSolved.length}</span></div>
+                        <div>Competitors: <span className="text-foreground font-medium">{searchContext.context.competitors.length}</span></div>
+                        <div>Use Cases: <span className="text-foreground font-medium">{searchContext.context.useCases.length}</span></div>
+                    </CardContent>
+                </Card>
+            )}
+
             <div className="grid gap-4 lg:grid-cols-2">
                 <Card>
                     <CardHeader>

app/(app)/opportunities/page.tsx

@@ -89,6 +89,7 @@ export default function OpportunitiesPage() {
   const [replyText, setReplyText] = useState('')
   const [stats, setStats] = useState<any>(null)
   const [searchError, setSearchError] = useState('')
+  const [missingSources, setMissingSources] = useState<any[]>([])
   const [statusFilter, setStatusFilter] = useState('all')
   const [intentFilter, setIntentFilter] = useState('all')
   const [minScore, setMinScore] = useState(0)
@@ -97,6 +98,7 @@ export default function OpportunitiesPage() {
   const [notesInput, setNotesInput] = useState('')
   const [tagsInput, setTagsInput] = useState('')
 
+  const projects = useQuery(api.projects.getProjects)
   const latestAnalysis = useQuery(
     api.analyses.getLatestByProject,
     selectedProjectId ? { projectId: selectedProjectId as any } : 'skip'
@@ -137,12 +139,21 @@ export default function OpportunitiesPage() {
     })) as Opportunity[]
   }, [savedOpportunities, opportunities])
 
+  const selectedSources = useQuery(
+    api.dataSources.getProjectDataSources,
+    selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+  )
+
+  const selectedProject = projects?.find((project: any) => project._id === selectedProjectId)
+  const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || []
+  const activeSources = selectedSources?.filter((source: any) =>
+    selectedSourceIds.includes(source._id)
+  ) || []
+
   useEffect(() => {
     const stored = localStorage.getItem('productAnalysis')
     if (stored) {
       setAnalysis(JSON.parse(stored))
-    } else {
-      router.push('/onboarding')
     }
 
     fetch('/api/opportunities')
@@ -160,6 +171,18 @@ export default function OpportunitiesPage() {
       })
   }, [router])
 
+  useEffect(() => {
+    if (!analysis && latestAnalysis) {
+      setAnalysis(latestAnalysis as any)
+    }
+  }, [analysis, latestAnalysis])
+
+  useEffect(() => {
+    if (!analysis && latestAnalysis === null) {
+      router.push('/onboarding')
+    }
+  }, [analysis, latestAnalysis, router])
+
   const togglePlatform = (platformId: string) => {
     setPlatforms(prev => prev.map(p => 
       p.id === platformId ? { ...p, enabled: !p.enabled } : p
@@ -193,7 +216,7 @@ export default function OpportunitiesPage() {
       const response = await fetch('/api/opportunities', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ analysis, config })
+        body: JSON.stringify({ projectId: selectedProjectId, config })
       })
 
       if (response.redirected) {
@@ -202,7 +225,11 @@ export default function OpportunitiesPage() {
       }
 
       const data = await response.json()
-      
+
+      if (!response.ok) {
+        throw new Error(data.error || 'Failed to search for opportunities')
+      }
+
       if (data.success) {
         const mapped = data.data.opportunities.map((opp: Opportunity) => ({
           ...opp,
@@ -211,6 +238,7 @@ export default function OpportunitiesPage() {
         setOpportunities(mapped)
         setGeneratedQueries(data.data.queries)
         setStats(data.data.stats)
+        setMissingSources(data.data.missingSources || [])
 
         await upsertOpportunities({
           projectId: selectedProjectId as any,
@@ -353,7 +381,11 @@ export default function OpportunitiesPage() {
         <div className="p-4 border-t border-border">
           <Button 
             onClick={executeSearch} 
-            disabled={isSearching || platforms.filter(p => p.enabled).length === 0}
+            disabled={
+              isSearching ||
+              platforms.filter(p => p.enabled).length === 0 ||
+              selectedSourceIds.length === 0
+            }
             className="w-full"
           >
             {isSearching ? <><Loader2 className="mr-2 h-4 w-4 animate-spin" /> Searching...</> : <><Search className="mr-2 h-4 w-4" /> Find Opportunities</>}
@@ -389,6 +421,41 @@ export default function OpportunitiesPage() {
             </Alert>
           )}
 
+          {/* Active Sources */}
+          {activeSources.length > 0 && (
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-sm">Active Data Sources</CardTitle>
+                <CardDescription>
+                  Sources selected for this project will drive opportunity search.
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="flex flex-wrap gap-2">
+                {activeSources.map((source: any) => (
+                  <Badge key={source._id} variant="secondary">
+                    {source.name || source.url}
+                  </Badge>
+                ))}
+              </CardContent>
+            </Card>
+          )}
+
+          {selectedSources && selectedSourceIds.length === 0 && (
+            <Alert variant="destructive">
+              <AlertDescription>
+                No data sources selected. Add and select sources to generate opportunities.
+              </AlertDescription>
+            </Alert>
+          )}
+
+          {missingSources.length > 0 && (
+            <Alert>
+              <AlertDescription>
+                Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+              </AlertDescription>
+            </Alert>
+          )}
+
           {/* Generated Queries */}
           {generatedQueries.length > 0 && (
             <Collapsible open={showQueries} onOpenChange={setShowQueries}>

app/api/opportunities/route.ts

@@ -1,44 +1,14 @@
 import { NextRequest, NextResponse } from 'next/server'
-import { isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { fetchQuery } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { generateSearchQueries, getDefaultPlatforms } from '@/lib/query-generator'
 import { executeSearches, scoreOpportunities } from '@/lib/search-executor'
 import type { EnhancedProductAnalysis, SearchConfig, PlatformConfig } from '@/lib/types'
 
 const searchSchema = z.object({
-  analysis: z.object({
-    productName: z.string(),
-    tagline: z.string(),
-    description: z.string(),
-    features: z.array(z.object({
-      name: z.string(),
-      description: z.string(),
-      benefits: z.array(z.string()),
-      useCases: z.array(z.string())
-    })),
-    problemsSolved: z.array(z.object({
-      problem: z.string(),
-      severity: z.enum(['high', 'medium', 'low']),
-      currentWorkarounds: z.array(z.string()),
-      emotionalImpact: z.string(),
-      searchTerms: z.array(z.string())
-    })),
-    keywords: z.array(z.object({
-      term: z.string(),
-      type: z.string(),
-      searchVolume: z.string(),
-      intent: z.string(),
-      funnel: z.string(),
-      emotionalIntensity: z.string()
-    })),
-    competitors: z.array(z.object({
-      name: z.string(),
-      differentiator: z.string(),
-      theirStrength: z.string(),
-      switchTrigger: z.string(),
-      theirWeakness: z.string()
-    }))
-  }),
+  projectId: z.string(),
   config: z.object({
     platforms: z.array(z.object({
       id: z.string(),
@@ -65,7 +35,23 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { analysis, config } = searchSchema.parse(body)
+    const { projectId, config } = searchSchema.parse(body)
+
+    const token = await convexAuthNextjsToken();
+    const searchContext = await fetchQuery(
+      api.projects.getSearchContext,
+      { projectId: projectId as any },
+      { token }
+    );
+
+    if (!searchContext.context) {
+      return NextResponse.json(
+        { error: 'No analysis available for selected sources.' },
+        { status: 400 }
+      );
+    }
+
+    const analysis = searchContext.context as EnhancedProductAnalysis
 
     console.log('🔍 Starting opportunity search...')
     console.log(`   Product: ${analysis.productName}`)
@@ -105,7 +91,8 @@ export async function POST(request: NextRequest) {
           platform: q.platform,
           strategy: q.strategy,
           priority: q.priority
-        }))
+        })),
+        missingSources: searchContext.missingSources ?? []
       }
     })
 

app/onboarding/page.tsx

@@ -24,6 +24,7 @@ const examples = [
 export default function OnboardingPage() {
   const router = useRouter()
   const addDataSource = useMutation(api.dataSources.addDataSource)
+  const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
   const createAnalysis = useMutation(api.analyses.createAnalysis)
   const [url, setUrl] = useState('')
   const [loading, setLoading] = useState(false)
@@ -56,6 +57,12 @@ export default function OnboardingPage() {
       dataSourceId: sourceId,
       analysis,
     })
+
+    await updateDataSourceStatus({
+      dataSourceId: sourceId,
+      analysisStatus: 'completed',
+      lastAnalyzedAt: Date.now(),
+    })
   }
 
   async function analyzeWebsite() {