feat: Implement data source management and analysis flow, allowing users to add and analyze websites for project opportunities.

app/(app)/dashboard/page.tsx

@@ -1,20 +1,36 @@
 "use client"
 
 import { useQuery } from "convex/react"
+import { useState } from "react"
 import { api } from "@/convex/_generated/api"
 import { useProject } from "@/components/project-context"
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
 import { Badge } from "@/components/ui/badge"
+import { Alert, AlertDescription } from "@/components/ui/alert"
+import { Button } from "@/components/ui/button"
+import { useMutation } from "convex/react"
 
 export default function Page() {
     const { selectedProjectId } = useProject()
     const projects = useQuery(api.projects.getProjects)
+    const dataSources = useQuery(
+        api.dataSources.getProjectDataSources,
+        selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+    )
+    const searchContext = useQuery(
+        api.projects.getSearchContext,
+        selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+    )
+    const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
+    const createAnalysis = useMutation(api.analyses.createAnalysis)
+    const [reanalyzingId, setReanalyzingId] = useState<string | null>(null)
     const analysis = useQuery(
         api.analyses.getLatestByProject,
         selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
     )
 
     const selectedProject = projects?.find((project) => project._id === selectedProjectId)
+    const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || []
     const isLoading = selectedProjectId && analysis === undefined
 
     if (!selectedProjectId && projects && projects.length === 0) {
@@ -47,6 +63,53 @@ export default function Page() {
         )
     }
 
+    const handleReanalyze = async (source: any) => {
+        if (!selectedProjectId) return
+
+        setReanalyzingId(source._id)
+        await updateDataSourceStatus({
+            dataSourceId: source._id,
+            analysisStatus: "pending",
+            lastError: undefined,
+            lastAnalyzedAt: undefined,
+        })
+
+        try {
+            const response = await fetch("/api/analyze", {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ url: source.url }),
+            })
+
+            const data = await response.json()
+
+            if (!response.ok) {
+                await updateDataSourceStatus({
+                    dataSourceId: source._id,
+                    analysisStatus: "failed",
+                    lastError: data.error || "Analysis failed",
+                    lastAnalyzedAt: Date.now(),
+                })
+                return
+            }
+
+            await createAnalysis({
+                projectId: selectedProjectId as any,
+                dataSourceId: source._id,
+                analysis: data.data,
+            })
+
+            await updateDataSourceStatus({
+                dataSourceId: source._id,
+                analysisStatus: "completed",
+                lastError: undefined,
+                lastAnalyzedAt: Date.now(),
+            })
+        } finally {
+            setReanalyzingId(null)
+        }
+    }
+
     return (
         <div className="flex flex-1 flex-col gap-6 p-4 lg:p-8">
             <div className="space-y-2">
@@ -60,6 +123,14 @@ export default function Page() {
                 <p className="max-w-3xl text-sm text-muted-foreground">{analysis.description}</p>
             </div>
 
+            {searchContext?.missingSources?.length > 0 && (
+                <Alert>
+                    <AlertDescription>
+                        Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+                    </AlertDescription>
+                </Alert>
+            )}
+
             <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-5">
                 <Card>
                     <CardHeader className="pb-2">
@@ -93,6 +164,53 @@ export default function Page() {
                 </Card>
             </div>
 
+            <Card>
+                <CardHeader>
+                    <CardTitle className="text-base">Data Sources</CardTitle>
+                </CardHeader>
+                <CardContent className="space-y-2 text-sm text-muted-foreground">
+                    {dataSources && dataSources.length > 0 ? (
+                        dataSources.map((source: any) => (
+                            <div key={source._id} className="flex items-center justify-between gap-3">
+                                <span className="truncate">{source.name || source.url}</span>
+                                <div className="flex items-center gap-2">
+                                    <Badge variant={selectedSourceIds.includes(source._id) ? "secondary" : "outline"}>
+                                        {selectedSourceIds.includes(source._id) ? "active" : "inactive"}
+                                    </Badge>
+                                    <Badge variant={source.analysisStatus === "completed" ? "secondary" : "outline"}>
+                                        {source.analysisStatus}
+                                    </Badge>
+                                    <Button
+                                        size="sm"
+                                        variant="outline"
+                                        onClick={() => handleReanalyze(source)}
+                                        disabled={reanalyzingId === source._id}
+                                    >
+                                        {reanalyzingId === source._id ? "Analyzing..." : "Re-analyze"}
+                                    </Button>
+                                </div>
+                            </div>
+                        ))
+                    ) : (
+                        <p>No data sources yet. Add a source during onboarding.</p>
+                    )}
+                </CardContent>
+            </Card>
+
+            {searchContext?.context && (
+                <Card>
+                    <CardHeader>
+                        <CardTitle className="text-base">Aggregated Context</CardTitle>
+                    </CardHeader>
+                    <CardContent className="grid gap-3 text-sm text-muted-foreground md:grid-cols-2">
+                        <div>Keywords: <span className="text-foreground font-medium">{searchContext.context.keywords.length}</span></div>
+                        <div>Problems: <span className="text-foreground font-medium">{searchContext.context.problemsSolved.length}</span></div>
+                        <div>Competitors: <span className="text-foreground font-medium">{searchContext.context.competitors.length}</span></div>
+                        <div>Use Cases: <span className="text-foreground font-medium">{searchContext.context.useCases.length}</span></div>
+                    </CardContent>
+                </Card>
+            )}
+
             <div className="grid gap-4 lg:grid-cols-2">
                 <Card>
                     <CardHeader>

app/(app)/opportunities/page.tsx

@@ -89,6 +89,7 @@ export default function OpportunitiesPage() {
   const [replyText, setReplyText] = useState('')
   const [stats, setStats] = useState<any>(null)
   const [searchError, setSearchError] = useState('')
+  const [missingSources, setMissingSources] = useState<any[]>([])
   const [statusFilter, setStatusFilter] = useState('all')
   const [intentFilter, setIntentFilter] = useState('all')
   const [minScore, setMinScore] = useState(0)
@@ -97,6 +98,7 @@ export default function OpportunitiesPage() {
   const [notesInput, setNotesInput] = useState('')
   const [tagsInput, setTagsInput] = useState('')
 
+  const projects = useQuery(api.projects.getProjects)
   const latestAnalysis = useQuery(
     api.analyses.getLatestByProject,
     selectedProjectId ? { projectId: selectedProjectId as any } : 'skip'
@@ -137,12 +139,21 @@ export default function OpportunitiesPage() {
     })) as Opportunity[]
   }, [savedOpportunities, opportunities])
 
+  const selectedSources = useQuery(
+    api.dataSources.getProjectDataSources,
+    selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+  )
+
+  const selectedProject = projects?.find((project: any) => project._id === selectedProjectId)
+  const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || []
+  const activeSources = selectedSources?.filter((source: any) =>
+    selectedSourceIds.includes(source._id)
+  ) || []
+
   useEffect(() => {
     const stored = localStorage.getItem('productAnalysis')
     if (stored) {
       setAnalysis(JSON.parse(stored))
-    } else {
-      router.push('/onboarding')
     }
 
     fetch('/api/opportunities')
@@ -160,6 +171,18 @@ export default function OpportunitiesPage() {
       })
   }, [router])
 
+  useEffect(() => {
+    if (!analysis && latestAnalysis) {
+      setAnalysis(latestAnalysis as any)
+    }
+  }, [analysis, latestAnalysis])
+
+  useEffect(() => {
+    if (!analysis && latestAnalysis === null) {
+      router.push('/onboarding')
+    }
+  }, [analysis, latestAnalysis, router])
+
   const togglePlatform = (platformId: string) => {
     setPlatforms(prev => prev.map(p => 
       p.id === platformId ? { ...p, enabled: !p.enabled } : p
@@ -193,7 +216,7 @@ export default function OpportunitiesPage() {
       const response = await fetch('/api/opportunities', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ analysis, config })
+        body: JSON.stringify({ projectId: selectedProjectId, config })
       })
 
       if (response.redirected) {
@@ -203,6 +226,10 @@ export default function OpportunitiesPage() {
 
       const data = await response.json()
 
+      if (!response.ok) {
+        throw new Error(data.error || 'Failed to search for opportunities')
+      }
+
       if (data.success) {
         const mapped = data.data.opportunities.map((opp: Opportunity) => ({
           ...opp,
@@ -211,6 +238,7 @@ export default function OpportunitiesPage() {
         setOpportunities(mapped)
         setGeneratedQueries(data.data.queries)
         setStats(data.data.stats)
+        setMissingSources(data.data.missingSources || [])
 
         await upsertOpportunities({
           projectId: selectedProjectId as any,
@@ -353,7 +381,11 @@ export default function OpportunitiesPage() {
         <div className="p-4 border-t border-border">
           <Button 
             onClick={executeSearch} 
-            disabled={isSearching || platforms.filter(p => p.enabled).length === 0}
+            disabled={
+              isSearching ||
+              platforms.filter(p => p.enabled).length === 0 ||
+              selectedSourceIds.length === 0
+            }
             className="w-full"
           >
             {isSearching ? <><Loader2 className="mr-2 h-4 w-4 animate-spin" /> Searching...</> : <><Search className="mr-2 h-4 w-4" /> Find Opportunities</>}
@@ -389,6 +421,41 @@ export default function OpportunitiesPage() {
             </Alert>
           )}
 
+          {/* Active Sources */}
+          {activeSources.length > 0 && (
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-sm">Active Data Sources</CardTitle>
+                <CardDescription>
+                  Sources selected for this project will drive opportunity search.
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="flex flex-wrap gap-2">
+                {activeSources.map((source: any) => (
+                  <Badge key={source._id} variant="secondary">
+                    {source.name || source.url}
+                  </Badge>
+                ))}
+              </CardContent>
+            </Card>
+          )}
+
+          {selectedSources && selectedSourceIds.length === 0 && (
+            <Alert variant="destructive">
+              <AlertDescription>
+                No data sources selected. Add and select sources to generate opportunities.
+              </AlertDescription>
+            </Alert>
+          )}
+
+          {missingSources.length > 0 && (
+            <Alert>
+              <AlertDescription>
+                Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+              </AlertDescription>
+            </Alert>
+          )}
+
           {/* Generated Queries */}
           {generatedQueries.length > 0 && (
             <Collapsible open={showQueries} onOpenChange={setShowQueries}>

app/api/opportunities/route.ts

@@ -1,44 +1,14 @@
 import { NextRequest, NextResponse } from 'next/server'
-import { isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { fetchQuery } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { generateSearchQueries, getDefaultPlatforms } from '@/lib/query-generator'
 import { executeSearches, scoreOpportunities } from '@/lib/search-executor'
 import type { EnhancedProductAnalysis, SearchConfig, PlatformConfig } from '@/lib/types'
 
 const searchSchema = z.object({
-  analysis: z.object({
+  projectId: z.string(),
-    productName: z.string(),
-    tagline: z.string(),
-    description: z.string(),
-    features: z.array(z.object({
-      name: z.string(),
-      description: z.string(),
-      benefits: z.array(z.string()),
-      useCases: z.array(z.string())
-    })),
-    problemsSolved: z.array(z.object({
-      problem: z.string(),
-      severity: z.enum(['high', 'medium', 'low']),
-      currentWorkarounds: z.array(z.string()),
-      emotionalImpact: z.string(),
-      searchTerms: z.array(z.string())
-    })),
-    keywords: z.array(z.object({
-      term: z.string(),
-      type: z.string(),
-      searchVolume: z.string(),
-      intent: z.string(),
-      funnel: z.string(),
-      emotionalIntensity: z.string()
-    })),
-    competitors: z.array(z.object({
-      name: z.string(),
-      differentiator: z.string(),
-      theirStrength: z.string(),
-      switchTrigger: z.string(),
-      theirWeakness: z.string()
-    }))
-  }),
   config: z.object({
     platforms: z.array(z.object({
       id: z.string(),
@@ -65,7 +35,23 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { analysis, config } = searchSchema.parse(body)
+    const { projectId, config } = searchSchema.parse(body)
+
+    const token = await convexAuthNextjsToken();
+    const searchContext = await fetchQuery(
+      api.projects.getSearchContext,
+      { projectId: projectId as any },
+      { token }
+    );
+
+    if (!searchContext.context) {
+      return NextResponse.json(
+        { error: 'No analysis available for selected sources.' },
+        { status: 400 }
+      );
+    }
+
+    const analysis = searchContext.context as EnhancedProductAnalysis
 
     console.log('🔍 Starting opportunity search...')
     console.log(`   Product: ${analysis.productName}`)
@@ -105,7 +91,8 @@ export async function POST(request: NextRequest) {
           platform: q.platform,
           strategy: q.strategy,
           priority: q.priority
-        }))
+        })),
+        missingSources: searchContext.missingSources ?? []
       }
     })
 

app/onboarding/page.tsx

@@ -24,6 +24,7 @@ const examples = [
 export default function OnboardingPage() {
   const router = useRouter()
   const addDataSource = useMutation(api.dataSources.addDataSource)
+  const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
   const createAnalysis = useMutation(api.analyses.createAnalysis)
   const [url, setUrl] = useState('')
   const [loading, setLoading] = useState(false)
@@ -56,6 +57,12 @@ export default function OnboardingPage() {
       dataSourceId: sourceId,
       analysis,
     })
+
+    await updateDataSourceStatus({
+      dataSourceId: sourceId,
+      analysisStatus: 'completed',
+      lastAnalyzedAt: Date.now(),
+    })
   }
 
   async function analyzeWebsite() {

components/app-sidebar.tsx

@@ -31,11 +31,22 @@ import { api } from "@/convex/_generated/api"
 import { Checkbox } from "@/components/ui/checkbox"
 import { Label } from "@/components/ui/label"
 import { useProject } from "@/components/project-context"
+import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
+import { Input } from "@/components/ui/input"
+import { Button } from "@/components/ui/button"
 
 export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
     const pathname = usePathname()
     const projects = useQuery(api.projects.getProjects);
     const { selectedProjectId, setSelectedProjectId } = useProject();
+    const addDataSource = useMutation(api.dataSources.addDataSource);
+    const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus);
+    const createAnalysis = useMutation(api.analyses.createAnalysis);
+    const [isAdding, setIsAdding] = React.useState(false);
+    const [sourceUrl, setSourceUrl] = React.useState("");
+    const [sourceName, setSourceName] = React.useState("");
+    const [sourceError, setSourceError] = React.useState<string | null>(null);
+    const [isSubmittingSource, setIsSubmittingSource] = React.useState(false);
 
     // Set default selected project
     React.useEffect(() => {
@@ -66,6 +77,71 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
         });
     };
 
+    const handleAddSource = async () => {
+        if (!sourceUrl) {
+            setSourceError("Please enter a URL.");
+            return;
+        }
+
+        setSourceError(null);
+        setIsSubmittingSource(true);
+
+        try {
+            const { sourceId, projectId } = await addDataSource({
+                projectId: selectedProjectId as any,
+                url: sourceUrl,
+                name: sourceName || sourceUrl,
+                type: "website",
+            });
+
+            await updateDataSourceStatus({
+                dataSourceId: sourceId,
+                analysisStatus: "pending",
+                lastError: undefined,
+                lastAnalyzedAt: undefined,
+            });
+
+            const response = await fetch("/api/analyze", {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ url: sourceUrl }),
+            });
+
+            const data = await response.json();
+
+            if (!response.ok) {
+                await updateDataSourceStatus({
+                    dataSourceId: sourceId,
+                    analysisStatus: "failed",
+                    lastError: data.error || "Analysis failed",
+                    lastAnalyzedAt: Date.now(),
+                });
+                throw new Error(data.error || "Analysis failed");
+            }
+
+            await createAnalysis({
+                projectId,
+                dataSourceId: sourceId,
+                analysis: data.data,
+            });
+
+            await updateDataSourceStatus({
+                dataSourceId: sourceId,
+                analysisStatus: "completed",
+                lastError: undefined,
+                lastAnalyzedAt: Date.now(),
+            });
+
+            setSourceUrl("");
+            setSourceName("");
+            setIsAdding(false);
+        } catch (err: any) {
+            setSourceError(err?.message || "Failed to add source.");
+        } finally {
+            setIsSubmittingSource(false);
+        }
+    };
+
     return (
         <Sidebar variant="inset" {...props}>
             <SidebarHeader>
@@ -193,6 +269,13 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                                         </Label>
                                     </div>
                                 ))}
+                                <Button
+                                    variant="outline"
+                                    size="sm"
+                                    onClick={() => setIsAdding(true)}
+                                >
+                                    Add Data Source
+                                </Button>
                             </div>
                         </SidebarGroupContent>
                     </SidebarGroup>
@@ -205,6 +288,49 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                     avatar: ""
                 }} />
             </SidebarFooter>
+            <Dialog open={isAdding} onOpenChange={setIsAdding}>
+                <DialogContent>
+                    <DialogHeader>
+                        <DialogTitle>Add Data Source</DialogTitle>
+                        <DialogDescription>
+                            Add a website to analyze for this project.
+                        </DialogDescription>
+                    </DialogHeader>
+                    <div className="space-y-3">
+                        <div className="space-y-2">
+                            <Label htmlFor="sourceUrl">Website URL</Label>
+                            <Input
+                                id="sourceUrl"
+                                placeholder="https://example.com"
+                                value={sourceUrl}
+                                onChange={(event) => setSourceUrl(event.target.value)}
+                                disabled={isSubmittingSource}
+                            />
+                        </div>
+                        <div className="space-y-2">
+                            <Label htmlFor="sourceName">Name (optional)</Label>
+                            <Input
+                                id="sourceName"
+                                placeholder="Product name"
+                                value={sourceName}
+                                onChange={(event) => setSourceName(event.target.value)}
+                                disabled={isSubmittingSource}
+                            />
+                        </div>
+                        {sourceError && (
+                            <div className="text-sm text-destructive">{sourceError}</div>
+                        )}
+                        <div className="flex justify-end gap-2">
+                            <Button variant="outline" onClick={() => setIsAdding(false)} disabled={isSubmittingSource}>
+                                Cancel
+                            </Button>
+                            <Button onClick={handleAddSource} disabled={isSubmittingSource}>
+                                {isSubmittingSource ? "Analyzing..." : "Add Source"}
+                            </Button>
+                        </div>
+                    </div>
+                </DialogContent>
+            </Dialog>
         </Sidebar>
     )
 }

convex/analyses.ts

@@ -19,6 +19,28 @@ export const getLatestByProject = query({
   },
 });
 
+export const getLatestByDataSource = query({
+  args: { dataSourceId: v.id("dataSources") },
+  handler: async (ctx, { dataSourceId }) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return null;
+
+    const dataSource = await ctx.db.get(dataSourceId);
+    if (!dataSource) return null;
+
+    const project = await ctx.db.get(dataSource.projectId);
+    if (!project || project.userId !== userId) return null;
+
+    return await ctx.db
+      .query("analyses")
+      .withIndex("by_dataSource_createdAt", (q) =>
+        q.eq("dataSourceId", dataSourceId)
+      )
+      .order("desc")
+      .first();
+  },
+});
+
 export const createAnalysis = mutation({
   args: {
     projectId: v.id("projects"),

convex/dataSources.ts

@@ -52,24 +52,68 @@ export const addDataSource = mutation({
             }
         }
 
-        const sourceId = await ctx.db.insert("dataSources", {
+        const existing = await ctx.db
-            projectId: projectId!, // Assert exists
+            .query("dataSources")
-            type: args.type,
+            .withIndex("by_project_url", (q) =>
-            url: args.url,
+                q.eq("projectId", projectId!).eq("url", args.url)
-            name: args.name,
+            )
-            analysisStatus: "pending",
+            .first();
-            // analysisResults not set initially
+
-        });
+        const sourceId = existing
+            ? existing._id
+            : await ctx.db.insert("dataSources", {
+                  projectId: projectId!, // Assert exists
+                  type: args.type,
+                  url: args.url,
+                  name: args.name,
+                  analysisStatus: "pending",
+                  lastAnalyzedAt: undefined,
+                  lastError: undefined,
+                  // analysisResults not set initially
+              });
 
         // Auto-select this source in the project config
         const project = await ctx.db.get(projectId!);
         if (project) {
             const currentSelected = project.dorkingConfig.selectedSourceIds;
-            await ctx.db.patch(projectId!, {
+            if (!currentSelected.includes(sourceId)) {
-                dorkingConfig: { selectedSourceIds: [...currentSelected, sourceId] }
+                await ctx.db.patch(projectId!, {
-            });
+                    dorkingConfig: { selectedSourceIds: [...currentSelected, sourceId] }
+                });
+            }
         }
 
         return { sourceId, projectId: projectId! };
     },
 });
+
+export const updateDataSourceStatus = mutation({
+    args: {
+        dataSourceId: v.id("dataSources"),
+        analysisStatus: v.union(
+            v.literal("pending"),
+            v.literal("completed"),
+            v.literal("failed")
+        ),
+        lastError: v.optional(v.string()),
+        lastAnalyzedAt: v.optional(v.number()),
+    },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) throw new Error("Unauthorized");
+
+        const dataSource = await ctx.db.get(args.dataSourceId);
+        if (!dataSource) throw new Error("Data source not found");
+
+        const project = await ctx.db.get(dataSource.projectId);
+        if (!project || project.userId !== userId) {
+            throw new Error("Project not found or unauthorized");
+        }
+
+        await ctx.db.patch(args.dataSourceId, {
+            analysisStatus: args.analysisStatus,
+            lastError: args.lastError,
+            lastAnalyzedAt: args.lastAnalyzedAt,
+        });
+    },
+});

convex/projects.ts

@@ -67,3 +67,162 @@ export const toggleDataSourceConfig = mutation({
         });
     },
 });
+
+export const getSearchContext = query({
+    args: { projectId: v.id("projects") },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) return { context: null, selectedSourceIds: [], missingSources: [] };
+
+        const project = await ctx.db.get(args.projectId);
+        if (!project || project.userId !== userId) {
+            return { context: null, selectedSourceIds: [], missingSources: [] };
+        }
+
+        const selectedSourceIds = project.dorkingConfig.selectedSourceIds || [];
+        if (selectedSourceIds.length === 0) {
+            return { context: null, selectedSourceIds, missingSources: [] };
+        }
+
+        const analyses = [];
+        const missingSources: { sourceId: string; reason: string }[] = [];
+
+        for (const sourceId of selectedSourceIds) {
+            const dataSource = await ctx.db.get(sourceId);
+            if (!dataSource || dataSource.projectId !== project._id) {
+                missingSources.push({ sourceId: sourceId as string, reason: "not_found" });
+                continue;
+            }
+
+            const latest = await ctx.db
+                .query("analyses")
+                .withIndex("by_dataSource_createdAt", (q) => q.eq("dataSourceId", sourceId))
+                .order("desc")
+                .first();
+
+            if (!latest) {
+                missingSources.push({ sourceId: sourceId as string, reason: "no_analysis" });
+                continue;
+            }
+
+            analyses.push(latest);
+        }
+
+        if (analyses.length === 0) {
+            return { context: null, selectedSourceIds, missingSources };
+        }
+
+        const merged = mergeAnalyses(analyses, project.name);
+
+        return { context: merged, selectedSourceIds, missingSources };
+    },
+});
+
+function normalizeKey(value: string) {
+    return value.trim().toLowerCase();
+}
+
+function severityRank(severity: "high" | "medium" | "low") {
+    if (severity === "high") return 3;
+    if (severity === "medium") return 2;
+    return 1;
+}
+
+function mergeAnalyses(analyses: any[], fallbackName: string) {
+    const keywordMap = new Map<string, any>();
+    const problemMap = new Map<string, any>();
+    const competitorMap = new Map<string, any>();
+    const personaMap = new Map<string, any>();
+    const useCaseMap = new Map<string, any>();
+    const featureMap = new Map<string, any>();
+
+    let latestScrapedAt = analyses[0].scrapedAt;
+
+    for (const analysis of analyses) {
+        if (analysis.scrapedAt > latestScrapedAt) {
+            latestScrapedAt = analysis.scrapedAt;
+        }
+
+        for (const keyword of analysis.keywords || []) {
+            const key = normalizeKey(keyword.term);
+            if (!keywordMap.has(key)) {
+                keywordMap.set(key, keyword);
+            } else if (keywordMap.get(key)?.type !== "differentiator" && keyword.type === "differentiator") {
+                keywordMap.set(key, keyword);
+            }
+        }
+
+        for (const problem of analysis.problemsSolved || []) {
+            const key = normalizeKey(problem.problem);
+            const existing = problemMap.get(key);
+            if (!existing || severityRank(problem.severity) > severityRank(existing.severity)) {
+                problemMap.set(key, problem);
+            }
+        }
+
+        for (const competitor of analysis.competitors || []) {
+            const key = normalizeKey(competitor.name);
+            if (!competitorMap.has(key)) {
+                competitorMap.set(key, competitor);
+            }
+        }
+
+        for (const persona of analysis.personas || []) {
+            const key = normalizeKey(`${persona.name}:${persona.role}`);
+            if (!personaMap.has(key)) {
+                personaMap.set(key, persona);
+            }
+        }
+
+        for (const useCase of analysis.useCases || []) {
+            const key = normalizeKey(useCase.scenario);
+            if (!useCaseMap.has(key)) {
+                useCaseMap.set(key, useCase);
+            }
+        }
+
+        for (const feature of analysis.features || []) {
+            const key = normalizeKey(feature.name);
+            if (!featureMap.has(key)) {
+                featureMap.set(key, feature);
+            }
+        }
+    }
+
+    const keywords = Array.from(keywordMap.values())
+        .sort((a, b) => {
+            const aDiff = a.type === "differentiator" ? 0 : 1;
+            const bDiff = b.type === "differentiator" ? 0 : 1;
+            if (aDiff !== bDiff) return aDiff - bDiff;
+            return a.term.length - b.term.length;
+        })
+        .slice(0, 80);
+
+    const problemsSolved = Array.from(problemMap.values())
+        .sort((a, b) => severityRank(b.severity) - severityRank(a.severity))
+        .slice(0, 15);
+
+    const competitors = Array.from(competitorMap.values()).slice(0, 10);
+    const personas = Array.from(personaMap.values()).slice(0, 6);
+    const useCases = Array.from(useCaseMap.values()).slice(0, 10);
+    const features = Array.from(featureMap.values()).slice(0, 20);
+
+    const base = analyses[0];
+
+    return {
+        productName: base.productName || fallbackName,
+        tagline: base.tagline || "",
+        description: base.description || "",
+        category: base.category || "",
+        positioning: base.positioning || "",
+        features,
+        problemsSolved,
+        personas,
+        keywords,
+        useCases,
+        competitors,
+        dorkQueries: [],
+        scrapedAt: latestScrapedAt,
+        analysisVersion: "aggregated",
+    };
+}

convex/schema.ts

@@ -22,6 +22,8 @@ const schema = defineSchema({
             v.literal("completed"),
             v.literal("failed")
         ),
+        lastAnalyzedAt: v.optional(v.number()),
+        lastError: v.optional(v.string()),
         analysisResults: v.optional(
             v.object({
                 features: v.array(v.string()),
@@ -31,7 +33,7 @@ const schema = defineSchema({
             })
         ),
         metadata: v.optional(v.any()),
-    }),
+    }).index("by_project_url", ["projectId", "url"]),
     analyses: defineTable({
         projectId: v.id("projects"),
         dataSourceId: v.id("dataSources"),
@@ -121,7 +123,8 @@ const schema = defineSchema({
         })),
         scrapedAt: v.string(),
     })
-        .index("by_project_createdAt", ["projectId", "createdAt"]),
+        .index("by_project_createdAt", ["projectId", "createdAt"])
+        .index("by_dataSource_createdAt", ["dataSourceId", "createdAt"]),
     opportunities: defineTable({
         projectId: v.id("projects"),
         analysisId: v.optional(v.id("analyses")),