feat: Implement analysis job tracking with progress timeline and enhanced data source status management.

.env.example

@@ -1,5 +1,5 @@
 # Required: OpenAI API key
-OPENAI_API_KEY=sk-...
+OPENAI_API_KEY=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
 
 # Optional: Serper.dev API key for reliable Google search
-SERPER_API_KEY=...
+SERPER_API_KEY=f9ae0a793cbac4116edd6482e377330e75c4db22

app/(app)/dashboard/page.tsx

@@ -9,6 +9,7 @@ import { Badge } from "@/components/ui/badge"
 import { Alert, AlertDescription } from "@/components/ui/alert"
 import { Button } from "@/components/ui/button"
 import { useMutation } from "convex/react"
+import { Progress } from "@/components/ui/progress"
 
 export default function Page() {
     const { selectedProjectId } = useProject()
@@ -21,8 +22,13 @@ export default function Page() {
         api.projects.getSearchContext,
         selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
     )
+    const analysisJobs = useQuery(
+        api.analysisJobs.listByProject,
+        selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+    )
     const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
     const createAnalysis = useMutation(api.analyses.createAnalysis)
+    const createAnalysisJob = useMutation(api.analysisJobs.create)
     const [reanalyzingId, setReanalyzingId] = useState<string | null>(null)
     const analysis = useQuery(
         api.analyses.getLatestByProject,
@@ -75,10 +81,15 @@ export default function Page() {
         })
 
         try {
+            const jobId = await createAnalysisJob({
+                projectId: selectedProjectId as any,
+                dataSourceId: source._id,
+            })
+
             const response = await fetch("/api/analyze", {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ url: source.url }),
+                body: JSON.stringify({ url: source.url, jobId }),
             })
 
             const data = await response.json()
@@ -114,10 +125,10 @@ export default function Page() {
         <div className="flex flex-1 flex-col gap-6 p-4 lg:p-8">
             <div className="space-y-2">
                 <div className="flex items-center gap-2">
-                    <h1 className="text-2xl font-semibold">{analysis.productName}</h1>
+                    <h1 className="text-2xl font-semibold">
-                    {selectedProject?.name && (
+                        {selectedProject?.name || analysis.productName}
-                        <Badge variant="outline">{selectedProject.name}</Badge>
+                    </h1>
-                    )}
+                    <Badge variant="outline">{analysis.productName}</Badge>
                 </div>
                 <p className="text-muted-foreground">{analysis.tagline}</p>
                 <p className="max-w-3xl text-sm text-muted-foreground">{analysis.description}</p>
@@ -126,7 +137,15 @@ export default function Page() {
             {searchContext?.missingSources?.length > 0 && (
                 <Alert>
                     <AlertDescription>
-                        Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+                        Some selected sources don&apos;t have analysis yet:{" "}
+                        {searchContext.missingSources
+                            .map((missing: any) =>
+                                dataSources?.find((source: any) => source._id === missing.sourceId)?.name ||
+                                dataSources?.find((source: any) => source._id === missing.sourceId)?.url ||
+                                missing.sourceId
+                            )
+                            .join(", ")}
+                        . Run onboarding or re-analyze them for best results.
                     </AlertDescription>
                 </Alert>
             )}
@@ -197,6 +216,56 @@ export default function Page() {
                 </CardContent>
             </Card>
 
+            {analysisJobs && analysisJobs.length > 0 && (
+                <Card>
+                    <CardHeader>
+                        <CardTitle className="text-base">Analysis Jobs</CardTitle>
+                    </CardHeader>
+                    <CardContent className="space-y-2 text-sm text-muted-foreground">
+                        {analysisJobs.slice(0, 5).map((job: any) => {
+                            const sourceName = dataSources?.find((source: any) => source._id === job.dataSourceId)?.name
+                                || dataSources?.find((source: any) => source._id === job.dataSourceId)?.url
+                                || "Unknown source"
+                            return (
+                                <div key={job._id} className="space-y-2 rounded-md border border-border p-3">
+                                    <div className="flex items-center justify-between gap-3">
+                                        <div className="truncate">
+                                            <span className="font-medium text-foreground">{sourceName}</span>{" "}
+                                            <span className="text-muted-foreground">
+                                                ({job.status})
+                                            </span>
+                                        </div>
+                                        {job.status === "failed" && (
+                                            <Button
+                                                size="sm"
+                                                variant="outline"
+                                                onClick={() => {
+                                                    const source = dataSources?.find((s: any) => s._id === job.dataSourceId)
+                                                    if (source) void handleReanalyze(source)
+                                                }}
+                                            >
+                                                Retry
+                                            </Button>
+                                        )}
+                                    </div>
+                                    {(job.status === "running" || job.status === "pending") && (
+                                        <div className="space-y-1">
+                                            <Progress value={typeof job.progress === "number" ? job.progress : 10} />
+                                            <div className="text-xs text-muted-foreground">
+                                                {typeof job.progress === "number" ? `${job.progress}% complete` : "Starting..."}
+                                            </div>
+                                        </div>
+                                    )}
+                                    {job.status === "failed" && job.error && (
+                                        <div className="text-xs text-destructive">{job.error}</div>
+                                    )}
+                                </div>
+                            )
+                        })}
+                    </CardContent>
+                </Card>
+            )}
+
             {searchContext?.context && (
                 <Card>
                     <CardHeader>

app/(app)/data-sources/[id]/page.tsx

@@ -0,0 +1,264 @@
+"use client"
+
+import { useParams, useRouter } from "next/navigation"
+import { useMutation, useQuery } from "convex/react"
+import { api } from "@/convex/_generated/api"
+import { Badge } from "@/components/ui/badge"
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
+import { Button } from "@/components/ui/button"
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog"
+import { Settings } from "lucide-react"
+import * as React from "react"
+
+function formatDate(timestamp?: number) {
+  if (!timestamp) return "Not analyzed yet";
+  return new Date(timestamp).toLocaleString();
+}
+
+export default function DataSourceDetailPage() {
+  const params = useParams<{ id: string }>()
+  const router = useRouter()
+  const dataSourceId = params?.id
+  const dataSource = useQuery(
+    api.dataSources.getById,
+    dataSourceId ? { dataSourceId: dataSourceId as any } : "skip"
+  )
+  const analysis = useQuery(
+    api.analyses.getLatestByDataSource,
+    dataSourceId ? { dataSourceId: dataSourceId as any } : "skip"
+  )
+  const removeDataSource = useMutation(api.dataSources.remove)
+  const [isDeleting, setIsDeleting] = React.useState(false)
+  const [isDialogOpen, setIsDialogOpen] = React.useState(false)
+
+  if (dataSource === undefined) {
+    return (
+      <div className="p-8">
+        <div className="text-sm text-muted-foreground">Loading data source…</div>
+      </div>
+    )
+  }
+
+  if (!dataSource) {
+    return (
+      <div className="p-8">
+        <h1 className="text-2xl font-semibold">Data source not found</h1>
+        <p className="mt-2 text-sm text-muted-foreground">
+          This data source may have been removed or you no longer have access.
+        </p>
+      </div>
+    )
+  }
+
+  const statusVariant =
+    dataSource.analysisStatus === "completed"
+      ? "secondary"
+      : dataSource.analysisStatus === "failed"
+      ? "destructive"
+      : "outline"
+
+  return (
+    <div className="flex flex-1 flex-col gap-6 p-8">
+      <div className="flex flex-col gap-2">
+        <div className="flex flex-wrap items-start justify-between gap-3">
+          <h1 className="text-3xl font-semibold">
+            {dataSource.name || "Data Source"}
+          </h1>
+          <div className="flex items-center gap-2">
+            <Badge variant={statusVariant}>
+              {dataSource.analysisStatus}
+            </Badge>
+            <Button
+              variant="ghost"
+              size="icon"
+              onClick={() => setIsDialogOpen(true)}
+              aria-label="Data source settings"
+            >
+              <Settings className="size-4" />
+            </Button>
+          </div>
+        </div>
+        <p className="text-sm text-muted-foreground">{dataSource.url}</p>
+        <div className="flex flex-wrap gap-3 text-xs text-muted-foreground">
+          <span>Last analyzed: {formatDate(dataSource.lastAnalyzedAt)}</span>
+          {dataSource.lastError && (
+            <span className="text-destructive">
+              Error: {dataSource.lastError}
+            </span>
+          )}
+        </div>
+      </div>
+
+      {analysis ? (
+        <>
+          <div className="grid gap-4 md:grid-cols-3">
+            <Card>
+              <CardHeader className="pb-2">
+                <CardTitle className="text-sm">Features</CardTitle>
+              </CardHeader>
+              <CardContent className="text-2xl font-semibold">
+                {analysis.features.length}
+              </CardContent>
+            </Card>
+            <Card>
+              <CardHeader className="pb-2">
+                <CardTitle className="text-sm">Keywords</CardTitle>
+              </CardHeader>
+              <CardContent className="text-2xl font-semibold">
+                {analysis.keywords.length}
+              </CardContent>
+            </Card>
+            <Card>
+              <CardHeader className="pb-2">
+                <CardTitle className="text-sm">Personas</CardTitle>
+              </CardHeader>
+              <CardContent className="text-2xl font-semibold">
+                {analysis.personas.length}
+              </CardContent>
+            </Card>
+          </div>
+
+          <Card>
+            <CardHeader>
+              <CardTitle className="text-base">Overview</CardTitle>
+            </CardHeader>
+            <CardContent className="space-y-2 text-sm text-muted-foreground">
+              <div>
+                <span className="font-medium text-foreground">Product:</span>{" "}
+                {analysis.productName}
+              </div>
+              <div>
+                <span className="font-medium text-foreground">Tagline:</span>{" "}
+                {analysis.tagline}
+              </div>
+              <div>
+                <span className="font-medium text-foreground">Category:</span>{" "}
+                {analysis.category}
+              </div>
+              <div>
+                <span className="font-medium text-foreground">Positioning:</span>{" "}
+                {analysis.positioning}
+              </div>
+              <div className="pt-2">{analysis.description}</div>
+            </CardContent>
+          </Card>
+
+          <div className="grid gap-4 lg:grid-cols-2">
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-base">Top Features</CardTitle>
+              </CardHeader>
+              <CardContent className="space-y-3 text-sm">
+                {analysis.features.slice(0, 6).map((feature) => (
+                  <div key={feature.name}>
+                    <div className="font-medium">{feature.name}</div>
+                    <div className="text-muted-foreground">
+                      {feature.description}
+                    </div>
+                  </div>
+                ))}
+              </CardContent>
+            </Card>
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-base">Pain Points</CardTitle>
+              </CardHeader>
+              <CardContent className="space-y-3 text-sm">
+                {analysis.problemsSolved.slice(0, 6).map((problem) => (
+                  <div key={problem.problem}>
+                    <div className="font-medium">{problem.problem}</div>
+                    <div className="text-muted-foreground">
+                      Severity: {problem.severity} · {problem.emotionalImpact}
+                    </div>
+                  </div>
+                ))}
+              </CardContent>
+            </Card>
+          </div>
+
+          <div className="grid gap-4 lg:grid-cols-2">
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-base">Personas</CardTitle>
+              </CardHeader>
+              <CardContent className="space-y-3 text-sm">
+                {analysis.personas.slice(0, 4).map((persona) => (
+                  <div key={`${persona.name}-${persona.role}`}>
+                    <div className="font-medium">
+                      {persona.name} · {persona.role}
+                    </div>
+                    <div className="text-muted-foreground">
+                      {persona.industry} · {persona.companySize}
+                    </div>
+                  </div>
+                ))}
+              </CardContent>
+            </Card>
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-base">Keywords</CardTitle>
+              </CardHeader>
+              <CardContent className="flex flex-wrap gap-2">
+                {analysis.keywords.slice(0, 12).map((keyword) => (
+                  <Badge key={keyword.term} variant="outline">
+                    {keyword.term}
+                  </Badge>
+                ))}
+              </CardContent>
+            </Card>
+          </div>
+        </>
+      ) : (
+        <Card>
+          <CardHeader>
+            <CardTitle className="text-base">Analysis</CardTitle>
+          </CardHeader>
+          <CardContent className="text-sm text-muted-foreground">
+            No analysis available yet. Trigger a new analysis to populate this
+            data source.
+          </CardContent>
+        </Card>
+      )}
+
+      <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen}>
+        <DialogContent>
+          <DialogHeader>
+            <DialogTitle>Delete data source</DialogTitle>
+            <DialogDescription>
+              This removes the data source and its analyses from the project. This
+              cannot be undone.
+            </DialogDescription>
+          </DialogHeader>
+          <DialogFooter>
+            <Button
+              variant="outline"
+              onClick={() => setIsDialogOpen(false)}
+              disabled={isDeleting}
+            >
+              Cancel
+            </Button>
+            <Button
+              variant="destructive"
+              onClick={async () => {
+                if (!dataSourceId) return
+                setIsDeleting(true)
+                await removeDataSource({ dataSourceId: dataSourceId as any })
+                router.push("/dashboard")
+              }}
+              disabled={isDeleting}
+            >
+              Delete
+            </Button>
+          </DialogFooter>
+        </DialogContent>
+      </Dialog>
+    </div>
+  )
+}

app/(app)/opportunities/page.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useEffect, useMemo, useState } from 'react'
+import { useEffect, useMemo, useRef, useState } from 'react'
 import { useRouter } from 'next/navigation'
 import { useMutation, useQuery } from 'convex/react'
 import { api } from '@/convex/_generated/api'
@@ -10,6 +10,7 @@ import { Badge } from '@/components/ui/badge'
 import { Checkbox } from '@/components/ui/checkbox'
 import { Slider } from '@/components/ui/slider'
 import { Alert, AlertDescription } from '@/components/ui/alert'
+import { Progress } from '@/components/ui/progress'
 import { Separator } from '@/components/ui/separator'
 import { ScrollArea } from '@/components/ui/scroll-area'
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
@@ -37,6 +38,7 @@ import {
   ExternalLink, 
   MessageSquare, 
   Twitter,
+  Globe,
   Users,
   HelpCircle,
   Filter,
@@ -55,8 +57,10 @@ import type {
   EnhancedProductAnalysis, 
   Opportunity, 
   PlatformConfig, 
-  SearchStrategy 
+  SearchStrategy,
+  SearchConfig
 } from '@/lib/types'
+import { estimateSearchTime } from '@/lib/query-generator'
 
 const STRATEGY_INFO: Record<SearchStrategy, { name: string; description: string }> = {
   'direct-keywords': { name: 'Direct Keywords', description: 'People looking for your product category' },
@@ -68,11 +72,64 @@ const STRATEGY_INFO: Record<SearchStrategy, { name: string; description: string
   'recommendation': { name: 'Recommendations', description: '"What do you use" requests' }
 }
 
+const STRATEGY_GROUPS: { title: string; description: string; strategies: SearchStrategy[] }[] = [
+  {
+    title: "High intent",
+    description: "People actively looking or comparing options.",
+    strategies: ["direct-keywords", "competitor-alternative", "comparison", "recommendation"],
+  },
+  {
+    title: "Problem-driven",
+    description: "Pain and frustration expressed in public threads.",
+    strategies: ["problem-pain", "emotional-frustrated"],
+  },
+  {
+    title: "Learning intent",
+    description: "Educational and how-to discovery signals.",
+    strategies: ["how-to"],
+  },
+]
+
+const GOAL_PRESETS: {
+  id: string
+  title: string
+  description: string
+  strategies: SearchStrategy[]
+  intensity: 'broad' | 'balanced' | 'targeted'
+  maxQueries: number
+}[] = [
+  {
+    id: "high-intent",
+    title: "High-intent leads",
+    description: "Shortlist people actively searching to buy or switch.",
+    strategies: ["direct-keywords", "competitor-alternative", "comparison", "recommendation"],
+    intensity: "targeted",
+    maxQueries: 30,
+  },
+  {
+    id: "pain-first",
+    title: "Problem pain",
+    description: "Find people expressing frustration or blockers.",
+    strategies: ["problem-pain", "emotional-frustrated"],
+    intensity: "balanced",
+    maxQueries: 40,
+  },
+  {
+    id: "market-scan",
+    title: "Market scan",
+    description: "Broader sweep to map demand and platforms.",
+    strategies: ["direct-keywords", "problem-pain", "how-to", "recommendation"],
+    intensity: "broad",
+    maxQueries: 50,
+  },
+]
+
 export default function OpportunitiesPage() {
   const router = useRouter()
   const { selectedProjectId } = useProject()
   const upsertOpportunities = useMutation(api.opportunities.upsertBatch)
   const updateOpportunity = useMutation(api.opportunities.updateStatus)
+  const createSearchJob = useMutation(api.searchJobs.create)
   const [analysis, setAnalysis] = useState<EnhancedProductAnalysis | null>(null)
   const [platforms, setPlatforms] = useState<PlatformConfig[]>([])
   const [strategies, setStrategies] = useState<SearchStrategy[]>([
@@ -81,6 +138,8 @@ export default function OpportunitiesPage() {
     'competitor-alternative'
   ])
   const [intensity, setIntensity] = useState<'broad' | 'balanced' | 'targeted'>('balanced')
+  const [maxQueries, setMaxQueries] = useState(50)
+  const [goalPreset, setGoalPreset] = useState<string>('high-intent')
   const [isSearching, setIsSearching] = useState(false)
   const [opportunities, setOpportunities] = useState<Opportunity[]>([])
   const [generatedQueries, setGeneratedQueries] = useState<any[]>([])
@@ -90,6 +149,7 @@ export default function OpportunitiesPage() {
   const [stats, setStats] = useState<any>(null)
   const [searchError, setSearchError] = useState('')
   const [missingSources, setMissingSources] = useState<any[]>([])
+  const [lastSearchConfig, setLastSearchConfig] = useState<SearchConfig | null>(null)
   const [statusFilter, setStatusFilter] = useState('all')
   const [intentFilter, setIntentFilter] = useState('all')
   const [minScore, setMinScore] = useState(0)
@@ -97,6 +157,8 @@ export default function OpportunitiesPage() {
   const [statusInput, setStatusInput] = useState('new')
   const [notesInput, setNotesInput] = useState('')
   const [tagsInput, setTagsInput] = useState('')
+  const planLoadedRef = useRef<string | null>(null)
+  const defaultPlatformsRef = useRef<PlatformConfig[] | null>(null)
 
   const projects = useQuery(api.projects.getProjects)
   const latestAnalysis = useQuery(
@@ -143,12 +205,18 @@ export default function OpportunitiesPage() {
     api.dataSources.getProjectDataSources,
     selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
   )
+  const searchJobs = useQuery(
+    api.searchJobs.listByProject,
+    selectedProjectId ? { projectId: selectedProjectId as any } : "skip"
+  )
 
   const selectedProject = projects?.find((project: any) => project._id === selectedProjectId)
   const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || []
   const activeSources = selectedSources?.filter((source: any) =>
     selectedSourceIds.includes(source._id)
   ) || []
+  const enabledPlatforms = platforms.filter((platform) => platform.enabled)
+  const estimatedMinutes = estimateSearchTime(Math.max(maxQueries, 1), enabledPlatforms.map((platform) => platform.id))
 
   useEffect(() => {
     const stored = localStorage.getItem('productAnalysis')
@@ -167,16 +235,76 @@ export default function OpportunitiesPage() {
       .then(data => {
         if (data) {
           setPlatforms(data.platforms)
+          defaultPlatformsRef.current = data.platforms
         }
       })
   }, [router])
 
+  useEffect(() => {
+    if (!selectedProjectId) return
+    if (platforms.length === 0) return
+    if (planLoadedRef.current === selectedProjectId) return
+
+    const key = `searchPlan:${selectedProjectId}`
+    const stored = localStorage.getItem(key)
+    if (stored) {
+      try {
+        const parsed = JSON.parse(stored)
+        if (Array.isArray(parsed.strategies)) {
+          setStrategies(parsed.strategies)
+        }
+        if (parsed.intensity === 'broad' || parsed.intensity === 'balanced' || parsed.intensity === 'targeted') {
+          setIntensity(parsed.intensity)
+        }
+        if (typeof parsed.maxQueries === 'number') {
+          setMaxQueries(Math.min(Math.max(parsed.maxQueries, 10), 50))
+        }
+        if (typeof parsed.goalPreset === 'string') {
+          setGoalPreset(parsed.goalPreset)
+        }
+        if (Array.isArray(parsed.platformIds)) {
+          setPlatforms((prev) =>
+            prev.map((platform) => ({
+              ...platform,
+              enabled: parsed.platformIds.includes(platform.id),
+            }))
+          )
+        }
+      } catch {
+        // Ignore invalid cached config.
+      }
+    } else if (defaultPlatformsRef.current) {
+      setStrategies(['direct-keywords', 'problem-pain', 'competitor-alternative'])
+      setIntensity('balanced')
+      setMaxQueries(50)
+      setGoalPreset('high-intent')
+      setPlatforms(defaultPlatformsRef.current)
+    }
+
+    planLoadedRef.current = selectedProjectId
+  }, [selectedProjectId, platforms])
+
   useEffect(() => {
     if (!analysis && latestAnalysis) {
       setAnalysis(latestAnalysis as any)
     }
   }, [analysis, latestAnalysis])
 
+  useEffect(() => {
+    if (!selectedProjectId) return
+    if (planLoadedRef.current !== selectedProjectId) return
+
+    const key = `searchPlan:${selectedProjectId}`
+    const payload = {
+      goalPreset,
+      strategies,
+      intensity,
+      maxQueries,
+      platformIds: platforms.filter((platform) => platform.enabled).map((platform) => platform.id),
+    }
+    localStorage.setItem(key, JSON.stringify(payload))
+  }, [selectedProjectId, goalPreset, strategies, intensity, maxQueries, platforms])
+
   useEffect(() => {
     if (!analysis && latestAnalysis === null) {
       router.push('/onboarding')
@@ -197,7 +325,16 @@ export default function OpportunitiesPage() {
     )
   }
 
-  const executeSearch = async () => {
+  const applyGoalPreset = (presetId: string) => {
+    const preset = GOAL_PRESETS.find((item) => item.id === presetId)
+    if (!preset) return
+    setGoalPreset(preset.id)
+    setStrategies(preset.strategies)
+    setIntensity(preset.intensity)
+    setMaxQueries(preset.maxQueries)
+  }
+
+  const executeSearch = async (overrideConfig?: SearchConfig) => {
     if (!analysis) return
     if (!selectedProjectId) return
 
@@ -206,17 +343,27 @@ export default function OpportunitiesPage() {
     setSearchError('')
     
     try {
-      const config = {
+      const config = overrideConfig ?? {
-        platforms,
+        platforms: platforms.map((platform) => ({
+          ...platform,
+          icon: platform.icon ?? "",
+          searchTemplate: platform.searchTemplate ?? "",
+        })),
         strategies,
         intensity,
-        maxResults: intensity === 'broad' ? 80 : intensity === 'balanced' ? 50 : 30
+        maxResults: Math.min(maxQueries, 50)
       }
+      setLastSearchConfig(config as SearchConfig)
+
+      const jobId = await createSearchJob({
+        projectId: selectedProjectId as any,
+        config,
+      })
 
       const response = await fetch('/api/opportunities', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ projectId: selectedProjectId, config })
+        body: JSON.stringify({ projectId: selectedProjectId, config, jobId })
       })
 
       if (response.redirected) {
@@ -224,13 +371,14 @@ export default function OpportunitiesPage() {
         return
       }
 
-      const data = await response.json()
+      const responseText = await response.text()
+      const data = responseText ? JSON.parse(responseText) : null
 
       if (!response.ok) {
-        throw new Error(data.error || 'Failed to search for opportunities')
+        throw new Error(data?.error || 'Failed to search for opportunities')
       }
 
-      if (data.success) {
+      if (data?.success) {
         const mapped = data.data.opportunities.map((opp: Opportunity) => ({
           ...opp,
           status: 'new',
@@ -256,6 +404,8 @@ export default function OpportunitiesPage() {
             softPitch: opp.softPitch,
           })),
         })
+      } else {
+        throw new Error('Search returned no data')
       }
     } catch (error: any) {
       console.error('Search error:', error)
@@ -307,80 +457,162 @@ export default function OpportunitiesPage() {
 
   if (!analysis) return null
 
+  const latestJob = searchJobs && searchJobs.length > 0 ? searchJobs[0] : null
+
   return (
     <div className="flex h-screen">
       {/* Sidebar */}
-      <div className="w-80 border-r border-border bg-card flex flex-col">
+      <div className="w-96 border-r border-border bg-card flex flex-col">
-        <div className="p-4 border-b border-border">
+        <div className="p-4 border-b border-border space-y-1">
           <h2 className="font-semibold flex items-center gap-2">
             <Target className="h-5 w-5" />
-            Search Configuration
+            Search Plan
           </h2>
+          <p className="text-xs text-muted-foreground">
+            Pick a goal, tune channels, then run the scan.
+          </p>
         </div>
 
         <ScrollArea className="flex-1 p-4 space-y-6">
+          {/* Goal presets */}
+          <div className="space-y-3">
+            <Label className="text-sm font-medium uppercase text-muted-foreground">Goal</Label>
+            <div className="grid gap-2">
+              {GOAL_PRESETS.map((preset) => (
+                <button
+                  key={preset.id}
+                  type="button"
+                  onClick={() => applyGoalPreset(preset.id)}
+                  className={`rounded-lg border px-3 py-2 text-left transition ${
+                    goalPreset === preset.id
+                      ? "border-foreground/60 bg-muted/50"
+                      : "border-border/60 hover:border-foreground/30 hover:bg-muted/40"
+                  }`}
+                >
+                  <div className="text-sm font-semibold">{preset.title}</div>
+                  <div className="text-xs text-muted-foreground">{preset.description}</div>
+                </button>
+              ))}
+            </div>
+          </div>
+
+          <Separator />
+
           {/* Platforms */}
           <div className="space-y-3">
-            <Label className="text-sm font-medium uppercase text-muted-foreground">Platforms</Label>
+            <Label className="text-sm font-medium uppercase text-muted-foreground">Channels</Label>
-            <div className="space-y-2">
+            <div className="grid gap-2">
-              {platforms.map(platform => (
+              {platforms.map(platform => {
-                <div key={platform.id} className="flex items-center space-x-2">
+                const isEnabled = platform.enabled
-                  <Checkbox
+                const iconMap: Record<string, JSX.Element> = {
-                    id={platform.id}
+                  reddit: <MessageSquare className="h-4 w-4" />,
-                    checked={platform.enabled}
+                  twitter: <Twitter className="h-4 w-4" />,
-                    onCheckedChange={() => togglePlatform(platform.id)}
+                  hackernews: <Zap className="h-4 w-4" />,
-                  />
+                  indiehackers: <Users className="h-4 w-4" />,
-                  <Label htmlFor={platform.id} className="cursor-pointer flex-1">{platform.name}</Label>
+                  quora: <HelpCircle className="h-4 w-4" />,
-                </div>
+                  stackoverflow: <Filter className="h-4 w-4" />,
-              ))}
+                  linkedin: <Globe className="h-4 w-4" />
+                }
+                return (
+                  <button
+                    key={platform.id}
+                    type="button"
+                    onClick={() => togglePlatform(platform.id)}
+                    className={`flex items-center justify-between rounded-lg border px-3 py-2 text-left transition ${
+                      isEnabled
+                        ? "border-foreground/50 bg-muted/50"
+                        : "border-border/60 hover:border-foreground/30 hover:bg-muted/40"
+                    }`}
+                  >
+                    <div className="flex items-center gap-2">
+                      <div className="flex h-7 w-7 items-center justify-center rounded-md bg-background">
+                        {iconMap[platform.id] || <Globe className="h-4 w-4" />}
+                      </div>
+                      <div>
+                        <div className="text-sm font-medium">{platform.name}</div>
+                        <div className="text-xs text-muted-foreground">
+                          {isEnabled ? "Included" : "Excluded"}
+                        </div>
+                      </div>
+                    </div>
+                    <Checkbox checked={isEnabled} />
+                  </button>
+                )
+              })}
             </div>
           </div>
 
           <Separator />
 
           {/* Strategies */}
-          <div className="space-y-3">
+          <div className="space-y-4">
-            <Label className="text-sm font-medium uppercase text-muted-foreground">Strategies</Label>
+            <Label className="text-sm font-medium uppercase text-muted-foreground">Signals</Label>
-            <div className="space-y-2">
+            {STRATEGY_GROUPS.map((group) => (
-              {(Object.keys(STRATEGY_INFO) as SearchStrategy[]).map(strategy => (
+              <div key={group.title} className="space-y-2">
-                <div key={strategy} className="flex items-start space-x-2">
+                <div>
-                  <Checkbox
+                  <div className="text-sm font-semibold">{group.title}</div>
-                    id={strategy}
+                  <div className="text-xs text-muted-foreground">{group.description}</div>
-                    checked={strategies.includes(strategy)}
-                    onCheckedChange={() => toggleStrategy(strategy)}
-                  />
-                  <div className="flex-1">
-                    <Label htmlFor={strategy} className="cursor-pointer">
-                      {STRATEGY_INFO[strategy].name}
-                    </Label>
-                    <p className="text-xs text-muted-foreground">{STRATEGY_INFO[strategy].description}</p>
-                  </div>
                 </div>
-              ))}
+                <div className="space-y-2">
-            </div>
+                  {group.strategies.map((strategy) => (
+                    <div key={strategy} className="flex items-start gap-2 rounded-md border border-border/60 px-2 py-2">
+                      <Checkbox
+                        id={strategy}
+                        checked={strategies.includes(strategy)}
+                        onCheckedChange={() => toggleStrategy(strategy)}
+                      />
+                      <div className="flex-1">
+                        <Label htmlFor={strategy} className="cursor-pointer text-sm font-medium">
+                          {STRATEGY_INFO[strategy].name}
+                        </Label>
+                        <p className="text-xs text-muted-foreground">{STRATEGY_INFO[strategy].description}</p>
+                      </div>
+                    </div>
+                  ))}
+                </div>
+              </div>
+            ))}
           </div>
 
           <Separator />
 
-          {/* Intensity */}
+          {/* Depth + Queries */}
-          <div className="space-y-3">
+          <div className="space-y-4">
-            <Label className="text-sm font-medium uppercase text-muted-foreground">Intensity</Label>
+            <Label className="text-sm font-medium uppercase text-muted-foreground">Depth</Label>
-            <Slider
+            <div className="space-y-3">
-              value={[intensity === 'broad' ? 0 : intensity === 'balanced' ? 50 : 100]}
+              <div className="flex items-center justify-between text-xs text-muted-foreground">
-              onValueChange={([v]) => setIntensity(v < 33 ? 'broad' : v < 66 ? 'balanced' : 'targeted')}
+                <span>Broad</span>
-              max={100}
+                <span>Targeted</span>
-              step={50}
+              </div>
-            />
+              <Slider
-            <div className="flex justify-between text-xs text-muted-foreground">
+                value={[intensity === 'broad' ? 0 : intensity === 'balanced' ? 50 : 100]}
-              <span>Broad</span>
+                onValueChange={([v]) => setIntensity(v < 33 ? 'broad' : v < 66 ? 'balanced' : 'targeted')}
-              <span>Targeted</span>
+                max={100}
+                step={50}
+              />
+            </div>
+            <div className="space-y-2">
+              <div className="flex items-center justify-between">
+                <Label className="text-xs font-medium uppercase text-muted-foreground">Max Queries</Label>
+                <span className="text-xs text-muted-foreground">{maxQueries}</span>
+              </div>
+              <Slider
+                value={[maxQueries]}
+                onValueChange={([v]) => setMaxQueries(v)}
+                min={10}
+                max={50}
+                step={5}
+              />
+              <div className="text-xs text-muted-foreground">
+                Up to {maxQueries} queries · est. {estimatedMinutes} min
+              </div>
             </div>
           </div>
         </ScrollArea>
 
-        <div className="p-4 border-t border-border">
+        <div className="p-4 border-t border-border space-y-2">
           <Button 
-            onClick={executeSearch} 
+            onClick={() => executeSearch()} 
             disabled={
               isSearching ||
               platforms.filter(p => p.enabled).length === 0 ||
@@ -390,6 +622,19 @@ export default function OpportunitiesPage() {
           >
             {isSearching ? <><Loader2 className="mr-2 h-4 w-4 animate-spin" /> Searching...</> : <><Search className="mr-2 h-4 w-4" /> Find Opportunities</>}
           </Button>
+          <div className="flex flex-wrap gap-2 text-xs text-muted-foreground">
+            <span>{enabledPlatforms.length || 0} channels</span>
+            <span>·</span>
+            <span>{strategies.length} signals</span>
+            <span>·</span>
+            <span>max {maxQueries} queries</span>
+          </div>
+          {platforms.filter(p => p.enabled).length === 0 && (
+            <p className="text-xs text-muted-foreground">Select at least one platform to search.</p>
+          )}
+          {selectedSourceIds.length === 0 && (
+            <p className="text-xs text-muted-foreground">Select data sources to build search context.</p>
+          )}
         </div>
       </div>
 
@@ -397,7 +642,7 @@ export default function OpportunitiesPage() {
       <div className="flex-1 overflow-auto p-6">
         <div className="max-w-4xl mx-auto space-y-6">
           {/* Header */}
-          <div className="flex items-center justify-between">
+          <div className="flex flex-wrap items-center justify-between gap-3">
             <div>
               <h1 className="text-2xl font-bold">Opportunity Finder</h1>
               <p className="text-muted-foreground">Discover potential customers for {analysis.productName}</p>
@@ -415,6 +660,42 @@ export default function OpportunitiesPage() {
               </div>
             )}
           </div>
+          <div className="flex flex-wrap gap-2 text-xs text-muted-foreground">
+            <Badge variant="outline">Goal: {GOAL_PRESETS.find((preset) => preset.id === goalPreset)?.title || "Custom"}</Badge>
+            <Badge variant="outline">Intensity: {intensity}</Badge>
+            <Badge variant="outline">Max queries: {maxQueries}</Badge>
+          </div>
+          {latestJob && (latestJob.status === "running" || latestJob.status === "pending") && (
+            <Card>
+              <CardHeader>
+                <CardTitle className="text-sm">Search in progress</CardTitle>
+                <CardDescription>
+                  Current job: {latestJob.status}
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="space-y-2">
+                <Progress value={typeof latestJob.progress === "number" ? latestJob.progress : 10} />
+                <div className="text-xs text-muted-foreground">
+                  {typeof latestJob.progress === "number" ? `${latestJob.progress}% complete` : "Starting..."}
+                </div>
+              </CardContent>
+            </Card>
+          )}
+          {latestJob && latestJob.status === "failed" && (
+            <Alert variant="destructive">
+              <AlertDescription>
+                Search failed: {latestJob.error || "Unknown error"}.{" "}
+                <Button
+                  variant="outline"
+                  size="sm"
+                  className="ml-2"
+                  onClick={() => executeSearch((latestJob.config as SearchConfig) || lastSearchConfig || undefined)}
+                >
+                  Retry
+                </Button>
+              </AlertDescription>
+            </Alert>
+          )}
           {searchError && (
             <Alert variant="destructive">
               <AlertDescription>{searchError}</AlertDescription>
@@ -451,7 +732,15 @@ export default function OpportunitiesPage() {
           {missingSources.length > 0 && (
             <Alert>
               <AlertDescription>
-                Some selected sources don&apos;t have analysis yet. Run onboarding or re-analyze them for best results.
+                Some selected sources don&apos;t have analysis yet:{" "}
+                {missingSources
+                  .map((missing) =>
+                    activeSources.find((source: any) => source._id === missing.sourceId)?.name ||
+                    activeSources.find((source: any) => source._id === missing.sourceId)?.url ||
+                    missing.sourceId
+                  )
+                  .join(", ")}
+                . Run onboarding or re-analyze them for best results.
               </AlertDescription>
             </Alert>
           )}

app/api/analyze-manual/route.ts

@@ -1,5 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server'
-import { isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { fetchMutation } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { analyzeFromText } from '@/lib/scraper'
 import { performDeepAnalysis } from '@/lib/analysis-pipeline'
@@ -7,10 +9,18 @@ import { performDeepAnalysis } from '@/lib/analysis-pipeline'
 const bodySchema = z.object({
   productName: z.string().min(1),
   description: z.string().min(1),
-  features: z.string()
+  features: z.string(),
+  jobId: z.optional(z.string())
 })
 
 export async function POST(request: NextRequest) {
+  let jobId: string | undefined
+  let timeline: {
+    key: string
+    label: string
+    status: "pending" | "running" | "completed" | "failed"
+    detail?: string
+  }[] = []
   try {
     if (!(await isAuthenticatedNextjs())) {
       const redirectUrl = new URL("/auth", request.url);
@@ -21,9 +31,69 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { productName, description, features } = bodySchema.parse(body)
+    const parsed = bodySchema.parse(body)
+    const { productName, description, features } = parsed
+    jobId = parsed.jobId
+
+    const token = await convexAuthNextjsToken();
+    timeline = [
+      { key: "scrape", label: "Prepare input", status: "pending" },
+      { key: "features", label: "Pass 1: Features", status: "pending" },
+      { key: "competitors", label: "Pass 2: Competitors", status: "pending" },
+      { key: "keywords", label: "Pass 3: Keywords", status: "pending" },
+      { key: "problems", label: "Pass 4: Problems & Personas", status: "pending" },
+      { key: "useCases", label: "Pass 5: Use cases", status: "pending" },
+      { key: "dorkQueries", label: "Pass 6: Dork queries", status: "pending" },
+      { key: "finalize", label: "Finalize analysis", status: "pending" },
+    ]
+    const updateTimeline = async ({
+      key,
+      status,
+      detail,
+      progress,
+      finalStatus,
+    }: {
+      key: string
+      status: "pending" | "running" | "completed" | "failed"
+      detail?: string
+      progress?: number
+      finalStatus?: "running" | "completed" | "failed"
+    }) => {
+      if (!jobId) return
+      timeline = timeline.map((item) =>
+        item.key === key ? { ...item, status, detail: detail ?? item.detail } : item
+      )
+      await fetchMutation(
+        api.analysisJobs.update,
+        {
+          jobId: jobId as any,
+          status: finalStatus || "running",
+          progress,
+          stage: key,
+          timeline,
+        },
+        { token }
+      )
+    }
+    if (jobId) {
+      await updateTimeline({ key: "scrape", status: "running", progress: 10 })
+    }
 
     if (!process.env.OPENAI_API_KEY) {
+      if (jobId) {
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: "OpenAI API key not configured",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      }
       return NextResponse.json(
         { error: 'OpenAI API key not configured' },
         { status: 500 }
@@ -32,10 +102,49 @@ export async function POST(request: NextRequest) {
 
     console.log('📝 Creating content from manual input...')
     const scrapedContent = await analyzeFromText(productName, description, features)
+    if (jobId) {
+      await updateTimeline({
+        key: "scrape",
+        status: "completed",
+        detail: "Manual input prepared",
+        progress: 20,
+      })
+    }
 
     console.log('🤖 Starting enhanced analysis...')
-    const analysis = await performDeepAnalysis(scrapedContent)
+    const progressMap: Record<string, number> = {
+      features: 35,
+      competitors: 50,
+      keywords: 65,
+      problems: 78,
+      useCases: 88,
+      dorkQueries: 95,
+    }
+    const analysis = await performDeepAnalysis(scrapedContent, async (update) => {
+      await updateTimeline({
+        key: update.key,
+        status: update.status,
+        detail: update.detail,
+        progress: progressMap[update.key] ?? 80,
+      })
+    })
     console.log(`   ✓ Analysis complete: ${analysis.features.length} features, ${analysis.keywords.length} keywords`)
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "running",
+        progress: 98,
+      })
+    }
+
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "completed",
+        progress: 100,
+        finalStatus: "completed",
+      })
+    }
 
     return NextResponse.json({
       success: true,
@@ -53,6 +162,26 @@ export async function POST(request: NextRequest) {
   } catch (error: any) {
     console.error('❌ Manual analysis error:', error)
 
+    if (jobId) {
+      try {
+        const token = await convexAuthNextjsToken();
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: error.message || "Manual analysis failed",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      } catch {
+        // Best-effort job update only.
+      }
+    }
+    
     if (error.name === 'ZodError') {
       return NextResponse.json(
         { error: 'Please provide product name and description' },

app/api/analyze/route.ts

@@ -1,14 +1,24 @@
 import { NextRequest, NextResponse } from 'next/server'
-import { isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
+import { fetchMutation } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { scrapeWebsite, ScrapingError } from '@/lib/scraper'
 import { performDeepAnalysis } from '@/lib/analysis-pipeline'
 
 const bodySchema = z.object({
-  url: z.string().min(1)
+  url: z.string().min(1),
+  jobId: z.optional(z.string())
 })
 
 export async function POST(request: NextRequest) {
+  let jobId: string | undefined
+  let timeline: {
+    key: string
+    label: string
+    status: "pending" | "running" | "completed" | "failed"
+    detail?: string
+  }[] = []
   try {
     if (!(await isAuthenticatedNextjs())) {
       const redirectUrl = new URL("/auth", request.url);
@@ -19,9 +29,70 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { url } = bodySchema.parse(body)
+    const parsed = bodySchema.parse(body)
+    const { url } = parsed
+    jobId = parsed.jobId
+
+    const token = await convexAuthNextjsToken();
+    timeline = [
+      { key: "scrape", label: "Scrape website", status: "pending" },
+      { key: "features", label: "Pass 1: Features", status: "pending" },
+      { key: "competitors", label: "Pass 2: Competitors", status: "pending" },
+      { key: "keywords", label: "Pass 3: Keywords", status: "pending" },
+      { key: "problems", label: "Pass 4: Problems & Personas", status: "pending" },
+      { key: "useCases", label: "Pass 5: Use cases", status: "pending" },
+      { key: "dorkQueries", label: "Pass 6: Dork queries", status: "pending" },
+      { key: "finalize", label: "Finalize analysis", status: "pending" },
+    ]
+
+    const updateTimeline = async ({
+      key,
+      status,
+      detail,
+      progress,
+      finalStatus,
+    }: {
+      key: string
+      status: "pending" | "running" | "completed" | "failed"
+      detail?: string
+      progress?: number
+      finalStatus?: "running" | "completed" | "failed"
+    }) => {
+      if (!jobId) return
+      timeline = timeline.map((item) =>
+        item.key === key ? { ...item, status, detail: detail ?? item.detail } : item
+      )
+      await fetchMutation(
+        api.analysisJobs.update,
+        {
+          jobId: jobId as any,
+          status: finalStatus || "running",
+          progress,
+          stage: key,
+          timeline,
+        },
+        { token }
+      )
+    }
+    if (jobId) {
+      await updateTimeline({ key: "scrape", status: "running", progress: 10 })
+    }
 
     if (!process.env.OPENAI_API_KEY) {
+      if (jobId) {
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: "OpenAI API key not configured",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      }
       return NextResponse.json(
         { error: 'OpenAI API key not configured' },
         { status: 500 }
@@ -31,10 +102,49 @@ export async function POST(request: NextRequest) {
     console.log(`🌐 Scraping: ${url}`)
     const scrapedContent = await scrapeWebsite(url)
     console.log(`   ✓ Scraped ${scrapedContent.headings.length} headings, ${scrapedContent.paragraphs.length} paragraphs`)
+    if (jobId) {
+      await updateTimeline({
+        key: "scrape",
+        status: "completed",
+        detail: `${scrapedContent.headings.length} headings, ${scrapedContent.paragraphs.length} paragraphs`,
+        progress: 20,
+      })
+    }
 
     console.log('🤖 Starting enhanced analysis...')
-    const analysis = await performDeepAnalysis(scrapedContent)
+    const progressMap: Record<string, number> = {
+      features: 35,
+      competitors: 50,
+      keywords: 65,
+      problems: 78,
+      useCases: 88,
+      dorkQueries: 95,
+    }
+    const analysis = await performDeepAnalysis(scrapedContent, async (update) => {
+      await updateTimeline({
+        key: update.key,
+        status: update.status,
+        detail: update.detail,
+        progress: progressMap[update.key] ?? 80,
+      })
+    })
     console.log(`   ✓ Analysis complete: ${analysis.features.length} features, ${analysis.keywords.length} keywords, ${analysis.dorkQueries.length} queries`)
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "running",
+        progress: 98,
+      })
+    }
+
+    if (jobId) {
+      await updateTimeline({
+        key: "finalize",
+        status: "completed",
+        progress: 100,
+        finalStatus: "completed",
+      })
+    }
 
     return NextResponse.json({
       success: true,
@@ -52,6 +162,26 @@ export async function POST(request: NextRequest) {
   } catch (error: any) {
     console.error('❌ Analysis error:', error)
 
+    if (jobId) {
+      try {
+        const token = await convexAuthNextjsToken();
+        await fetchMutation(
+          api.analysisJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: error.message || "Analysis failed",
+            timeline: timeline.map((item) =>
+              item.status === "running" ? { ...item, status: "failed" } : item
+            ),
+          },
+          { token }
+        );
+      } catch {
+        // Best-effort job update only.
+      }
+    }
+    
     if (error instanceof ScrapingError) {
       return NextResponse.json(
         { 

app/api/opportunities/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server";
-import { fetchQuery } from "convex/nextjs";
+import { fetchMutation, fetchQuery } from "convex/nextjs";
 import { api } from "@/convex/_generated/api";
 import { z } from 'zod'
 import { generateSearchQueries, getDefaultPlatforms } from '@/lib/query-generator'
@@ -9,13 +9,14 @@ import type { EnhancedProductAnalysis, SearchConfig, PlatformConfig } from '@/li
 
 const searchSchema = z.object({
   projectId: z.string(),
+  jobId: z.optional(z.string()),
   config: z.object({
     platforms: z.array(z.object({
       id: z.string(),
       name: z.string(),
-      icon: z.string(),
+      icon: z.string().optional(),
       enabled: z.boolean(),
-      searchTemplate: z.string(),
+      searchTemplate: z.string().optional(),
       rateLimit: z.number()
     })),
     strategies: z.array(z.string()),
@@ -25,6 +26,7 @@ const searchSchema = z.object({
 })
 
 export async function POST(request: NextRequest) {
+  let jobId: string | undefined
   try {
     if (!(await isAuthenticatedNextjs())) {
       const redirectUrl = new URL("/auth", request.url);
@@ -35,9 +37,18 @@ export async function POST(request: NextRequest) {
     }
 
     const body = await request.json()
-    const { projectId, config } = searchSchema.parse(body)
+    const parsed = searchSchema.parse(body)
+    const { projectId, config } = parsed
+    jobId = parsed.jobId
 
     const token = await convexAuthNextjsToken();
+    if (jobId) {
+      await fetchMutation(
+        api.searchJobs.update,
+        { jobId: jobId as any, status: "running", progress: 10 },
+        { token }
+      );
+    }
     const searchContext = await fetchQuery(
       api.projects.getSearchContext,
       { projectId: projectId as any },
@@ -45,6 +56,13 @@ export async function POST(request: NextRequest) {
     );
 
     if (!searchContext.context) {
+      if (jobId) {
+        await fetchMutation(
+          api.searchJobs.update,
+          { jobId: jobId as any, status: "failed", error: "No analysis available." },
+          { token }
+        );
+      }
       return NextResponse.json(
         { error: 'No analysis available for selected sources.' },
         { status: 400 }
@@ -60,18 +78,51 @@ export async function POST(request: NextRequest) {
 
     // Generate queries
     console.log('   Generating search queries...')
-    const queries = generateSearchQueries(analysis as EnhancedProductAnalysis, config as SearchConfig)
+    const enforcedConfig: SearchConfig = {
+      ...(config as SearchConfig),
+      maxResults: Math.min((config as SearchConfig).maxResults || 50, 50),
+    }
+    const queries = generateSearchQueries(analysis as EnhancedProductAnalysis, enforcedConfig)
     console.log(`   ✓ Generated ${queries.length} queries`)
+    if (jobId) {
+      await fetchMutation(
+        api.searchJobs.update,
+        { jobId: jobId as any, status: "running", progress: 40 },
+        { token }
+      );
+    }
 
     // Execute searches
     console.log('   Executing searches...')
     const searchResults = await executeSearches(queries)
     console.log(`   ✓ Found ${searchResults.length} raw results`)
+    if (jobId) {
+      await fetchMutation(
+        api.searchJobs.update,
+        { jobId: jobId as any, status: "running", progress: 70 },
+        { token }
+      );
+    }
 
     // Score and rank
     console.log('   Scoring opportunities...')
     const opportunities = scoreOpportunities(searchResults, analysis as EnhancedProductAnalysis)
     console.log(`   ✓ Scored ${opportunities.length} opportunities`)
+    if (jobId) {
+      await fetchMutation(
+        api.searchJobs.update,
+        { jobId: jobId as any, status: "running", progress: 90 },
+        { token }
+      );
+    }
+
+    if (jobId) {
+      await fetchMutation(
+        api.searchJobs.update,
+        { jobId: jobId as any, status: "completed", progress: 100 },
+        { token }
+      );
+    }
 
     return NextResponse.json({
       success: true,
@@ -97,17 +148,36 @@ export async function POST(request: NextRequest) {
     })
 
   } catch (error: any) {
-    console.error('❌ Opportunity search error:', error)
+    const errorMessage =
+      error instanceof Error ? error.message : typeof error === "string" ? error : "Search failed"
+    console.error("❌ Opportunity search error:", errorMessage)
 
-    if (error.name === 'ZodError') {
+    if (jobId) {
+      try {
+        const token = await convexAuthNextjsToken();
+        await fetchMutation(
+          api.searchJobs.update,
+          {
+            jobId: jobId as any,
+            status: "failed",
+            error: errorMessage
+          },
+          { token }
+        );
+      } catch {
+        // Best-effort job update only.
+      }
+    }
+    
+    if (error?.name === 'ZodError') {
       return NextResponse.json(
-        { error: 'Invalid request format', details: error.errors },
+        { error: 'Invalid request format', details: error?.errors },
         { status: 400 }
       )
     }
 
     return NextResponse.json(
-      { error: error.message || 'Failed to search for opportunities' },
+      { error: errorMessage || 'Failed to search for opportunities' },
       { status: 500 }
     )
   }

app/onboarding/page.tsx

@@ -11,8 +11,9 @@ import { Skeleton } from '@/components/ui/skeleton'
 import { Alert, AlertDescription } from '@/components/ui/alert'
 import { ArrowRight, Globe, Loader2, Sparkles, AlertCircle, ArrowLeft } from 'lucide-react'
 import type { EnhancedProductAnalysis, Keyword } from '@/lib/types'
-import { useMutation } from 'convex/react'
+import { useMutation, useQuery } from 'convex/react'
 import { api } from '@/convex/_generated/api'
+import { AnalysisTimeline } from '@/components/analysis-timeline'
 
 const examples = [
   { name: 'Notion', url: 'https://notion.so' },
@@ -26,6 +27,7 @@ export default function OnboardingPage() {
   const addDataSource = useMutation(api.dataSources.addDataSource)
   const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus)
   const createAnalysis = useMutation(api.analyses.createAnalysis)
+  const createAnalysisJob = useMutation(api.analysisJobs.create)
   const [url, setUrl] = useState('')
   const [loading, setLoading] = useState(false)
   const [progress, setProgress] = useState('')
@@ -36,33 +38,56 @@ export default function OnboardingPage() {
   const [manualProductName, setManualProductName] = useState('')
   const [manualDescription, setManualDescription] = useState('')
   const [manualFeatures, setManualFeatures] = useState('')
+  const [pendingSourceId, setPendingSourceId] = useState<string | null>(null)
+  const [pendingProjectId, setPendingProjectId] = useState<string | null>(null)
+  const [pendingJobId, setPendingJobId] = useState<string | null>(null)
+  const analysisJob = useQuery(
+    api.analysisJobs.getById,
+    pendingJobId ? { jobId: pendingJobId as any } : "skip"
+  )
 
   const persistAnalysis = async ({
     analysis,
     sourceUrl,
     sourceName,
+    projectId,
+    dataSourceId,
   }: {
     analysis: EnhancedProductAnalysis
     sourceUrl: string
     sourceName: string
+    projectId?: string
+    dataSourceId?: string
   }) => {
-    const { sourceId, projectId } = await addDataSource({
+    const resolved = projectId && dataSourceId
-      url: sourceUrl,
+      ? { projectId, sourceId: dataSourceId }
-      name: sourceName,
+      : await addDataSource({
-      type: 'website',
+          url: sourceUrl,
-    })
+          name: sourceName,
+          type: 'website',
+        })
 
-    await createAnalysis({
+    try {
-      projectId,
+      await createAnalysis({
-      dataSourceId: sourceId,
+        projectId: resolved.projectId,
-      analysis,
+        dataSourceId: resolved.sourceId,
-    })
+        analysis,
+      })
 
-    await updateDataSourceStatus({
+      await updateDataSourceStatus({
-      dataSourceId: sourceId,
+        dataSourceId: resolved.sourceId,
-      analysisStatus: 'completed',
+        analysisStatus: 'completed',
-      lastAnalyzedAt: Date.now(),
+        lastAnalyzedAt: Date.now(),
-    })
+      })
+    } catch (err: any) {
+      await updateDataSourceStatus({
+        dataSourceId: resolved.sourceId,
+        analysisStatus: 'failed',
+        lastError: err?.message || 'Failed to save analysis',
+        lastAnalyzedAt: Date.now(),
+      })
+      throw err
+    }
   }
 
   async function analyzeWebsite() {
@@ -71,12 +96,35 @@ export default function OnboardingPage() {
     setLoading(true)
     setError('')
     setProgress('Scraping website...')
+    let manualFallback = false
 
     try {
+      const { sourceId, projectId } = await addDataSource({
+        url,
+        name: url.replace(/^https?:\/\//, '').replace(/\/$/, ''),
+        type: 'website',
+      })
+
+      await updateDataSourceStatus({
+        dataSourceId: sourceId,
+        analysisStatus: 'pending',
+        lastError: undefined,
+        lastAnalyzedAt: undefined,
+      })
+
+      setPendingSourceId(sourceId)
+      setPendingProjectId(projectId)
+
+      const jobId = await createAnalysisJob({
+        projectId,
+        dataSourceId: sourceId,
+      })
+      setPendingJobId(jobId)
+
       const response = await fetch('/api/analyze', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ url }),
+        body: JSON.stringify({ url, jobId }),
       })
 
       if (response.redirected) {
@@ -90,6 +138,7 @@ export default function OnboardingPage() {
         if (data.needsManualInput) {
           setShowManualInput(true)
           setManualProductName(url.replace(/^https?:\/\//, '').replace(/\/$/, ''))
+          manualFallback = true
           throw new Error(data.error)
         }
         throw new Error(data.error || 'Failed to analyze')
@@ -106,8 +155,14 @@ export default function OnboardingPage() {
         analysis: data.data,
         sourceUrl: url,
         sourceName: data.data.productName,
+        projectId,
+        dataSourceId: sourceId,
       })
 
+      setPendingSourceId(null)
+      setPendingProjectId(null)
+      setPendingJobId(null)
+
       setProgress('Redirecting to dashboard...')
 
       // Redirect to dashboard with product name in query
@@ -116,6 +171,14 @@ export default function OnboardingPage() {
     } catch (err: any) {
       console.error('Analysis error:', err)
       setError(err.message || 'Failed to analyze website')
+      if (pendingSourceId && !manualFallback) {
+        await updateDataSourceStatus({
+          dataSourceId: pendingSourceId,
+          analysisStatus: 'failed',
+          lastError: err?.message || 'Failed to analyze',
+          lastAnalyzedAt: Date.now(),
+        })
+      }
     } finally {
       setLoading(false)
     }
@@ -172,13 +235,38 @@ export default function OnboardingPage() {
       }
 
       // Send to API to enhance with AI
+      let resolvedProjectId = pendingProjectId
+      let resolvedSourceId = pendingSourceId
+      let resolvedJobId = pendingJobId
+
+      if (!resolvedProjectId || !resolvedSourceId) {
+        const created = await addDataSource({
+          url: `manual:${manualProductName}`,
+          name: manualProductName,
+          type: 'website',
+        })
+        resolvedProjectId = created.projectId
+        resolvedSourceId = created.sourceId
+        setPendingProjectId(created.projectId)
+        setPendingSourceId(created.sourceId)
+      }
+
+      if (!resolvedJobId && resolvedProjectId && resolvedSourceId) {
+        resolvedJobId = await createAnalysisJob({
+          projectId: resolvedProjectId,
+          dataSourceId: resolvedSourceId,
+        })
+        setPendingJobId(resolvedJobId)
+      }
+
       const response = await fetch('/api/analyze-manual', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({
           productName: manualProductName,
           description: manualDescription,
-          features: manualFeatures
+          features: manualFeatures,
+          jobId: resolvedJobId || undefined,
         }),
       })
 
@@ -206,18 +294,36 @@ export default function OnboardingPage() {
       }))
 
       setProgress('Saving analysis...')
+      const manualSourceUrl = pendingSourceId
+        ? 'manual-input'
+        : `manual:${finalAnalysis.productName}`
+
       await persistAnalysis({
         analysis: finalAnalysis,
-        sourceUrl: 'manual-input',
+        sourceUrl: manualSourceUrl,
         sourceName: finalAnalysis.productName,
+        projectId: resolvedProjectId || undefined,
+        dataSourceId: resolvedSourceId || undefined,
       })
 
+      setPendingSourceId(null)
+      setPendingProjectId(null)
+      setPendingJobId(null)
+
       // Redirect to dashboard
       const params = new URLSearchParams({ product: finalAnalysis.productName })
       router.push(`/dashboard?${params.toString()}`)
     } catch (err: any) {
       console.error('Manual analysis error:', err)
       setError(err.message || 'Failed to analyze')
+      if (pendingSourceId) {
+        await updateDataSourceStatus({
+          dataSourceId: pendingSourceId,
+          analysisStatus: 'failed',
+          lastError: err?.message || 'Failed to analyze',
+          lastAnalyzedAt: Date.now(),
+        })
+      }
     } finally {
       setLoading(false)
     }
@@ -294,10 +400,14 @@ export default function OnboardingPage() {
                       <Loader2 className="h-4 w-4 animate-spin" />
                       {progress}
                     </div>
-                    <div className="space-y-2">
+                    {analysisJob?.timeline?.length ? (
-                      <Skeleton className="h-4 w-full" />
+                      <AnalysisTimeline items={analysisJob.timeline} />
-                      <Skeleton className="h-4 w-4/5" />
+                    ) : (
-                    </div>
+                      <div className="space-y-2">
+                        <Skeleton className="h-4 w-full" />
+                        <Skeleton className="h-4 w-4/5" />
+                      </div>
+                    )}
                   </div>
                 )}
 
@@ -400,11 +510,15 @@ export default function OnboardingPage() {
                     <Loader2 className="h-4 w-4 animate-spin" />
                     {progress}
                   </div>
-                  <div className="space-y-2">
+                  {analysisJob?.timeline?.length ? (
-                    <Skeleton className="h-4 w-full" />
+                    <AnalysisTimeline items={analysisJob.timeline} />
-                    <Skeleton className="h-4 w-4/5" />
+                  ) : (
-                    <Skeleton className="h-4 w-3/5" />
+                    <div className="space-y-2">
-                  </div>
+                      <Skeleton className="h-4 w-full" />
+                      <Skeleton className="h-4 w-4/5" />
+                      <Skeleton className="h-4 w-3/5" />
+                    </div>
+                  )}
                 </div>
               )}
 

components/analysis-timeline.tsx

@@ -0,0 +1,82 @@
+"use client"
+
+import { CheckCircle2, AlertTriangle, Loader2 } from "lucide-react"
+import { cn } from "@/lib/utils"
+
+type TimelineItem = {
+  key: string
+  label: string
+  status: "pending" | "running" | "completed" | "failed"
+  detail?: string
+}
+
+function StatusIcon({ status }: { status: TimelineItem["status"] }) {
+  if (status === "completed") {
+    return <CheckCircle2 className="h-4 w-4 text-foreground" />
+  }
+  if (status === "failed") {
+    return <AlertTriangle className="h-4 w-4 text-destructive" />
+  }
+  if (status === "running") {
+    return <Loader2 className="h-4 w-4 animate-spin text-muted-foreground" />
+  }
+  return <span className="h-2.5 w-2.5 rounded-full border border-muted-foreground/60" />
+}
+
+export function AnalysisTimeline({ items }: { items: TimelineItem[] }) {
+  if (!items.length) return null
+
+  return (
+    <div className="rounded-lg border border-border/60 bg-muted/20 p-4">
+      <div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">
+        Analysis timeline
+      </div>
+      <div className="mt-3 space-y-3">
+        {items.map((item, index) => {
+          const isPending = item.status === "pending"
+          const nextStatus = items[index + 1]?.status
+          const isStrongLine =
+            nextStatus &&
+            (item.status === "completed" || item.status === "running") &&
+            (nextStatus === "completed" || nextStatus === "running")
+          return (
+            <div key={item.key} className="relative pl-6">
+              <span
+                className={cn(
+                  "absolute left-[6px] top-3 bottom-[-12px] w-px",
+                  index === items.length - 1 ? "hidden" : "bg-border/40",
+                  isStrongLine && "bg-foreground/60"
+                )}
+              />
+              <div
+                className={cn(
+                  "flex items-start gap-3 text-sm transition",
+                  isPending && "scale-[0.96] opacity-60"
+                )}
+              >
+                <div className="mt-0.5 flex h-5 w-5 items-center justify-center rounded-full bg-background shadow-sm">
+                  <StatusIcon status={item.status} />
+                </div>
+                <div className="min-w-0">
+                  <div
+                    className={cn(
+                      "font-medium",
+                      item.status === "failed" && "text-destructive"
+                    )}
+                  >
+                    {item.label}
+                  </div>
+                  {item.detail && (
+                    <div className="text-xs text-muted-foreground">
+                      {item.detail}
+                    </div>
+                  )}
+                </div>
+              </div>
+            </div>
+          )
+        })}
+      </div>
+    </div>
+  )
+}

components/app-sidebar.tsx

@@ -7,10 +7,13 @@ import {
     Command,
     Frame,
     HelpCircle,
+    Settings,
     Settings2,
     Terminal,
     Target,
-    Plus
+    Plus,
+    ArrowUpRight,
+    ChevronsUpDown
 } from "lucide-react"
 
 import { NavUser } from "@/components/nav-user"
@@ -26,6 +29,14 @@ import {
     SidebarGroupLabel,
     SidebarGroupContent,
 } from "@/components/ui/sidebar"
+import {
+    DropdownMenu,
+    DropdownMenuContent,
+    DropdownMenuItem,
+    DropdownMenuLabel,
+    DropdownMenuSeparator,
+    DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu"
 import { useQuery, useMutation } from "convex/react"
 import { api } from "@/convex/_generated/api"
 import { Checkbox } from "@/components/ui/checkbox"
@@ -34,19 +45,49 @@ import { useProject } from "@/components/project-context"
 import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
 import { Input } from "@/components/ui/input"
 import { Button } from "@/components/ui/button"
+import { Textarea } from "@/components/ui/textarea"
+import { AnalysisTimeline } from "@/components/analysis-timeline"
 
 export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
     const pathname = usePathname()
     const projects = useQuery(api.projects.getProjects);
+    const currentUser = useQuery(api.users.getCurrent);
     const { selectedProjectId, setSelectedProjectId } = useProject();
     const addDataSource = useMutation(api.dataSources.addDataSource);
     const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus);
     const createAnalysis = useMutation(api.analyses.createAnalysis);
+    const createAnalysisJob = useMutation(api.analysisJobs.create);
+    const updateAnalysisJob = useMutation(api.analysisJobs.update);
     const [isAdding, setIsAdding] = React.useState(false);
+    const [isCreatingProject, setIsCreatingProject] = React.useState(false);
+    const [projectName, setProjectName] = React.useState("");
+    const [projectDefault, setProjectDefault] = React.useState(true);
+    const [projectError, setProjectError] = React.useState<string | null>(null);
+    const [isSubmittingProject, setIsSubmittingProject] = React.useState(false);
+    const createProject = useMutation(api.projects.createProject);
+    const updateProject = useMutation(api.projects.updateProject);
+    const [isEditingProject, setIsEditingProject] = React.useState(false);
+    const [editingProjectId, setEditingProjectId] = React.useState<string | null>(null);
+    const [editingProjectName, setEditingProjectName] = React.useState("");
+    const [editingProjectDefault, setEditingProjectDefault] = React.useState(false);
+    const [editingProjectError, setEditingProjectError] = React.useState<string | null>(null);
+    const [isSubmittingEdit, setIsSubmittingEdit] = React.useState(false);
     const [sourceUrl, setSourceUrl] = React.useState("");
     const [sourceName, setSourceName] = React.useState("");
     const [sourceError, setSourceError] = React.useState<string | null>(null);
+    const [sourceNotice, setSourceNotice] = React.useState<string | null>(null);
     const [isSubmittingSource, setIsSubmittingSource] = React.useState(false);
+    const [manualMode, setManualMode] = React.useState(false);
+    const [manualProductName, setManualProductName] = React.useState("");
+    const [manualDescription, setManualDescription] = React.useState("");
+    const [manualFeatures, setManualFeatures] = React.useState("");
+    const [pendingSourceId, setPendingSourceId] = React.useState<string | null>(null);
+    const [pendingProjectId, setPendingProjectId] = React.useState<string | null>(null);
+    const [pendingJobId, setPendingJobId] = React.useState<string | null>(null);
+    const analysisJob = useQuery(
+        api.analysisJobs.getById,
+        pendingJobId ? { jobId: pendingJobId as any } : "skip"
+    );
 
     // Set default selected project
     React.useEffect(() => {
@@ -67,6 +108,7 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
 
     const selectedProject = projects?.find(p => p._id === selectedProjectId);
     const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || [];
+    const selectedProjectName = selectedProject?.name || "Select Project";
 
     const handleToggle = async (sourceId: string, checked: boolean) => {
         if (!selectedProjectId) return;
@@ -87,15 +129,27 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
         setIsSubmittingSource(true);
 
         try {
-            const { sourceId, projectId } = await addDataSource({
+            const result = await addDataSource({
                 projectId: selectedProjectId as any,
                 url: sourceUrl,
                 name: sourceName || sourceUrl,
                 type: "website",
             });
 
+            if ((result as any).isExisting) {
+                setSourceNotice("This source already exists and was reused.");
+            }
+
+            const jobId = await createAnalysisJob({
+                projectId: result.projectId,
+                dataSourceId: result.sourceId,
+            });
+            setPendingSourceId(result.sourceId);
+            setPendingProjectId(result.projectId);
+            setPendingJobId(jobId);
+
             await updateDataSourceStatus({
-                dataSourceId: sourceId,
+                dataSourceId: result.sourceId,
                 analysisStatus: "pending",
                 lastError: undefined,
                 lastAnalyzedAt: undefined,
@@ -104,14 +158,25 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
             const response = await fetch("/api/analyze", {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ url: sourceUrl }),
+                body: JSON.stringify({ url: sourceUrl, jobId }),
             });
 
             const data = await response.json();
 
             if (!response.ok) {
+                if (data.needsManualInput) {
+                    setManualMode(true);
+                    setManualProductName(
+                        sourceName || sourceUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")
+                    );
+                    setPendingSourceId(result.sourceId);
+                    setPendingProjectId(result.projectId);
+                    setSourceError(data.error || "Manual input required.");
+                    return;
+                }
+
                 await updateDataSourceStatus({
-                    dataSourceId: sourceId,
+                    dataSourceId: result.sourceId,
                     analysisStatus: "failed",
                     lastError: data.error || "Analysis failed",
                     lastAnalyzedAt: Date.now(),
@@ -120,13 +185,13 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
             }
 
             await createAnalysis({
-                projectId,
+                projectId: result.projectId,
-                dataSourceId: sourceId,
+                dataSourceId: result.sourceId,
                 analysis: data.data,
             });
 
             await updateDataSourceStatus({
-                dataSourceId: sourceId,
+                dataSourceId: result.sourceId,
                 analysisStatus: "completed",
                 lastError: undefined,
                 lastAnalyzedAt: Date.now(),
@@ -134,6 +199,14 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
 
             setSourceUrl("");
             setSourceName("");
+            setSourceNotice(null);
+            setManualMode(false);
+            setManualProductName("");
+            setManualDescription("");
+            setManualFeatures("");
+            setPendingSourceId(null);
+            setPendingProjectId(null);
+            setPendingJobId(null);
             setIsAdding(false);
         } catch (err: any) {
             setSourceError(err?.message || "Failed to add source.");
@@ -142,22 +215,138 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
         }
     };
 
+    const handleManualAnalyze = async () => {
+        if (!manualProductName || !manualDescription) {
+            setSourceError("Product name and description are required.");
+            return;
+        }
+
+        if (!pendingSourceId || !pendingProjectId) {
+            setSourceError("Missing pending source.");
+            return;
+        }
+
+        setIsSubmittingSource(true);
+        setSourceError(null);
+
+        try {
+            const response = await fetch("/api/analyze-manual", {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    productName: manualProductName,
+                    description: manualDescription,
+                    features: manualFeatures,
+                    jobId: pendingJobId || undefined,
+                }),
+            });
+
+            const data = await response.json();
+
+            if (!response.ok) {
+                await updateDataSourceStatus({
+                    dataSourceId: pendingSourceId as any,
+                    analysisStatus: "failed",
+                    lastError: data.error || "Manual analysis failed",
+                    lastAnalyzedAt: Date.now(),
+                });
+                throw new Error(data.error || "Manual analysis failed");
+            }
+
+            await createAnalysis({
+                projectId: pendingProjectId as any,
+                dataSourceId: pendingSourceId as any,
+                analysis: data.data,
+            });
+
+            await updateDataSourceStatus({
+                dataSourceId: pendingSourceId as any,
+                analysisStatus: "completed",
+                lastError: undefined,
+                lastAnalyzedAt: Date.now(),
+            });
+
+            setSourceUrl("");
+            setSourceName("");
+            setManualMode(false);
+            setManualProductName("");
+            setManualDescription("");
+            setManualFeatures("");
+            setPendingSourceId(null);
+            setPendingProjectId(null);
+            setPendingJobId(null);
+            setIsAdding(false);
+        } catch (err: any) {
+            setSourceError(err?.message || "Manual analysis failed.");
+        } finally {
+            setIsSubmittingSource(false);
+        }
+    };
+
     return (
         <Sidebar variant="inset" {...props}>
             <SidebarHeader>
                 <SidebarMenu>
                     <SidebarMenuItem>
-                        <SidebarMenuButton size="lg" asChild>
+                        <DropdownMenu>
-                            <a href="#">
+                            <DropdownMenuTrigger asChild>
-                                <div className="flex aspect-square size-8 items-center justify-center rounded-lg bg-sidebar-primary text-sidebar-primary-foreground">
+                                <SidebarMenuButton size="lg">
-                                    <Command className="size-4" />
+                                    <div className="flex aspect-square size-8 items-center justify-center rounded-lg bg-sidebar-primary text-sidebar-primary-foreground">
-                                </div>
+                                        <Command className="size-4" />
-                                <div className="grid flex-1 text-left text-sm leading-tight">
+                                    </div>
-                                    <span className="truncate font-semibold uppercase">Sanati</span>
+                                    <div className="grid flex-1 text-left text-sm leading-tight">
-                                    <span className="truncate text-xs">Pro</span>
+                                        <span className="truncate font-semibold">{selectedProjectName}</span>
-                                </div>
+                                        <span className="truncate text-xs text-muted-foreground">Projects</span>
-                            </a>
+                                    </div>
-                        </SidebarMenuButton>
+                                    <ChevronsUpDown className="ml-auto size-4" />
+                                </SidebarMenuButton>
+                            </DropdownMenuTrigger>
+                            <DropdownMenuContent side="bottom" align="start" className="w-64">
+                                <DropdownMenuLabel>Switch project</DropdownMenuLabel>
+                                <DropdownMenuSeparator />
+                                {projects?.map((project) => (
+                                    <DropdownMenuItem
+                                        key={project._id}
+                                        className="justify-between"
+                                        onSelect={(event) => {
+                                            if (event.defaultPrevented) return
+                                            setSelectedProjectId(project._id)
+                                        }}
+                                    >
+                                        <div className="flex items-center gap-2">
+                                            <Frame className="text-muted-foreground" />
+                                            <span className="truncate">{project.name}</span>
+                                        </div>
+                                        <button
+                                            type="button"
+                                            data-project-settings
+                                            className="rounded-md p-1 text-muted-foreground hover:text-foreground"
+                                            onPointerDown={(event) => {
+                                                event.preventDefault()
+                                                event.stopPropagation()
+                                            }}
+                                            onClick={(event) => {
+                                                event.preventDefault()
+                                                event.stopPropagation();
+                                                setEditingProjectId(project._id);
+                                                setEditingProjectName(project.name);
+                                                setEditingProjectDefault(project.isDefault);
+                                                setEditingProjectError(null);
+                                                setIsEditingProject(true);
+                                            }}
+                                            aria-label={`Project settings for ${project.name}`}
+                                        >
+                                            <Settings className="size-4" />
+                                        </button>
+                                    </DropdownMenuItem>
+                                ))}
+                                <DropdownMenuSeparator />
+                                <DropdownMenuItem onSelect={() => setIsCreatingProject(true)}>
+                                    <Plus />
+                                    Create Project
+                                </DropdownMenuItem>
+                            </DropdownMenuContent>
+                        </DropdownMenu>
                     </SidebarMenuItem>
                 </SidebarMenu>
             </SidebarHeader>
@@ -219,32 +408,6 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                     </SidebarGroupContent>
                 </SidebarGroup>
 
-                {/* Projects (Simple List for now, can be switcher) */}
-                <SidebarGroup className="group-data-[collapsible=icon]:hidden">
-                    <SidebarGroupLabel>Projects</SidebarGroupLabel>
-                    <SidebarGroupContent>
-                        <SidebarMenu>
-                            {projects?.map((project) => (
-                                <SidebarMenuItem key={project._id}>
-                                    <SidebarMenuButton
-                                        onClick={() => setSelectedProjectId(project._id)}
-                                        isActive={selectedProjectId === project._id}
-                                    >
-                                        <Frame className="text-muted-foreground" />
-                                        <span>{project.name}</span>
-                                    </SidebarMenuButton>
-                                </SidebarMenuItem>
-                            ))}
-                            <SidebarMenuItem>
-                                <SidebarMenuButton className="text-muted-foreground">
-                                    <Plus />
-                                    <span>Create Project</span>
-                                </SidebarMenuButton>
-                            </SidebarMenuItem>
-                        </SidebarMenu>
-                    </SidebarGroupContent>
-                </SidebarGroup>
-
                 {/* Data Sources Config */}
                 {selectedProjectId && (
                     <SidebarGroup>
@@ -258,15 +421,30 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                                     <div className="text-sm text-muted-foreground pl-2">No data sources yet.</div>
                                 )}
                                 {dataSources?.map((source) => (
-                                    <div key={source._id} className="flex items-center space-x-2">
+                                    <div
-                                        <Checkbox
+                                        key={source._id}
-                                            id={source._id}
+                                        className="flex items-center justify-between gap-2 rounded-md px-2 py-1 hover:bg-muted/40"
-                                            checked={selectedSourceIds.includes(source._id)}
+                                    >
-                                            onCheckedChange={(checked) => handleToggle(source._id, checked === true)}
+                                        <div className="flex min-w-0 items-center gap-2">
-                                        />
+                                            <Checkbox
-                                        <Label htmlFor={source._id} className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 truncate cursor-pointer">
+                                                id={source._id}
-                                            {source.name || source.url}
+                                                checked={selectedSourceIds.includes(source._id)}
-                                        </Label>
+                                                onCheckedChange={(checked) => handleToggle(source._id, checked === true)}
+                                            />
+                                            <Label
+                                                htmlFor={source._id}
+                                                className="truncate text-sm font-medium leading-none cursor-pointer"
+                                            >
+                                                {source.name || source.url}
+                                            </Label>
+                                        </div>
+                                        <Link
+                                            href={`/data-sources/${source._id}`}
+                                            className="inline-flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground"
+                                        >
+                                            Details
+                                            <ArrowUpRight className="size-3" />
+                                        </Link>
                                     </div>
                                 ))}
                                 <Button
@@ -282,13 +460,45 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                 )}
             </SidebarContent>
             <SidebarFooter>
-                <NavUser user={{
+                {currentUser && (currentUser.name || currentUser.email) && (
-                    name: "User",
+                    <NavUser
-                    email: "user@example.com",
+                        user={{
-                    avatar: ""
+                            name: currentUser.name || currentUser.email || "",
-                }} />
+                            email: currentUser.email || "",
+                            avatar: currentUser.image || "",
+                        }}
+                    />
+                )}
             </SidebarFooter>
-            <Dialog open={isAdding} onOpenChange={setIsAdding}>
+            <Dialog
+                open={isAdding}
+                onOpenChange={(open) => {
+                    if (!open && manualMode && pendingSourceId) {
+                        updateDataSourceStatus({
+                            dataSourceId: pendingSourceId as any,
+                            analysisStatus: "failed",
+                            lastError: "Manual input cancelled",
+                            lastAnalyzedAt: Date.now(),
+                        });
+                    }
+                    if (!open && manualMode && pendingJobId) {
+                        updateAnalysisJob({
+                            jobId: pendingJobId as any,
+                            status: "failed",
+                            error: "Manual input cancelled",
+                        });
+                    }
+                    if (!open) {
+                        setManualMode(false);
+                        setSourceError(null);
+                        setSourceNotice(null);
+                        setPendingSourceId(null);
+                        setPendingProjectId(null);
+                        setPendingJobId(null);
+                    }
+                    setIsAdding(open);
+                }}
+            >
                 <DialogContent>
                     <DialogHeader>
                         <DialogTitle>Add Data Source</DialogTitle>
@@ -297,35 +507,224 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
                         </DialogDescription>
                     </DialogHeader>
                     <div className="space-y-3">
-                        <div className="space-y-2">
+                        {!manualMode && (
-                            <Label htmlFor="sourceUrl">Website URL</Label>
+                            <>
-                            <Input
+                                <div className="space-y-2">
-                                id="sourceUrl"
+                                    <Label htmlFor="sourceUrl">Website URL</Label>
-                                placeholder="https://example.com"
+                                    <Input
-                                value={sourceUrl}
+                                        id="sourceUrl"
-                                onChange={(event) => setSourceUrl(event.target.value)}
+                                        placeholder="https://example.com"
-                                disabled={isSubmittingSource}
+                                        value={sourceUrl}
-                            />
+                                        onChange={(event) => setSourceUrl(event.target.value)}
-                        </div>
+                                        disabled={isSubmittingSource}
-                        <div className="space-y-2">
+                                    />
-                            <Label htmlFor="sourceName">Name (optional)</Label>
+                                </div>
-                            <Input
+                                <div className="space-y-2">
-                                id="sourceName"
+                                    <Label htmlFor="sourceName">Name (optional)</Label>
-                                placeholder="Product name"
+                                    <Input
-                                value={sourceName}
+                                        id="sourceName"
-                                onChange={(event) => setSourceName(event.target.value)}
+                                        placeholder="Product name"
-                                disabled={isSubmittingSource}
+                                        value={sourceName}
-                            />
+                                        onChange={(event) => setSourceName(event.target.value)}
-                        </div>
+                                        disabled={isSubmittingSource}
+                                    />
+                                </div>
+                            </>
+                        )}
+                        {manualMode && (
+                            <>
+                                <div className="space-y-2">
+                                    <Label htmlFor="manualProductName">Product Name</Label>
+                                    <Input
+                                        id="manualProductName"
+                                        value={manualProductName}
+                                        onChange={(event) => setManualProductName(event.target.value)}
+                                        disabled={isSubmittingSource}
+                                    />
+                                </div>
+                                <div className="space-y-2">
+                                    <Label htmlFor="manualDescription">Description</Label>
+                                    <Textarea
+                                        id="manualDescription"
+                                        value={manualDescription}
+                                        onChange={(event) => setManualDescription(event.target.value)}
+                                        disabled={isSubmittingSource}
+                                        rows={3}
+                                    />
+                                </div>
+                                <div className="space-y-2">
+                                    <Label htmlFor="manualFeatures">Key Features (one per line)</Label>
+                                    <Textarea
+                                        id="manualFeatures"
+                                        value={manualFeatures}
+                                        onChange={(event) => setManualFeatures(event.target.value)}
+                                        disabled={isSubmittingSource}
+                                        rows={3}
+                                    />
+                                </div>
+                            </>
+                        )}
+                        {sourceNotice && (
+                            <div className="text-sm text-muted-foreground">{sourceNotice}</div>
+                        )}
                         {sourceError && (
                             <div className="text-sm text-destructive">{sourceError}</div>
                         )}
+                        {analysisJob?.timeline?.length ? (
+                            <AnalysisTimeline items={analysisJob.timeline} />
+                        ) : null}
                         <div className="flex justify-end gap-2">
-                            <Button variant="outline" onClick={() => setIsAdding(false)} disabled={isSubmittingSource}>
+                            <Button
+                                variant="outline"
+                                onClick={() => setIsAdding(false)}
+                                disabled={isSubmittingSource}
+                            >
                                 Cancel
                             </Button>
-                            <Button onClick={handleAddSource} disabled={isSubmittingSource}>
+                            {manualMode ? (
-                                {isSubmittingSource ? "Analyzing..." : "Add Source"}
+                                <Button onClick={handleManualAnalyze} disabled={isSubmittingSource}>
+                                    {isSubmittingSource ? "Analyzing..." : "Analyze Manually"}
+                                </Button>
+                            ) : (
+                                <Button onClick={handleAddSource} disabled={isSubmittingSource}>
+                                    {isSubmittingSource ? "Analyzing..." : "Add Source"}
+                                </Button>
+                            )}
+                        </div>
+                    </div>
+                </DialogContent>
+            </Dialog>
+            <Dialog open={isEditingProject} onOpenChange={setIsEditingProject}>
+                <DialogContent>
+                    <DialogHeader>
+                        <DialogTitle>Project Settings</DialogTitle>
+                        <DialogDescription>
+                            Update the project name and default status.
+                        </DialogDescription>
+                    </DialogHeader>
+                    <div className="space-y-3">
+                        <div className="space-y-2">
+                            <Label htmlFor="editProjectName">Project Name</Label>
+                            <Input
+                                id="editProjectName"
+                                value={editingProjectName}
+                                onChange={(event) => setEditingProjectName(event.target.value)}
+                                disabled={isSubmittingEdit}
+                            />
+                        </div>
+                        <div className="flex items-center gap-2">
+                            <Checkbox
+                                id="editProjectDefault"
+                                checked={editingProjectDefault}
+                                onCheckedChange={(checked) => setEditingProjectDefault(checked === true)}
+                                disabled={isSubmittingEdit}
+                            />
+                            <Label htmlFor="editProjectDefault">Set as default</Label>
+                        </div>
+                        {editingProjectError && (
+                            <div className="text-sm text-destructive">{editingProjectError}</div>
+                        )}
+                        <div className="flex justify-end gap-2">
+                            <Button
+                                variant="outline"
+                                onClick={() => setIsEditingProject(false)}
+                                disabled={isSubmittingEdit}
+                            >
+                                Cancel
+                            </Button>
+                            <Button
+                                onClick={async () => {
+                                    if (!editingProjectId) return;
+                                    if (!editingProjectName.trim()) {
+                                        setEditingProjectError("Project name is required.");
+                                        return;
+                                    }
+                                    setIsSubmittingEdit(true);
+                                    setEditingProjectError(null);
+                                    try {
+                                        await updateProject({
+                                            projectId: editingProjectId as any,
+                                            name: editingProjectName.trim(),
+                                            isDefault: editingProjectDefault,
+                                        });
+                                        setIsEditingProject(false);
+                                    } catch (err: any) {
+                                        setEditingProjectError(err?.message || "Failed to update project.");
+                                    } finally {
+                                        setIsSubmittingEdit(false);
+                                    }
+                                }}
+                                disabled={isSubmittingEdit}
+                            >
+                                Save Changes
+                            </Button>
+                        </div>
+                    </div>
+                </DialogContent>
+            </Dialog>
+            <Dialog open={isCreatingProject} onOpenChange={setIsCreatingProject}>
+                <DialogContent>
+                    <DialogHeader>
+                        <DialogTitle>Create Project</DialogTitle>
+                        <DialogDescription>
+                            Add a new project for a separate product or workflow.
+                        </DialogDescription>
+                    </DialogHeader>
+                    <div className="space-y-3">
+                        <div className="space-y-2">
+                            <Label htmlFor="projectName">Project Name</Label>
+                            <Input
+                                id="projectName"
+                                value={projectName}
+                                onChange={(event) => setProjectName(event.target.value)}
+                                disabled={isSubmittingProject}
+                            />
+                        </div>
+                        <div className="flex items-center gap-2 text-sm">
+                            <Checkbox
+                                id="projectDefault"
+                                checked={projectDefault}
+                                onCheckedChange={(checked) => setProjectDefault(checked === true)}
+                            />
+                            <Label htmlFor="projectDefault">Make this the default project</Label>
+                        </div>
+                        {projectError && (
+                            <div className="text-sm text-destructive">{projectError}</div>
+                        )}
+                        <div className="flex justify-end gap-2">
+                            <Button
+                                variant="outline"
+                                onClick={() => setIsCreatingProject(false)}
+                                disabled={isSubmittingProject}
+                            >
+                                Cancel
+                            </Button>
+                            <Button
+                                onClick={async () => {
+                                    if (!projectName.trim()) {
+                                        setProjectError("Project name is required.");
+                                        return;
+                                    }
+                                    setProjectError(null);
+                                    setIsSubmittingProject(true);
+                                    try {
+                                        const projectId = await createProject({
+                                            name: projectName.trim(),
+                                            isDefault: projectDefault,
+                                        });
+                                        setSelectedProjectId(projectId as any);
+                                        setProjectName("");
+                                        setProjectDefault(true);
+                                        setIsCreatingProject(false);
+                                    } catch (err: any) {
+                                        setProjectError(err?.message || "Failed to create project.");
+                                    } finally {
+                                        setIsSubmittingProject(false);
+                                    }
+                                }}
+                                disabled={isSubmittingProject}
+                            >
+                                {isSubmittingProject ? "Creating..." : "Create Project"}
                             </Button>
                         </div>
                     </div>

components/hero-shader.tsx

@@ -16,8 +16,10 @@ export function HeroShader() {
         let animationFrameId: number;
 
         const resize = () => {
-            canvas.width = canvas.offsetWidth;
+            const dpr = window.devicePixelRatio || 1;
-            canvas.height = canvas.offsetHeight;
+            canvas.width = canvas.offsetWidth * dpr;
+            canvas.height = canvas.offsetHeight * dpr;
+            ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
         };
         resize();
         window.addEventListener("resize", resize);
@@ -36,15 +38,25 @@ export function HeroShader() {
 
             ctx.fillStyle = "rgba(255, 255, 255, 0.5)"; // stroke(w, 116) approx white with alpha
 
+            // Model space tuned for ~400x400 tweetcart output.
+            const baseSize = 400;
+            const scale = Math.min(canvas.width, canvas.height) / baseSize;
+
+            const canvasWidth = canvas.offsetWidth;
+            const canvasHeight = canvas.offsetHeight;
+
             // Center of drawing - positioned to the right and aligned with content
-            const cx = canvas.width * 0.7;
+            const cx = canvasWidth * 0.7;
-            const cy = canvas.height * 0.4;
+            const cy = canvasHeight * 0.52;
 
             // Loop for points
             // for(t+=PI/90,i=1e4;i--;)a()
             t += Math.PI / 90;
 
-            for (let i = 10000; i > 0; i--) {
+            const density = Math.max(1, scale);
+            const pointCount = Math.min(18000, Math.floor(10000 * density));
+
+            for (let i = pointCount; i > 0; i--) {
                 // y = i / 790
                 let y = i / 790;
 
@@ -87,17 +99,17 @@ export function HeroShader() {
                 const c = d / 4 - t / 2 + (i % 2) * 3;
 
                 // q = y * k / 5 * (2 + sin(d*2 + y - t*4)) + 80
-                const q = y * k / 5 * (2 + Math.sin(d * 2 + y - t * 4)) + 300;
+                const q = y * k / 5 * (2 + Math.sin(d * 2 + y - t * 4)) + 80;
 
-                // x = q * cos(c) + 200
+                // Original offsets assume a 400x400 canvas; map from model space to screen space.
-                // y_out = q * sin(c) + d * 9 + 60
+                const modelX = q * Math.cos(c) + 200;
-                // 200 and 60 are likely offsets for 400x400 canvas.
+                const modelY = q * Math.sin(c) + d * 9 + 60;
-                // We should center it.
 
-                const x = (q * Math.cos(c)) + cx;
+                const x = cx + (modelX - 200) * scale;
-                const y_out = (q * Math.sin(c) + d * 9) + cy;
+                const y_out = cy + (modelY - 200) * scale;
+                const pointSize = Math.min(2 * scale, 3.5);
 
-                ctx.fillRect(x, y_out, 2.5, 2.5);
+                ctx.fillRect(x, y_out, pointSize, pointSize);
             }
 
             animationFrameId = requestAnimationFrame(draw);

components/nav-user.tsx

@@ -1,5 +1,7 @@
 "use client"
 
+import * as React from "react"
+
 import {
     BadgeCheck,
     Bell,
@@ -42,6 +44,19 @@ export function NavUser({
 }) {
     const { isMobile } = useSidebar()
     const { signOut } = useAuthActions()
+    const seed = React.useMemo(() => {
+        const base = user.email || user.name || "";
+        return base.trim() || "user";
+    }, [user.email, user.name]);
+    const avatarUrl = user.avatar || `https://api.dicebear.com/7.x/bottts/svg?seed=${encodeURIComponent(seed)}`;
+    const fallbackText = React.useMemo(() => {
+        const base = user.name || user.email || "";
+        const trimmed = base.trim();
+        if (!trimmed) return "";
+        const parts = trimmed.split(/\s+/);
+        if (parts.length === 1) return parts[0].slice(0, 2).toUpperCase();
+        return `${parts[0][0]}${parts[parts.length - 1][0]}`.toUpperCase();
+    }, [user.name, user.email])
 
     return (
         <SidebarMenu>
@@ -53,8 +68,8 @@ export function NavUser({
                             className="data-[state=open]:bg-sidebar-accent data-[state=open]:text-sidebar-accent-foreground"
                         >
                             <Avatar className="h-8 w-8 rounded-lg">
-                                <AvatarImage src={user.avatar} alt={user.name} />
+                                <AvatarImage src={avatarUrl} alt={user.name} />
-                                <AvatarFallback className="rounded-lg">CN</AvatarFallback>
+                                <AvatarFallback className="rounded-lg">{fallbackText}</AvatarFallback>
                             </Avatar>
                             <div className="grid flex-1 text-left text-sm leading-tight">
                                 <span className="truncate font-semibold">{user.name}</span>
@@ -72,8 +87,8 @@ export function NavUser({
                         <DropdownMenuLabel className="p-0 font-normal">
                             <div className="flex items-center gap-2 px-1 py-1.5 text-left text-sm">
                                 <Avatar className="h-8 w-8 rounded-lg">
-                                    <AvatarImage src={user.avatar} alt={user.name} />
+                                    <AvatarImage src={avatarUrl} alt={user.name} />
-                                    <AvatarFallback className="rounded-lg">CN</AvatarFallback>
+                                    <AvatarFallback className="rounded-lg">{fallbackText}</AvatarFallback>
                                 </Avatar>
                                 <div className="grid flex-1 text-left text-sm leading-tight">
                                     <span className="truncate font-semibold">{user.name}</span>

components/ui/progress.tsx

@@ -0,0 +1,30 @@
+"use client"
+
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+type ProgressProps = React.HTMLAttributes<HTMLDivElement> & {
+  value?: number
+}
+
+const Progress = React.forwardRef<HTMLDivElement, ProgressProps>(
+  ({ className, value = 0, ...props }, ref) => (
+    <div
+      ref={ref}
+      className={cn(
+        "relative h-2 w-full overflow-hidden rounded-full bg-secondary",
+        className
+      )}
+      {...props}
+    >
+      <div
+        className="h-full bg-primary transition-all"
+        style={{ width: `${Math.min(Math.max(value, 0), 100)}%` }}
+      />
+    </div>
+  )
+)
+Progress.displayName = "Progress"
+
+export { Progress }

convex/_generated/api.d.ts

@@ -8,10 +8,15 @@
  * @module
  */
 
+import type * as analyses from "../analyses.js";
+import type * as analysisJobs from "../analysisJobs.js";
 import type * as auth from "../auth.js";
 import type * as dataSources from "../dataSources.js";
 import type * as http from "../http.js";
+import type * as opportunities from "../opportunities.js";
 import type * as projects from "../projects.js";
+import type * as searchJobs from "../searchJobs.js";
+import type * as users from "../users.js";
 
 import type {
   ApiFromModules,
@@ -20,10 +25,15 @@ import type {
 } from "convex/server";
 
 declare const fullApi: ApiFromModules<{
+  analyses: typeof analyses;
+  analysisJobs: typeof analysisJobs;
   auth: typeof auth;
   dataSources: typeof dataSources;
   http: typeof http;
+  opportunities: typeof opportunities;
   projects: typeof projects;
+  searchJobs: typeof searchJobs;
+  users: typeof users;
 }>;
 
 /**

convex/analysisJobs.ts

@@ -0,0 +1,128 @@
+import { mutation, query } from "./_generated/server";
+import { v } from "convex/values";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const create = mutation({
+  args: {
+    projectId: v.id("projects"),
+    dataSourceId: v.optional(v.id("dataSources")),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const now = Date.now();
+    return await ctx.db.insert("analysisJobs", {
+      projectId: args.projectId,
+      dataSourceId: args.dataSourceId,
+      status: "pending",
+      progress: 0,
+      stage: undefined,
+      timeline: [],
+      createdAt: now,
+      updatedAt: now,
+    });
+  },
+});
+
+export const update = mutation({
+  args: {
+    jobId: v.id("analysisJobs"),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("completed"),
+      v.literal("failed")
+    ),
+    progress: v.optional(v.number()),
+    stage: v.optional(v.string()),
+    timeline: v.optional(v.array(v.object({
+      key: v.string(),
+      label: v.string(),
+      status: v.union(
+        v.literal("pending"),
+        v.literal("running"),
+        v.literal("completed"),
+        v.literal("failed")
+      ),
+      detail: v.optional(v.string()),
+    }))),
+    error: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) throw new Error("Job not found");
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const patch: Record<string, unknown> = {
+      status: args.status,
+      updatedAt: Date.now(),
+    };
+    if (args.progress !== undefined) patch.progress = args.progress;
+    if (args.error !== undefined) patch.error = args.error;
+    if (args.stage !== undefined) patch.stage = args.stage;
+    if (args.timeline !== undefined) patch.timeline = args.timeline;
+
+    await ctx.db.patch(args.jobId, patch);
+  },
+});
+
+export const getById = query({
+  args: {
+    jobId: v.id("analysisJobs"),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return null;
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) return null;
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) return null;
+
+    return job;
+  },
+});
+
+export const listByProject = query({
+  args: {
+    projectId: v.id("projects"),
+    status: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return [];
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) return [];
+
+    if (args.status) {
+      return await ctx.db
+        .query("analysisJobs")
+        .withIndex("by_project_status", (q) =>
+          q.eq("projectId", args.projectId).eq("status", args.status!)
+        )
+        .order("desc")
+        .collect();
+    }
+
+    return await ctx.db
+      .query("analysisJobs")
+      .withIndex("by_project_createdAt", (q) => q.eq("projectId", args.projectId))
+      .order("desc")
+      .collect();
+  },
+});

convex/dataSources.ts

@@ -19,6 +19,22 @@ export const getProjectDataSources = query({
     },
 });
 
+export const getById = query({
+    args: { dataSourceId: v.id("dataSources") },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) return null;
+
+        const dataSource = await ctx.db.get(args.dataSourceId);
+        if (!dataSource) return null;
+
+        const project = await ctx.db.get(dataSource.projectId);
+        if (!project || project.userId !== userId) return null;
+
+        return dataSource;
+    },
+});
+
 export const addDataSource = mutation({
     args: {
         projectId: v.optional(v.id("projects")), // Optional, if not provided, use default
@@ -30,6 +46,14 @@ export const addDataSource = mutation({
         const userId = await getAuthUserId(ctx);
         if (!userId) throw new Error("Unauthorized");
 
+        let normalizedUrl = args.url.trim();
+        if (normalizedUrl.startsWith("manual:")) {
+            // Keep manual sources as-is.
+        } else if (!normalizedUrl.startsWith("http")) {
+            normalizedUrl = `https://${normalizedUrl}`;
+        }
+        normalizedUrl = normalizedUrl.replace(/\/+$/, "");
+
         let projectId = args.projectId;
 
         // Use default project if not provided
@@ -55,7 +79,7 @@ export const addDataSource = mutation({
         const existing = await ctx.db
             .query("dataSources")
             .withIndex("by_project_url", (q) =>
-                q.eq("projectId", projectId!).eq("url", args.url)
+                q.eq("projectId", projectId!).eq("url", normalizedUrl)
             )
             .first();
 
@@ -64,7 +88,7 @@ export const addDataSource = mutation({
             : await ctx.db.insert("dataSources", {
                   projectId: projectId!, // Assert exists
                   type: args.type,
-                  url: args.url,
+                  url: normalizedUrl,
                   name: args.name,
                   analysisStatus: "pending",
                   lastAnalyzedAt: undefined,
@@ -83,7 +107,7 @@ export const addDataSource = mutation({
             }
         }
 
-        return { sourceId, projectId: projectId! };
+        return { sourceId, projectId: projectId!, isExisting: Boolean(existing) };
     },
 });
 
@@ -117,3 +141,46 @@ export const updateDataSourceStatus = mutation({
         });
     },
 });
+
+export const remove = mutation({
+    args: { dataSourceId: v.id("dataSources") },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) throw new Error("Unauthorized");
+
+        const dataSource = await ctx.db.get(args.dataSourceId);
+        if (!dataSource) throw new Error("Data source not found");
+
+        const project = await ctx.db.get(dataSource.projectId);
+        if (!project || project.userId !== userId) {
+            throw new Error("Project not found or unauthorized");
+        }
+
+        const updatedSelected = project.dorkingConfig.selectedSourceIds.filter(
+            (id) => id !== args.dataSourceId
+        );
+        await ctx.db.patch(project._id, {
+            dorkingConfig: { selectedSourceIds: updatedSelected },
+        });
+
+        const analyses = await ctx.db
+            .query("analyses")
+            .withIndex("by_dataSource_createdAt", (q) =>
+                q.eq("dataSourceId", args.dataSourceId)
+            )
+            .collect();
+        for (const analysis of analyses) {
+            await ctx.db.delete(analysis._id);
+        }
+
+        const analysisJobs = await ctx.db
+            .query("analysisJobs")
+            .filter((q) => q.eq(q.field("dataSourceId"), args.dataSourceId))
+            .collect();
+        for (const job of analysisJobs) {
+            await ctx.db.delete(job._id);
+        }
+
+        await ctx.db.delete(args.dataSourceId);
+    },
+});

convex/projects.ts

@@ -32,8 +32,17 @@ export const createProject = mutation({
         const userId = await getAuthUserId(ctx);
         if (!userId) throw new Error("Unauthorized");
 
-        // If setting as default, unset other defaults? For now assume handled by UI or logic
+        if (args.isDefault) {
-        // Actually simplicity: just create.
+            const existingDefaults = await ctx.db
+                .query("projects")
+                .withIndex("by_owner", (q) => q.eq("userId", userId))
+                .collect();
+            for (const project of existingDefaults) {
+                if (project.isDefault) {
+                    await ctx.db.patch(project._id, { isDefault: false });
+                }
+            }
+        }
 
         return await ctx.db.insert("projects", {
             userId,
@@ -44,6 +53,36 @@ export const createProject = mutation({
     },
 });
 
+export const updateProject = mutation({
+    args: { projectId: v.id("projects"), name: v.string(), isDefault: v.boolean() },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) throw new Error("Unauthorized");
+
+        const project = await ctx.db.get(args.projectId);
+        if (!project || project.userId !== userId) {
+            throw new Error("Project not found or unauthorized");
+        }
+
+        if (args.isDefault) {
+            const existingDefaults = await ctx.db
+                .query("projects")
+                .withIndex("by_owner", (q) => q.eq("userId", userId))
+                .collect();
+            for (const item of existingDefaults) {
+                if (item.isDefault && item._id !== args.projectId) {
+                    await ctx.db.patch(item._id, { isDefault: false });
+                }
+            }
+        }
+
+        await ctx.db.patch(args.projectId, {
+            name: args.name,
+            isDefault: args.isDefault,
+        });
+    },
+});
+
 export const toggleDataSourceConfig = mutation({
     args: { projectId: v.id("projects"), sourceId: v.id("dataSources"), selected: v.boolean() },
     handler: async (ctx, args) => {

convex/schema.ts

@@ -147,6 +147,50 @@ const schema = defineSchema({
         .index("by_project_status", ["projectId", "status"])
         .index("by_project_createdAt", ["projectId", "createdAt"])
         .index("by_project_url", ["projectId", "url"]),
+    analysisJobs: defineTable({
+        projectId: v.id("projects"),
+        dataSourceId: v.optional(v.id("dataSources")),
+        status: v.union(
+            v.literal("pending"),
+            v.literal("running"),
+            v.literal("completed"),
+            v.literal("failed")
+        ),
+        progress: v.optional(v.number()),
+        stage: v.optional(v.string()),
+        timeline: v.optional(v.array(v.object({
+            key: v.string(),
+            label: v.string(),
+            status: v.union(
+                v.literal("pending"),
+                v.literal("running"),
+                v.literal("completed"),
+                v.literal("failed")
+            ),
+            detail: v.optional(v.string()),
+        }))),
+        error: v.optional(v.string()),
+        createdAt: v.number(),
+        updatedAt: v.number(),
+    })
+        .index("by_project_status", ["projectId", "status"])
+        .index("by_project_createdAt", ["projectId", "createdAt"]),
+    searchJobs: defineTable({
+        projectId: v.id("projects"),
+        status: v.union(
+            v.literal("pending"),
+            v.literal("running"),
+            v.literal("completed"),
+            v.literal("failed")
+        ),
+        config: v.optional(v.any()),
+        progress: v.optional(v.number()),
+        error: v.optional(v.string()),
+        createdAt: v.number(),
+        updatedAt: v.number(),
+    })
+        .index("by_project_status", ["projectId", "status"])
+        .index("by_project_createdAt", ["projectId", "createdAt"]),
 });
 
 export default schema;

convex/searchJobs.ts

@@ -0,0 +1,92 @@
+import { mutation, query } from "./_generated/server";
+import { v } from "convex/values";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const create = mutation({
+  args: {
+    projectId: v.id("projects"),
+    config: v.optional(v.any()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const now = Date.now();
+    return await ctx.db.insert("searchJobs", {
+      projectId: args.projectId,
+      status: "pending",
+      config: args.config,
+      progress: 0,
+      createdAt: now,
+      updatedAt: now,
+    });
+  },
+});
+
+export const update = mutation({
+  args: {
+    jobId: v.id("searchJobs"),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("completed"),
+      v.literal("failed")
+    ),
+    progress: v.optional(v.number()),
+    error: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) throw new Error("Job not found");
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    await ctx.db.patch(args.jobId, {
+      status: args.status,
+      progress: args.progress,
+      error: args.error,
+      updatedAt: Date.now(),
+    });
+  },
+});
+
+export const listByProject = query({
+  args: {
+    projectId: v.id("projects"),
+    status: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return [];
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) return [];
+
+    if (args.status) {
+      return await ctx.db
+        .query("searchJobs")
+        .withIndex("by_project_status", (q) =>
+          q.eq("projectId", args.projectId).eq("status", args.status!)
+        )
+        .order("desc")
+        .collect();
+    }
+
+    return await ctx.db
+      .query("searchJobs")
+      .withIndex("by_project_createdAt", (q) => q.eq("projectId", args.projectId))
+      .order("desc")
+      .collect();
+  },
+});

convex/users.ts

@@ -0,0 +1,11 @@
+import { query } from "./_generated/server";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const getCurrent = query({
+  args: {},
+  handler: async (ctx) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return null;
+    return await ctx.db.get(userId);
+  },
+});

lib/analysis-pipeline.ts

@@ -209,35 +209,68 @@ function generateDorkQueries(keywords: Keyword[], problems: Problem[], useCases:
   return queries
 }
 
-export async function performDeepAnalysis(content: ScrapedContent): Promise<EnhancedProductAnalysis> {
+type AnalysisProgressUpdate = {
+  key: "features" | "competitors" | "keywords" | "problems" | "useCases" | "dorkQueries"
+  status: "running" | "completed"
+  detail?: string
+}
+
+export async function performDeepAnalysis(
+  content: ScrapedContent,
+  onProgress?: (update: AnalysisProgressUpdate) => void | Promise<void>
+): Promise<EnhancedProductAnalysis> {
   console.log('🔍 Starting deep analysis...')
   
   console.log('  📦 Pass 1: Features...')
+  await onProgress?.({ key: "features", status: "running" })
   const features = await extractFeatures(content)
   console.log(`     ✓ ${features.length} features`)
+  await onProgress?.({ key: "features", status: "completed", detail: `${features.length} features` })
   
   console.log('  🏆 Pass 2: Competitors...')
+  await onProgress?.({ key: "competitors", status: "running" })
   const competitors = await identifyCompetitors(content)
   console.log(`     ✓ ${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`)
+  await onProgress?.({
+    key: "competitors",
+    status: "completed",
+    detail: `${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`
+  })
   
   console.log('  🔑 Pass 3: Keywords...')
+  await onProgress?.({ key: "keywords", status: "running" })
   const keywords = await generateKeywords(features, content, competitors)
   console.log(`     ✓ ${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`)
+  await onProgress?.({
+    key: "keywords",
+    status: "completed",
+    detail: `${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`
+  })
   
   console.log('  🎯 Pass 4: Problems...')
+  await onProgress?.({ key: "problems", status: "running" })
   const [problems, personas] = await Promise.all([
     identifyProblems(features, content),
     generatePersonas(content, [])
   ])
   console.log(`     ✓ ${problems.length} problems, ${personas.length} personas`)
+  await onProgress?.({
+    key: "problems",
+    status: "completed",
+    detail: `${problems.length} problems, ${personas.length} personas`
+  })
   
   console.log('  💡 Pass 5: Use cases...')
+  await onProgress?.({ key: "useCases", status: "running" })
   const useCases = await generateUseCases(features, personas, problems)
   console.log(`     ✓ ${useCases.length} use cases`)
+  await onProgress?.({ key: "useCases", status: "completed", detail: `${useCases.length} use cases` })
   
   console.log('  🔎 Pass 6: Dork queries...')
+  await onProgress?.({ key: "dorkQueries", status: "running" })
   const dorkQueries = generateDorkQueries(keywords, problems, useCases, competitors)
   console.log(`     ✓ ${dorkQueries.length} queries`)
+  await onProgress?.({ key: "dorkQueries", status: "completed", detail: `${dorkQueries.length} queries` })
   
   const productName = content.title.split(/[\|\-–—:]/)[0].trim()
   const tagline = content.metaDescription.split('.')[0]

lib/query-generator.ts

@@ -43,7 +43,12 @@ export function generateSearchQueries(
     })
   })
   
-  return sortAndDedupeQueries(queries).slice(0, config.maxResults || 50)
+  const deduped = sortAndDedupeQueries(queries)
+  const limited = deduped.slice(0, config.maxResults || 50)
+  console.info(
+    `[opportunities] queries: generated=${queries.length} deduped=${deduped.length} limited=${limited.length}`
+  )
+  return limited
 }
 
 function buildStrategyQueries(