feat: Implement analysis job tracking with progress timeline and enhanced data source status management.

lib/analysis-pipeline.ts

@@ -209,35 +209,68 @@ function generateDorkQueries(keywords: Keyword[], problems: Problem[], useCases:
   return queries
 }
 
-export async function performDeepAnalysis(content: ScrapedContent): Promise<EnhancedProductAnalysis> {
+type AnalysisProgressUpdate = {
+  key: "features" | "competitors" | "keywords" | "problems" | "useCases" | "dorkQueries"
+  status: "running" | "completed"
+  detail?: string
+}
+
+export async function performDeepAnalysis(
+  content: ScrapedContent,
+  onProgress?: (update: AnalysisProgressUpdate) => void | Promise<void>
+): Promise<EnhancedProductAnalysis> {
   console.log('🔍 Starting deep analysis...')
   
   console.log('  📦 Pass 1: Features...')
+  await onProgress?.({ key: "features", status: "running" })
   const features = await extractFeatures(content)
   console.log(`     ✓ ${features.length} features`)
+  await onProgress?.({ key: "features", status: "completed", detail: `${features.length} features` })
   
   console.log('  🏆 Pass 2: Competitors...')
+  await onProgress?.({ key: "competitors", status: "running" })
   const competitors = await identifyCompetitors(content)
   console.log(`     ✓ ${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`)
+  await onProgress?.({
+    key: "competitors",
+    status: "completed",
+    detail: `${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`
+  })
   
   console.log('  🔑 Pass 3: Keywords...')
+  await onProgress?.({ key: "keywords", status: "running" })
   const keywords = await generateKeywords(features, content, competitors)
   console.log(`     ✓ ${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`)
+  await onProgress?.({
+    key: "keywords",
+    status: "completed",
+    detail: `${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`
+  })
   
   console.log('  🎯 Pass 4: Problems...')
+  await onProgress?.({ key: "problems", status: "running" })
   const [problems, personas] = await Promise.all([
     identifyProblems(features, content),
     generatePersonas(content, [])
   ])
   console.log(`     ✓ ${problems.length} problems, ${personas.length} personas`)
+  await onProgress?.({
+    key: "problems",
+    status: "completed",
+    detail: `${problems.length} problems, ${personas.length} personas`
+  })
   
   console.log('  💡 Pass 5: Use cases...')
+  await onProgress?.({ key: "useCases", status: "running" })
   const useCases = await generateUseCases(features, personas, problems)
   console.log(`     ✓ ${useCases.length} use cases`)
+  await onProgress?.({ key: "useCases", status: "completed", detail: `${useCases.length} use cases` })
   
   console.log('  🔎 Pass 6: Dork queries...')
+  await onProgress?.({ key: "dorkQueries", status: "running" })
   const dorkQueries = generateDorkQueries(keywords, problems, useCases, competitors)
   console.log(`     ✓ ${dorkQueries.length} queries`)
+  await onProgress?.({ key: "dorkQueries", status: "completed", detail: `${dorkQueries.length} queries` })
   
   const productName = content.title.split(/[\|\-–—:]/)[0].trim()
   const tagline = content.metaDescription.split('.')[0]

lib/query-generator.ts

@@ -43,7 +43,12 @@ export function generateSearchQueries(
     })
   })
   
-  return sortAndDedupeQueries(queries).slice(0, config.maxResults || 50)
+  const deduped = sortAndDedupeQueries(queries)
+  const limited = deduped.slice(0, config.maxResults || 50)
+  console.info(
+    `[opportunities] queries: generated=${queries.length} deduped=${deduped.length} limited=${limited.length}`
+  )
+  return limited
 }
 
 function buildStrategyQueries(