a

lib/analysis-pipeline.ts

@@ -15,6 +15,18 @@ const openai = new OpenAI({
   apiKey: process.env.OPENAI_API_KEY
 })
 
+type ProductProfile = {
+  productName: string
+  category: string
+  primaryJTBD: string
+  targetPersona: string
+  scopeBoundary: string
+  nonGoals: string[]
+  differentiators: string[]
+  evidence: { claim: string; snippet: string }[]
+  confidence: number
+}
+
 async function aiGenerate<T>(prompt: string, systemPrompt: string, temperature: number = 0.3): Promise<T> {
   const response = await openai.chat.completions.create({
     model: 'gpt-4o-mini',
@@ -39,7 +51,53 @@ async function aiGenerate<T>(prompt: string, systemPrompt: string, temperature:
   }
 }
 
-async function extractFeatures(content: ScrapedContent): Promise<Feature[]> {
+function buildEvidenceContext(content: ScrapedContent) {
+  return [
+    `Title: ${content.title}`,
+    `Description: ${content.metaDescription}`,
+    `Headings: ${content.headings.slice(0, 20).join('\n')}`,
+    `Feature Lists: ${content.featureList.slice(0, 20).join('\n')}`,
+    `Paragraphs: ${content.paragraphs.slice(0, 12).join('\n\n')}`,
+  ].join('\n\n')
+}
+
+async function extractProductProfile(content: ScrapedContent, extraPrompt?: string): Promise<ProductProfile> {
+  const systemPrompt = `You are a strict product analyst. Only use provided evidence. If uncertain, answer "unknown" and lower confidence. Return JSON only.`
+  const prompt = `Analyze the product using evidence only.
+
+${buildEvidenceContext(content)}
+
+Return JSON:
+{
+  "productName": "...",
+  "category": "...",
+  "primaryJTBD": "...",
+  "targetPersona": "...",
+  "scopeBoundary": "...",
+  "nonGoals": ["..."],
+  "differentiators": ["..."],
+  "evidence": [{"claim": "...", "snippet": "..."}],
+  "confidence": 0.0
+}
+
+Rules:
+- "category" should be a concrete market category, not "software/tool/platform".
+- "scopeBoundary" must state what the product does NOT do.
+- "nonGoals" should be explicit exclusions inferred from evidence.
+- "evidence.snippet" must quote or paraphrase short evidence from the text above.
+${extraPrompt ? `\nUser guidance: ${extraPrompt}` : ""}`
+
+  const result = await aiGenerate<ProductProfile>(prompt, systemPrompt, 0.2)
+  return {
+    ...result,
+    nonGoals: result.nonGoals?.slice(0, 6) ?? [],
+    differentiators: result.differentiators?.slice(0, 6) ?? [],
+    evidence: result.evidence?.slice(0, 6) ?? [],
+    confidence: typeof result.confidence === "number" ? result.confidence : 0.3,
+  }
+}
+
+async function extractFeatures(content: ScrapedContent, extraPrompt?: string): Promise<Feature[]> {
   const systemPrompt = `Extract EVERY feature from website content. Be exhaustive.`
   const prompt = `Extract features from:
 Title: ${content.title}
@@ -49,19 +107,79 @@ Paragraphs: ${content.paragraphs.slice(0, 10).join('\n\n')}
 Feature Lists: ${content.featureList.slice(0, 15).join('\n')}
 
 Return JSON: {"features": [{"name": "...", "description": "...", "benefits": ["..."], "useCases": ["..."]}]}
-Aim for 10-15 features.`
+Aim for 10-15 features.
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
 
   const result = await aiGenerate<{ features: Feature[] }>(prompt, systemPrompt, 0.4)
   return result.features.slice(0, 20)
 }
 
-async function identifyCompetitors(content: ScrapedContent): Promise<Competitor[]> {
-  const systemPrompt = `Identify real, named competitors. Use actual company/product names like "Asana", "Jira", "Monday.com", "Trello", "Notion". Never use generic names like "Competitor A".`
+async function generateCompetitorCandidates(
+  profile: ProductProfile,
+  extraPrompt?: string
+): Promise<{ candidates: { name: string; type: "direct" | "adjacent" | "generic"; rationale: string; confidence: number }[] }> {
+  const systemPrompt = `Generate candidate competitors based only on the product profile. Return JSON only.`
+  const prompt = `Product profile:
+Name: ${profile.productName}
+Category: ${profile.category}
+JTBD: ${profile.primaryJTBD}
+Target persona: ${profile.targetPersona}
+Scope boundary: ${profile.scopeBoundary}
+Non-goals: ${profile.nonGoals.join(", ") || "unknown"}
+Differentiators: ${profile.differentiators.join(", ") || "unknown"}
 
-  const prompt = `Identify 5-6 real competitors for: ${content.title}
-Description: ${content.metaDescription}
+Evidence (for context):
+${profile.evidence.map(e => `- ${e.claim}: ${e.snippet}`).join("\n")}
 
-Return EXACT JSON format:
+Rules:
+- Output real product/company names only.
+- Classify as "direct" if same JTBD + same persona + same category.
+- "adjacent" if overlap partially.
+- "generic" for broad tools people misuse for this (only include if evidence suggests).
+- Avoid broad suites unless the category is that suite.
+
+${extraPrompt ? `User guidance: ${extraPrompt}\n` : ""}
+
+Return JSON:
+{
+  "candidates": [
+    { "name": "Product", "type": "direct|adjacent|generic", "rationale": "...", "confidence": 0.0 }
+  ]
+}`
+
+  return await aiGenerate<{ candidates: { name: string; type: "direct" | "adjacent" | "generic"; rationale: string; confidence: number }[] }>(
+    prompt,
+    systemPrompt,
+    0.2
+  )
+}
+
+async function selectDirectCompetitors(
+  profile: ProductProfile,
+  candidates: { name: string; type: "direct" | "adjacent" | "generic"; rationale: string; confidence: number }[],
+  extraPrompt?: string
+): Promise<Competitor[]> {
+  const systemPrompt = `You are a strict verifier. Only accept direct competitors. Return JSON only.`
+  const prompt = `Product profile:
+Name: ${profile.productName}
+Category: ${profile.category}
+JTBD: ${profile.primaryJTBD}
+Target persona: ${profile.targetPersona}
+Scope boundary: ${profile.scopeBoundary}
+Non-goals: ${profile.nonGoals.join(", ") || "unknown"}
+Differentiators: ${profile.differentiators.join(", ") || "unknown"}
+
+Candidates:
+${candidates.map(c => `- ${c.name} (${c.type}) : ${c.rationale}`).join("\n")}
+
+Rules:
+- Only keep DIRECT competitors (same JTBD + persona + category).
+- Reject "generic" tools unless the category itself is generic.
+- Provide 3-6 competitors. If fewer, include the closest adjacent but label as direct only if truly overlapping.
+
+${extraPrompt ? `User guidance: ${extraPrompt}\n` : ""}
+
+Return JSON:
 {
   "competitors": [
     {
@@ -72,28 +190,35 @@ Return EXACT JSON format:
       "theirWeakness": "Their main weakness"
     }
   ]
+}`
+
+  const result = await aiGenerate<{ competitors: Competitor[] }>(prompt, systemPrompt, 0.2)
+  return result.competitors
+    .map(c => ({
+      ...c,
+      name: c.name.replace(/^Competitor\s+[A-Z]$/i, 'Alternative Solution').replace(/^Generic\s+/i, '')
+    }))
+    .filter(c => c.name.length > 1)
 }
 
-Include: Direct competitors (same space), Big players, Popular alternatives, Tools people misuse for this. Use ONLY real product names.`
-
-  const result = await aiGenerate<{ competitors: Competitor[] }>(prompt, systemPrompt, 0.3)
-  
-  // Validate competitor names aren't generic
-  return result.competitors.map(c => ({
-    ...c,
-    name: c.name.replace(/^Competitor\s+[A-Z]$/i, 'Alternative Solution').replace(/^Generic\s+/i, '')
-  })).filter(c => c.name.length > 1)
-}
-
-async function generateKeywords(features: Feature[], content: ScrapedContent, competitors: Competitor[]): Promise<Keyword[]> {
+async function generateKeywords(
+  features: Feature[],
+  content: ScrapedContent,
+  competitors: Competitor[],
+  extraPrompt?: string
+): Promise<Keyword[]> {
   const systemPrompt = `Generate search-ready phrases users would actually type.`
 
   const featuresText = features.map(f => f.name).join(', ')
-  const competitorNames = competitors.map(c => c.name).filter(n => n.length > 1).join(', ') || 'Jira, Asana, Monday, Trello'
+  const competitorNames = competitors.map(c => c.name).filter(n => n.length > 1).join(', ')
+
+  const differentiatorGuidance = competitorNames
+    ? `Generate 20+ differentiator phrases comparing to: ${competitorNames}`
+    : `If no competitors are provided, do not invent them. Reduce differentiator share to 5% using generic phrases like "alternatives to X category".`
 
   const prompt = `Generate 60-80 search phrases for: ${content.title}
 Features: ${featuresText}
-Competitors: ${competitorNames}
+Competitors: ${competitorNames || "None"}
 
 CRITICAL - Follow this priority:
 1. 60% 2-4 word phrases (e.g., "client onboarding checklist", "bug triage workflow")
@@ -102,7 +227,8 @@ CRITICAL - Follow this priority:
 
 Return JSON: {"keywords": [{"term": "phrase", "type": "differentiator|product|feature|problem|solution|competitor", "searchVolume": "high|medium|low", "intent": "informational|navigational|transactional", "funnel": "awareness|consideration|decision", "emotionalIntensity": "frustrated|curious|ready"}]}
 
-Generate 20+ differentiator phrases comparing to: ${competitorNames}`
+${differentiatorGuidance}
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
 
   const result = await aiGenerate<{ keywords: Keyword[] }>(prompt, systemPrompt, 0.4)
 
@@ -143,36 +269,52 @@ Generate 20+ differentiator phrases comparing to: ${competitorNames}`
   }).slice(0, 80)
 }
 
-async function identifyProblems(features: Feature[], content: ScrapedContent): Promise<Problem[]> {
+async function identifyProblems(
+  features: Feature[],
+  content: ScrapedContent,
+  extraPrompt?: string
+): Promise<Problem[]> {
   const systemPrompt = `Identify problems using JTBD framework.`
   const prompt = `Identify 8-12 problems solved by: ${features.map(f => f.name).join(', ')}
 Content: ${content.rawText.slice(0, 3000)}
 
-Return JSON: {"problems": [{"problem": "...", "severity": "high|medium|low", "currentWorkarounds": ["..."], "emotionalImpact": "...", "searchTerms": ["..."]}]}`
+Return JSON: {"problems": [{"problem": "...", "severity": "high|medium|low", "currentWorkarounds": ["..."], "emotionalImpact": "...", "searchTerms": ["..."]}]}
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
 
   const result = await aiGenerate<{ problems: Problem[] }>(prompt, systemPrompt, 0.4)
   return result.problems
 }
 
-async function generatePersonas(content: ScrapedContent, problems: Problem[]): Promise<Persona[]> {
+async function generatePersonas(
+  content: ScrapedContent,
+  problems: Problem[],
+  extraPrompt?: string
+): Promise<Persona[]> {
   const systemPrompt = `Create diverse user personas with search behavior.`
   const prompt = `Create 4-5 personas for: ${content.title}
 Description: ${content.metaDescription}
 Problems: ${problems.map(p => p.problem).slice(0, 5).join(', ')}
 
-Return JSON: {"personas": [{"name": "Descriptive name", "role": "Job title", "companySize": "e.g. 10-50 employees", "industry": "...", "painPoints": ["..."], "goals": ["..."], "techSavvy": "low|medium|high", "objections": ["..."], "searchBehavior": ["..."]}]}`
+Return JSON: {"personas": [{"name": "Descriptive name", "role": "Job title", "companySize": "e.g. 10-50 employees", "industry": "...", "painPoints": ["..."], "goals": ["..."], "techSavvy": "low|medium|high", "objections": ["..."], "searchBehavior": ["..."]}]}
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
 
   const result = await aiGenerate<{ personas: Persona[] }>(prompt, systemPrompt, 0.5)
   return result.personas
 }
 
-async function generateUseCases(features: Feature[], personas: Persona[], problems: Problem[]): Promise<UseCase[]> {
+async function generateUseCases(
+  features: Feature[],
+  personas: Persona[],
+  problems: Problem[],
+  extraPrompt?: string
+): Promise<UseCase[]> {
   const systemPrompt = `Create JTBD use case scenarios.`
   const prompt = `Create 10 use cases.
 Features: ${features.map(f => f.name).slice(0, 5).join(', ')}
 Problems: ${problems.map(p => p.problem).slice(0, 3).join(', ')}
 
-Return JSON: {"useCases": [{"scenario": "...", "trigger": "...", "emotionalState": "...", "currentWorkflow": ["..."], "desiredOutcome": "...", "alternativeProducts": ["..."], "whyThisProduct": "...", "churnRisk": ["..."]}]}`
+Return JSON: {"useCases": [{"scenario": "...", "trigger": "...", "emotionalState": "...", "currentWorkflow": ["..."], "desiredOutcome": "...", "alternativeProducts": ["..."], "whyThisProduct": "...", "churnRisk": ["..."]}]}
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
 
   const result = await aiGenerate<{ useCases: UseCase[] }>(prompt, systemPrompt, 0.5)
   return result.useCases
@@ -232,18 +374,128 @@ function generateDorkQueries(keywords: Keyword[], problems: Problem[], useCases:
   return queries
 }
 
+async function generateDorkQueriesWithAI(
+  analysis: EnhancedProductAnalysis,
+  extraPrompt?: string
+): Promise<DorkQuery[]> {
+  const systemPrompt = `Generate high-signal search queries for forums. Return JSON only.`
+  const prompt = `Create 40-60 dork queries.
+Product: ${analysis.productName}
+Category: ${analysis.category}
+Positioning: ${analysis.positioning}
+Keywords: ${analysis.keywords.map(k => k.term).slice(0, 25).join(", ")}
+Problems: ${analysis.problemsSolved.map(p => p.problem).slice(0, 10).join(", ")}
+Competitors: ${analysis.competitors.map(c => c.name).slice(0, 10).join(", ")}
+Use cases: ${analysis.useCases.map(u => u.scenario).slice(0, 8).join(", ")}
+
+Rules:
+- Use these platforms only: reddit, hackernews, indiehackers, quora, stackoverflow, twitter.
+- Include intent: looking-for, frustrated, alternative, comparison, problem-solving, tutorial.
+- Prefer query patterns like site:reddit.com "phrase" ...
+
+Return JSON:
+{"dorkQueries": [{"query": "...", "platform": "reddit|hackernews|indiehackers|twitter|quora|stackoverflow", "intent": "looking-for|frustrated|alternative|comparison|problem-solving|tutorial", "priority": "high|medium|low"}]}
+
+${extraPrompt ? `User guidance: ${extraPrompt}` : ""}`
+
+  const result = await aiGenerate<{ dorkQueries: DorkQuery[] }>(prompt, systemPrompt, 0.3)
+  return result.dorkQueries
+}
+
 type AnalysisProgressUpdate = {
   key: "features" | "competitors" | "keywords" | "problems" | "useCases" | "dorkQueries"
   status: "running" | "completed"
   detail?: string
 }
 
+export async function repromptSection(
+  sectionKey: "profile" | "features" | "competitors" | "keywords" | "problems" | "personas" | "useCases" | "dorkQueries",
+  content: ScrapedContent,
+  analysis: EnhancedProductAnalysis,
+  extraPrompt?: string
+): Promise<any> {
+  if (sectionKey === "profile") {
+    const profile = await extractProductProfile(content, extraPrompt);
+    const tagline = content.metaDescription.split(".")[0];
+    const positioning = [
+      profile.primaryJTBD && profile.primaryJTBD !== "unknown"
+        ? profile.primaryJTBD
+        : "",
+      profile.targetPersona && profile.targetPersona !== "unknown"
+        ? `for ${profile.targetPersona}`
+        : "",
+    ].filter(Boolean).join(" ");
+    return {
+      productName: profile.productName && profile.productName !== "unknown" ? profile.productName : analysis.productName,
+      tagline,
+      description: content.metaDescription,
+      category: profile.category && profile.category !== "unknown" ? profile.category : analysis.category,
+      positioning,
+      primaryJTBD: profile.primaryJTBD,
+      targetPersona: profile.targetPersona,
+      scopeBoundary: profile.scopeBoundary,
+      nonGoals: profile.nonGoals,
+      differentiators: profile.differentiators,
+      evidence: profile.evidence,
+      confidence: profile.confidence,
+    };
+  }
+
+  if (sectionKey === "features") {
+    return await extractFeatures(content, extraPrompt);
+  }
+
+  if (sectionKey === "competitors") {
+    const profile = await extractProductProfile(content, extraPrompt);
+    const candidateSet = await generateCompetitorCandidates(profile, extraPrompt);
+    return await selectDirectCompetitors(profile, candidateSet.candidates, extraPrompt);
+  }
+
+  if (sectionKey === "keywords") {
+    const features = analysis.features?.length ? analysis.features : await extractFeatures(content);
+    return await generateKeywords(features, content, analysis.competitors || [], extraPrompt);
+  }
+
+  if (sectionKey === "problems") {
+    const features = analysis.features?.length ? analysis.features : await extractFeatures(content);
+    return await identifyProblems(features, content, extraPrompt);
+  }
+
+  if (sectionKey === "personas") {
+    const problems = analysis.problemsSolved?.length
+      ? analysis.problemsSolved
+      : await identifyProblems(analysis.features || [], content);
+    return await generatePersonas(content, problems, extraPrompt);
+  }
+
+  if (sectionKey === "useCases") {
+    const features = analysis.features?.length ? analysis.features : await extractFeatures(content);
+    const problems = analysis.problemsSolved?.length
+      ? analysis.problemsSolved
+      : await identifyProblems(features, content);
+    const personas = analysis.personas?.length
+      ? analysis.personas
+      : await generatePersonas(content, problems);
+    return await generateUseCases(features, personas, problems, extraPrompt);
+  }
+
+  if (sectionKey === "dorkQueries") {
+    return await generateDorkQueriesWithAI(analysis, extraPrompt);
+  }
+
+  throw new Error(`Unsupported section key: ${sectionKey}`);
+}
+
 export async function performDeepAnalysis(
   content: ScrapedContent,
   onProgress?: (update: AnalysisProgressUpdate) => void | Promise<void>
 ): Promise<EnhancedProductAnalysis> {
   console.log('🔍 Starting deep analysis...')
   
+  console.log('  🧭 Product profiling...')
+  const productProfile = await extractProductProfile(content)
+  console.log(`     ✓ Profiled as ${productProfile.category} for ${productProfile.targetPersona} (conf ${productProfile.confidence})`)
+
   console.log('  📦 Pass 1: Features...')
   await onProgress?.({ key: "features", status: "running" })
   const features = await extractFeatures(content)
@@ -252,7 +504,8 @@ export async function performDeepAnalysis(
   
   console.log('  🏆 Pass 2: Competitors...')
   await onProgress?.({ key: "competitors", status: "running" })
-  const competitors = await identifyCompetitors(content)
+  const candidateSet = await generateCompetitorCandidates(productProfile)
+  const competitors = await selectDirectCompetitors(productProfile, candidateSet.candidates)
   console.log(`     ✓ ${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`)
   await onProgress?.({
     key: "competitors",
@@ -295,13 +548,21 @@ export async function performDeepAnalysis(
   
   const productName = content.title.split(/[\|\-–—:]/)[0].trim()
   const tagline = content.metaDescription.split('.')[0]
+  const positioning = [
+    productProfile.primaryJTBD && productProfile.primaryJTBD !== "unknown"
+      ? productProfile.primaryJTBD
+      : "",
+    productProfile.targetPersona && productProfile.targetPersona !== "unknown"
+      ? `for ${productProfile.targetPersona}`
+      : "",
+  ].filter(Boolean).join(" ")
   
   return {
-    productName,
+    productName: productProfile.productName && productProfile.productName !== "unknown" ? productProfile.productName : productName,
     tagline,
     description: content.metaDescription,
-    category: '',
-    positioning: '',
+    category: productProfile.category && productProfile.category !== "unknown" ? productProfile.category : '',
+    positioning,
     features,
     problemsSolved: problems,
     personas,
@@ -310,6 +571,6 @@ export async function performDeepAnalysis(
     competitors,
     dorkQueries,
     scrapedAt: new Date().toISOString(),
-    analysisVersion: '2.1-optimized'
+    analysisVersion: '2.2-profiled'
   }
 }

lib/query-generator.ts

@@ -6,7 +6,7 @@ import type {
   PlatformId
 } from './types'
 
-export function getDefaultPlatforms(): Record<PlatformId, { name: string; icon: string; rateLimit: number; enabled: boolean }> {
+export function getDefaultPlatforms(): Record<PlatformId, { name: string; icon: string; rateLimit: number; enabled: boolean; searchTemplate: string }> {
   return {
     reddit: {
       name: 'Reddit',
@@ -285,7 +285,7 @@ function buildRecommendationQueries(
   }))
 }
 
-const SITE_OPERATORS: Record<PlatformId, string> = {
+const SITE_OPERATORS: Record<string, string> = {
   reddit: 'site:reddit.com',
   twitter: 'site:twitter.com OR site:x.com',
   hackernews: 'site:news.ycombinator.com',
@@ -295,7 +295,7 @@ const SITE_OPERATORS: Record<PlatformId, string> = {
   linkedin: 'site:linkedin.com',
 }
 
-const DEFAULT_TEMPLATES: Record<PlatformId, string> = {
+const DEFAULT_TEMPLATES: Record<string, string> = {
   reddit: '{site} {term} {intent}',
   twitter: '{site} {term} {intent}',
   hackernews: '{site} ("Ask HN" OR "Show HN") {term} {intent}',
@@ -310,15 +310,18 @@ function applyTemplate(template: string, vars: Record<string, string>): string {
 }
 
 function buildPlatformQuery(
-  platform: { id: PlatformId; searchTemplate?: string },
+  platform: { id: PlatformId; searchTemplate?: string; site?: string },
   term: string,
   intent: string
 ): string {
   const template = (platform.searchTemplate && platform.searchTemplate.trim().length > 0)
     ? platform.searchTemplate
-    : DEFAULT_TEMPLATES[platform.id]
+    : DEFAULT_TEMPLATES[platform.id] ?? '{site} {term} {intent}'
+  const siteOperator = platform.site && platform.site.trim().length > 0
+    ? `site:${platform.site.trim()}`
+    : (SITE_OPERATORS[platform.id] ?? '')
   const raw = applyTemplate(template, {
-    site: SITE_OPERATORS[platform.id],
+    site: siteOperator,
     term,
     intent,
   })

lib/search-executor.ts

@@ -95,9 +95,7 @@ export function scoreOpportunities(
     seen.add(result.url)
     
     const scored = scoreSingleOpportunity(result, analysis)
-    if (scored.relevanceScore >= 0.3) {
-      opportunities.push(scored)
-    }
+    opportunities.push(scored)
   }
   
   return opportunities.sort((a, b) => b.relevanceScore - a.relevanceScore)

lib/types.ts

@@ -1,6 +1,6 @@
 // Enhanced Types for Deep Analysis
 
-export type PlatformId = 'reddit' | 'twitter' | 'hackernews' | 'indiehackers' | 'quora' | 'stackoverflow' | 'linkedin'
+export type PlatformId = string
 
 export type SearchStrategy =
   | 'direct-keywords'
@@ -18,6 +18,8 @@ export interface PlatformConfig {
   enabled: boolean
   searchTemplate: string
   rateLimit: number
+  site?: string
+  custom?: boolean
 }
 
 export interface SearchConfig {