feat: Refine keyword generation logic, enforce Serper API usage, and enhance search query construction with platform-specific templates.

lib/analysis-pipeline.ts

@@ -86,36 +86,59 @@ Include: Direct competitors (same space), Big players, Popular alternatives, Too
 }
 
 async function generateKeywords(features: Feature[], content: ScrapedContent, competitors: Competitor[]): Promise<Keyword[]> {
-  const systemPrompt = `Generate SEO keywords. PRIORITY: 1) Single words, 2) Differentiation keywords showing competitive advantage.`
+  const systemPrompt = `Generate search-ready phrases users would actually type.`
 
   const featuresText = features.map(f => f.name).join(', ')
   const competitorNames = competitors.map(c => c.name).filter(n => n.length > 1).join(', ') || 'Jira, Asana, Monday, Trello'
-  
-  const prompt = `Generate 60-80 keywords for: ${content.title}
+
+  const prompt = `Generate 60-80 search phrases for: ${content.title}
 Features: ${featuresText}
 Competitors: ${competitorNames}
 
 CRITICAL - Follow this priority:
-1. 40% SINGLE WORDS (e.g., "tracker", "automate", "sync", "fast")
-2. 30% DIFFERENTIATION keywords (e.g., "vs-jira", "asana-alternative", "faster", "simpler")
-3. 30% Short 2-word phrases only when needed
+1. 60% 2-4 word phrases (e.g., "client onboarding checklist", "bug triage workflow")
+2. 25% differentiation phrases (e.g., "asana alternative", "faster than jira")
+3. 15% single-word brand terms only (product/competitor names)
 
-Return JSON: {"keywords": [{"term": "word", "type": "differentiator|product|feature|problem|solution|competitor", "searchVolume": "high|medium|low", "intent": "informational|navigational|transactional", "funnel": "awareness|consideration|decision", "emotionalIntensity": "frustrated|curious|ready"}]}
+Return JSON: {"keywords": [{"term": "phrase", "type": "differentiator|product|feature|problem|solution|competitor", "searchVolume": "high|medium|low", "intent": "informational|navigational|transactional", "funnel": "awareness|consideration|decision", "emotionalIntensity": "frustrated|curious|ready"}]}
 
-Generate 20+ differentiator keywords comparing to: ${competitorNames}`
+Generate 20+ differentiator phrases comparing to: ${competitorNames}`
+
+  const result = await aiGenerate<{ keywords: Keyword[] }>(prompt, systemPrompt, 0.4)
+
+  const stopTerms = new Set([
+    'platform',
+    'solution',
+    'tool',
+    'software',
+    'app',
+    'system',
+    'product',
+    'service',
+  ])
+
+  const normalized = result.keywords
+    .map((keyword) => ({ ...keyword, term: keyword.term.trim() }))
+    .filter((keyword) => keyword.term.length > 2)
+    .filter((keyword) => {
+      const words = keyword.term.split(/\s+/).filter(Boolean)
+      if (words.length === 1) {
+        return keyword.type === 'product' || keyword.type === 'competitor' || keyword.type === 'differentiator'
+      }
+      return words.length <= 4
+    })
+    .filter((keyword) => !stopTerms.has(keyword.term.toLowerCase()))
 
-  const result = await aiGenerate<{ keywords: Keyword[] }>(prompt, systemPrompt, 0.5)
-  
   // Sort: differentiators first, then by word count
-  return result.keywords.sort((a, b) => {
+  return normalized.sort((a, b) => {
     const aDiff = a.type === 'differentiator' ? 0 : 1
     const bDiff = b.type === 'differentiator' ? 0 : 1
     if (aDiff !== bDiff) return aDiff - bDiff
-    
+
     const aWords = a.term.split(/\s+/).length
     const bWords = b.term.split(/\s+/).length
     if (aWords !== bWords) return aWords - bWords
-    
+
     return a.term.length - b.term.length
   }).slice(0, 80)
 }
@@ -249,10 +272,8 @@ export async function performDeepAnalysis(
   
   console.log('  🎯 Pass 4: Problems...')
   await onProgress?.({ key: "problems", status: "running" })
-  const [problems, personas] = await Promise.all([
-    identifyProblems(features, content),
-    generatePersonas(content, [])
-  ])
+  const problems = await identifyProblems(features, content)
+  const personas = await generatePersonas(content, problems)
   console.log(`     ✓ ${problems.length} problems, ${personas.length} personas`)
   await onProgress?.({
     key: "problems",

lib/query-generator.ts

@@ -6,25 +6,57 @@ import type {
   PlatformId
 } from './types'
 
-const DEFAULT_PLATFORMS: Record<PlatformId, { name: string; rateLimit: number }> = {
-  reddit: { name: 'Reddit', rateLimit: 30 },
-  twitter: { name: 'X/Twitter', rateLimit: 20 },
-  hackernews: { name: 'Hacker News', rateLimit: 30 },
-  indiehackers: { name: 'Indie Hackers', rateLimit: 20 },
-  quora: { name: 'Quora', rateLimit: 20 },
-  stackoverflow: { name: 'Stack Overflow', rateLimit: 30 },
-  linkedin: { name: 'LinkedIn', rateLimit: 15 }
-}
-
 export function getDefaultPlatforms(): Record<PlatformId, { name: string; icon: string; rateLimit: number; enabled: boolean }> {
   return {
-    reddit: { name: 'Reddit', icon: 'MessageSquare', rateLimit: 30, enabled: true },
-    twitter: { name: 'X/Twitter', icon: 'Twitter', rateLimit: 20, enabled: true },
-    hackernews: { name: 'Hacker News', icon: 'HackerIcon', rateLimit: 30, enabled: true },
-    indiehackers: { name: 'Indie Hackers', icon: 'Users', rateLimit: 20, enabled: false },
-    quora: { name: 'Quora', icon: 'HelpCircle', rateLimit: 20, enabled: false },
-    stackoverflow: { name: 'Stack Overflow', rateLimit: 30, enabled: false },
-    linkedin: { name: 'LinkedIn', rateLimit: 15, enabled: false }
+    reddit: {
+      name: 'Reddit',
+      icon: 'MessageSquare',
+      rateLimit: 30,
+      enabled: true,
+      searchTemplate: '{site} {term} {intent}',
+    },
+    twitter: {
+      name: 'X/Twitter',
+      icon: 'Twitter',
+      rateLimit: 20,
+      enabled: true,
+      searchTemplate: '{site} {term} {intent}',
+    },
+    hackernews: {
+      name: 'Hacker News',
+      icon: 'HackerIcon',
+      rateLimit: 30,
+      enabled: true,
+      searchTemplate: '{site} ("Ask HN" OR "Show HN") {term} {intent}',
+    },
+    indiehackers: {
+      name: 'Indie Hackers',
+      icon: 'Users',
+      rateLimit: 20,
+      enabled: false,
+      searchTemplate: '{site} {term} {intent}',
+    },
+    quora: {
+      name: 'Quora',
+      icon: 'HelpCircle',
+      rateLimit: 20,
+      enabled: false,
+      searchTemplate: '{site} {term} {intent}',
+    },
+    stackoverflow: {
+      name: 'Stack Overflow',
+      icon: 'Filter',
+      rateLimit: 30,
+      enabled: false,
+      searchTemplate: '{site} {term} {intent}',
+    },
+    linkedin: {
+      name: 'LinkedIn',
+      icon: 'Globe',
+      rateLimit: 15,
+      enabled: false,
+      searchTemplate: '{site} {term} {intent}',
+    }
   }
 }
 
@@ -38,7 +70,7 @@ export function generateSearchQueries(
   
   enabledPlatforms.forEach(platform => {
     config.strategies.forEach(strategy => {
-      const strategyQueries = buildStrategyQueries(strategy, analysis, platform.id)
+      const strategyQueries = buildStrategyQueries(strategy, analysis, platform)
       queries.push(...strategyQueries)
     })
   })
@@ -54,7 +86,7 @@ export function generateSearchQueries(
 function buildStrategyQueries(
   strategy: SearchStrategy,
   analysis: EnhancedProductAnalysis,
-  platform: PlatformId
+  platform: { id: PlatformId; searchTemplate?: string }
 ): GeneratedQuery[] {
   
   switch (strategy) {
@@ -84,25 +116,24 @@ function buildStrategyQueries(
   }
 }
 
-function buildDirectKeywordQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
+function buildDirectKeywordQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
   const keywords = analysis.keywords
-    .filter(k => k.type === 'product' || k.type === 'feature')
+    .filter(k => k.type === 'product' || k.type === 'feature' || k.type === 'solution')
     .slice(0, 8)
-  
-  const templates: Record<PlatformId, string[]> = {
-    reddit: ['("looking for" OR "need" OR "recommendation")', '("what do you use" OR "suggestion")'],
-    twitter: ['("looking for" OR "need")', '("any recommendations" OR "suggestions")'],
-    hackernews: ['("Ask HN")'],
-    indiehackers: ['("looking for" OR "need")'],
-    quora: ['("what is the best" OR "how do I choose")'],
-    stackoverflow: ['("best tool for" OR "recommendation")'],
-    linkedin: ['("looking for" OR "seeking")']
-  }
-  
-  return keywords.flatMap(kw => 
-    (templates[platform] || templates.reddit).map(template => ({
-      query: buildPlatformQuery(platform, `"${kw.term}" ${template}`),
-      platform,
+
+  const intentPhrases = [
+    '("looking for" OR "need" OR "recommendation")',
+    '("what do you use" OR "suggestions")',
+    '("any alternatives" OR "best tool")',
+  ]
+
+  return keywords.flatMap((kw) =>
+    intentPhrases.map((intent) => ({
+      query: buildPlatformQuery(platform, quoteTerm(kw.term), intent),
+      platform: platform.id,
       strategy: 'direct-keywords' as SearchStrategy,
       priority: 3,
       expectedIntent: 'looking'
@@ -110,72 +141,99 @@ function buildDirectKeywordQueries(analysis: EnhancedProductAnalysis, platform:
   )
 }
 
-function buildProblemQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
-  const highSeverityProblems = analysis.problemsSolved
-    .filter(p => p.severity === 'high')
-    .slice(0, 5)
-  
-  return highSeverityProblems.flatMap(problem => [
+function buildProblemQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
+  const problems = analysis.problemsSolved
+    .filter((p) => p.severity === 'high' || p.severity === 'medium')
+    .slice(0, 6)
+
+  const intentPhrases = [
+    '("how do I" OR "how to" OR "fix")',
+    '("frustrated" OR "stuck" OR "struggling")',
+    '("best way" OR "recommendation")',
+  ]
+
+  return problems.flatMap((problem) => {
+    const terms = problem.searchTerms && problem.searchTerms.length > 0
+      ? problem.searchTerms.slice(0, 3)
+      : [problem.problem]
+
+    return terms.flatMap((term) =>
+      intentPhrases.map((intent) => ({
+        query: buildPlatformQuery(platform, quoteTerm(term), intent),
+        platform: platform.id,
+        strategy: 'problem-pain' as SearchStrategy,
+        priority: 5,
+        expectedIntent: 'frustrated',
+      }))
+    )
+  })
+}
+
+function buildCompetitorQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
+  const competitors = analysis.competitors
+    .filter((comp) => comp.name && comp.name.length > 2)
+    .slice(0, 6)
+
+  const switchIntent = '("switching" OR "moving from" OR "left" OR "canceling")'
+  const compareIntent = '("vs" OR "versus" OR "compared to" OR "better than")'
+  const painIntent = '("frustrated" OR "issues with" OR "problems with")'
+
+  return competitors.flatMap((comp) => [
     {
-      query: buildPlatformQuery(platform, `"${problem.problem}" ("how to" OR "fix" OR "solve")`),
-      platform,
-      strategy: 'problem-pain',
+      query: buildPlatformQuery(platform, quoteTerm(comp.name), switchIntent),
+      platform: platform.id,
+      strategy: 'competitor-alternative' as SearchStrategy,
       priority: 5,
-      expectedIntent: 'frustrated'
+      expectedIntent: 'comparing',
     },
-    ...problem.searchTerms.slice(0, 2).map(term => ({
-      query: buildPlatformQuery(platform, `"${term}"`),
-      platform,
-      strategy: 'problem-pain',
+    {
+      query: buildPlatformQuery(platform, quoteTerm(comp.name), compareIntent),
+      platform: platform.id,
+      strategy: 'competitor-alternative' as SearchStrategy,
       priority: 4,
-      expectedIntent: 'frustrated'
-    }))
+      expectedIntent: 'comparing',
+    },
+    {
+      query: buildPlatformQuery(platform, quoteTerm(comp.name), painIntent),
+      platform: platform.id,
+      strategy: 'competitor-alternative' as SearchStrategy,
+      priority: 5,
+      expectedIntent: 'frustrated',
+    },
   ])
 }
 
-function buildCompetitorQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
-  const competitors = analysis.competitors.slice(0, 5)
-  
-  return competitors.flatMap(comp => [
-    {
-      query: buildPlatformQuery(platform, `"${comp.name}" ("alternative" OR "switching from" OR "moving away")`),
-      platform,
-      strategy: 'competitor-alternative',
-      priority: 5,
-      expectedIntent: 'comparing'
-    },
-    {
-      query: buildPlatformQuery(platform, `"${comp.name}" ("vs" OR "versus" OR "compared to" OR "better than")`),
-      platform,
-      strategy: 'competitor-alternative',
-      priority: 4,
-      expectedIntent: 'comparing'
-    },
-    {
-      query: buildPlatformQuery(platform, `"${comp.name}" ("frustrated" OR "disappointed" OR "problems with")`),
-      platform,
-      strategy: 'competitor-alternative',
-      priority: 5,
-      expectedIntent: 'frustrated'
-    }
-  ])
-}
-
-function buildHowToQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
+function buildHowToQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
   const keywords = analysis.keywords
     .filter(k => k.type === 'feature' || k.type === 'solution')
     .slice(0, 6)
   
   return keywords.map(kw => ({
-    query: buildPlatformQuery(platform, `"how to" "${kw.term}" ("tutorial" OR "guide" OR "help")`),
-    platform,
+    query: buildPlatformQuery(
+      platform,
+      `"how to" ${quoteTerm(kw.term)}`,
+      '("tutorial" OR "guide" OR "help")'
+    ),
+    platform: platform.id,
     strategy: 'how-to',
     priority: 2,
     expectedIntent: 'learning'
   }))
 }
 
-function buildEmotionalQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
+function buildEmotionalQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
   const keywords = analysis.keywords
     .filter(k => k.type === 'product' || k.type === 'problem')
     .slice(0, 5)
@@ -184,8 +242,8 @@ function buildEmotionalQueries(analysis: EnhancedProductAnalysis, platform: Plat
   
   return keywords.flatMap(kw => 
     emotionalTerms.slice(0, 3).map(term => ({
-      query: buildPlatformQuery(platform, `"${kw.term}" ${term}`),
-      platform,
+      query: buildPlatformQuery(platform, quoteTerm(kw.term), term),
+      platform: platform.id,
       strategy: 'emotional-frustrated',
       priority: 4,
       expectedIntent: 'frustrated'
@@ -193,46 +251,83 @@ function buildEmotionalQueries(analysis: EnhancedProductAnalysis, platform: Plat
   )
 }
 
-function buildComparisonQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
+function buildComparisonQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
   const keywords = analysis.keywords
     .filter(k => k.type === 'product' || k.type === 'differentiator')
     .slice(0, 4)
   
   return keywords.map(kw => ({
-    query: buildPlatformQuery(platform, `"${kw.term}" ("vs" OR "versus" OR "or" OR "comparison")`),
-    platform,
+    query: buildPlatformQuery(platform, quoteTerm(kw.term), '("vs" OR "versus" OR "or" OR "comparison")'),
+    platform: platform.id,
     strategy: 'comparison',
     priority: 3,
     expectedIntent: 'comparing'
   }))
 }
 
-function buildRecommendationQueries(analysis: EnhancedProductAnalysis, platform: PlatformId): GeneratedQuery[] {
+function buildRecommendationQueries(
+  analysis: EnhancedProductAnalysis,
+  platform: { id: PlatformId; searchTemplate?: string }
+): GeneratedQuery[] {
   const keywords = analysis.keywords
     .filter(k => k.type === 'product' || k.type === 'feature')
     .slice(0, 5)
   
   return keywords.map(kw => ({
-    query: buildPlatformQuery(platform, `("what do you use" OR "recommendation" OR "suggest") "${kw.term}"`),
-    platform,
+    query: buildPlatformQuery(platform, quoteTerm(kw.term), '("what do you use" OR "recommendation" OR "suggest")'),
+    platform: platform.id,
     strategy: 'recommendation',
     priority: 3,
     expectedIntent: 'recommending'
   }))
 }
 
-function buildPlatformQuery(platform: PlatformId, query: string): string {
-  const siteOperators: Record<PlatformId, string> = {
-    reddit: 'site:reddit.com',
-    twitter: 'site:twitter.com OR site:x.com',
-    hackernews: 'site:news.ycombinator.com',
-    indiehackers: 'site:indiehackers.com',
-    quora: 'site:quora.com',
-    stackoverflow: 'site:stackoverflow.com',
-    linkedin: 'site:linkedin.com'
-  }
-  
-  return `${siteOperators[platform]} ${query}`
+const SITE_OPERATORS: Record<PlatformId, string> = {
+  reddit: 'site:reddit.com',
+  twitter: 'site:twitter.com OR site:x.com',
+  hackernews: 'site:news.ycombinator.com',
+  indiehackers: 'site:indiehackers.com',
+  quora: 'site:quora.com',
+  stackoverflow: 'site:stackoverflow.com',
+  linkedin: 'site:linkedin.com',
+}
+
+const DEFAULT_TEMPLATES: Record<PlatformId, string> = {
+  reddit: '{site} {term} {intent}',
+  twitter: '{site} {term} {intent}',
+  hackernews: '{site} ("Ask HN" OR "Show HN") {term} {intent}',
+  indiehackers: '{site} {term} {intent}',
+  quora: '{site} {term} {intent}',
+  stackoverflow: '{site} {term} {intent}',
+  linkedin: '{site} {term} {intent}',
+}
+
+function applyTemplate(template: string, vars: Record<string, string>): string {
+  return template.replace(/\{(\w+)\}/g, (_match, key) => vars[key] || '')
+}
+
+function buildPlatformQuery(
+  platform: { id: PlatformId; searchTemplate?: string },
+  term: string,
+  intent: string
+): string {
+  const template = (platform.searchTemplate && platform.searchTemplate.trim().length > 0)
+    ? platform.searchTemplate
+    : DEFAULT_TEMPLATES[platform.id]
+  const raw = applyTemplate(template, {
+    site: SITE_OPERATORS[platform.id],
+    term,
+    intent,
+  })
+  return raw.replace(/\s+/g, ' ').trim()
+}
+
+function quoteTerm(term: string): string {
+  const cleaned = term.replace(/["]+/g, '').trim()
+  return cleaned.includes(' ') ? `"${cleaned}"` : `"${cleaned}"`
 }
 
 function sortAndDedupeQueries(queries: GeneratedQuery[]): GeneratedQuery[] {

lib/search-executor.ts

@@ -46,13 +46,11 @@ export async function executeSearches(
 }
 
 async function executeSingleSearch(query: GeneratedQuery): Promise<SearchResult[]> {
-  // Use Serper API if available
-  if (process.env.SERPER_API_KEY) {
-    return searchWithSerper(query)
+  if (!process.env.SERPER_API_KEY) {
+    throw new Error('SERPER_API_KEY is not configured.')
   }
-  
-  // Fallback to direct scraping (less reliable)
-  return searchDirect(query)
+
+  return searchWithSerper(query)
 }
 
 async function searchWithSerper(query: GeneratedQuery): Promise<SearchResult[]> {
@@ -85,46 +83,6 @@ async function searchWithSerper(query: GeneratedQuery): Promise<SearchResult[]>
   }))
 }
 
-async function searchDirect(query: GeneratedQuery): Promise<SearchResult[]> {
-  // Simple direct search as fallback
-  const encodedQuery = encodeURIComponent(query.query)
-  const url = `https://www.google.com/search?q=${encodedQuery}&num=5`
-  
-  try {
-    const response = await fetch(url, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-      }
-    })
-    
-    const html = await response.text()
-    const results: SearchResult[] = []
-    
-    // Basic regex parsing
-    const resultBlocks = html.match(/<div class="g"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/g) || []
-    
-    for (const block of resultBlocks.slice(0, 5)) {
-      const titleMatch = block.match(/<h3[^>]*>(.*?)<\/h3>/)
-      const linkMatch = block.match(/<a href="([^"]+)"/)
-      const snippetMatch = block.match(/<div class="VwiC3b[^"]*"[^>]*>(.*?)<\/div>/)
-      
-      if (titleMatch && linkMatch) {
-        results.push({
-          title: titleMatch[1].replace(/<[^>]+>/g, ''),
-          url: linkMatch[1],
-          snippet: snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '') : '',
-          platform: query.platform
-        })
-      }
-    }
-    
-    return results
-  } catch (err) {
-    console.error('Direct search failed:', err)
-    return []
-  }
-}
-
 export function scoreOpportunities(
   results: SearchResult[],
   analysis: EnhancedProductAnalysis

lib/types.ts

@@ -23,7 +23,6 @@ export interface PlatformConfig {
 export interface SearchConfig {
   platforms: PlatformConfig[]
   strategies: SearchStrategy[]
-  intensity: 'broad' | 'balanced' | 'targeted'
   maxResults: number
   timeFilter?: 'past-day' | 'past-week' | 'past-month' | 'past-year' | 'all'
 }