dates

2026-02-04 12:51:41 +00:00
parent 4fdbfb0fb3
commit f1e13f87f6
19 changed files with 722 additions and 67 deletions
--- a/lib/analysis-pipeline.ts
+++ b/lib/analysis-pipeline.ts
@@ -10,6 +10,7 @@ import type {
  Competitor,
  DorkQuery 
 } from './types'
+import { logServer } from "@/lib/server-logger";

 const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY
@@ -46,7 +47,13 @@ async function aiGenerate<T>(prompt: string, systemPrompt: string, temperature:
  
  try { return JSON.parse(jsonStr) as T }
  catch (e) {
-    console.error('Failed to parse JSON:', jsonStr.substring(0, 200))
+    await logServer({
+      level: "error",
+      message: "Failed to parse JSON from AI response",
+      labels: ["analysis-pipeline", "ai", "error"],
+      payload: { sample: jsonStr.substring(0, 200) },
+      source: "lib/analysis-pipeline",
+    });
    throw new Error('Invalid JSON response from AI')
  }
 }
@@ -490,60 +497,149 @@ export async function performDeepAnalysis(
  content: ScrapedContent,
  onProgress?: (update: AnalysisProgressUpdate) => void | Promise<void>
 ): Promise<EnhancedProductAnalysis> {
-  console.log('🔍 Starting deep analysis...')
+  await logServer({
+    level: "info",
+    message: "Starting deep analysis",
+    labels: ["analysis-pipeline", "start"],
+    source: "lib/analysis-pipeline",
+  });
  
-  console.log('  🧭 Product profiling...')
+  await logServer({
+    level: "info",
+    message: "Product profiling",
+    labels: ["analysis-pipeline", "profile"],
+    source: "lib/analysis-pipeline",
+  });
  const productProfile = await extractProductProfile(content)
-  console.log(`     ✓ Profiled as ${productProfile.category} for ${productProfile.targetPersona} (conf ${productProfile.confidence})`)
+  await logServer({
+    level: "info",
+    message: "Product profile complete",
+    labels: ["analysis-pipeline", "profile"],
+    payload: {
+      category: productProfile.category,
+      targetPersona: productProfile.targetPersona,
+      confidence: productProfile.confidence,
+    },
+    source: "lib/analysis-pipeline",
+  });

-  console.log('  📦 Pass 1: Features...')
+  await logServer({
+    level: "info",
+    message: "Pass 1: Features",
+    labels: ["analysis-pipeline", "features"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "features", status: "running" })
  const features = await extractFeatures(content)
-  console.log(`     ✓ ${features.length} features`)
+  await logServer({
+    level: "info",
+    message: "Features extracted",
+    labels: ["analysis-pipeline", "features"],
+    payload: { count: features.length },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "features", status: "completed", detail: `${features.length} features` })
  
-  console.log('  🏆 Pass 2: Competitors...')
+  await logServer({
+    level: "info",
+    message: "Pass 2: Competitors",
+    labels: ["analysis-pipeline", "competitors"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "competitors", status: "running" })
  const candidateSet = await generateCompetitorCandidates(productProfile)
  const competitors = await selectDirectCompetitors(productProfile, candidateSet.candidates)
-  console.log(`     ✓ ${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`)
+  await logServer({
+    level: "info",
+    message: "Competitors extracted",
+    labels: ["analysis-pipeline", "competitors"],
+    payload: { count: competitors.length, names: competitors.map((c) => c.name) },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({
    key: "competitors",
    status: "completed",
    detail: `${competitors.length} competitors: ${competitors.map(c => c.name).join(', ')}`
  })
  
-  console.log('  🔑 Pass 3: Keywords...')
+  await logServer({
+    level: "info",
+    message: "Pass 3: Keywords",
+    labels: ["analysis-pipeline", "keywords"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "keywords", status: "running" })
  const keywords = await generateKeywords(features, content, competitors)
-  console.log(`     ✓ ${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`)
+  await logServer({
+    level: "info",
+    message: "Keywords extracted",
+    labels: ["analysis-pipeline", "keywords"],
+    payload: {
+      count: keywords.length,
+      differentiators: keywords.filter((k) => k.type === "differentiator").length,
+    },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({
    key: "keywords",
    status: "completed",
    detail: `${keywords.length} keywords (${keywords.filter(k => k.type === 'differentiator').length} differentiators)`
  })
  
-  console.log('  🎯 Pass 4: Problems...')
+  await logServer({
+    level: "info",
+    message: "Pass 4: Problems",
+    labels: ["analysis-pipeline", "problems"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "problems", status: "running" })
  const problems = await identifyProblems(features, content)
  const personas = await generatePersonas(content, problems)
-  console.log(`     ✓ ${problems.length} problems, ${personas.length} personas`)
+  await logServer({
+    level: "info",
+    message: "Problems and personas extracted",
+    labels: ["analysis-pipeline", "problems"],
+    payload: { problems: problems.length, personas: personas.length },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({
    key: "problems",
    status: "completed",
    detail: `${problems.length} problems, ${personas.length} personas`
  })
  
-  console.log('  💡 Pass 5: Use cases...')
+  await logServer({
+    level: "info",
+    message: "Pass 5: Use cases",
+    labels: ["analysis-pipeline", "use-cases"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "useCases", status: "running" })
  const useCases = await generateUseCases(features, personas, problems)
-  console.log(`     ✓ ${useCases.length} use cases`)
+  await logServer({
+    level: "info",
+    message: "Use cases extracted",
+    labels: ["analysis-pipeline", "use-cases"],
+    payload: { count: useCases.length },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "useCases", status: "completed", detail: `${useCases.length} use cases` })
  
-  console.log('  🔎 Pass 6: Dork queries...')
+  await logServer({
+    level: "info",
+    message: "Pass 6: Dork queries",
+    labels: ["analysis-pipeline", "dork-queries"],
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "dorkQueries", status: "running" })
  const dorkQueries = generateDorkQueries(keywords, problems, useCases, competitors)
-  console.log(`     ✓ ${dorkQueries.length} queries`)
+  await logServer({
+    level: "info",
+    message: "Dork queries extracted",
+    labels: ["analysis-pipeline", "dork-queries"],
+    payload: { count: dorkQueries.length },
+    source: "lib/analysis-pipeline",
+  });
  await onProgress?.({ key: "dorkQueries", status: "completed", detail: `${dorkQueries.length} queries` })
  
  const productName = content.title.split(/[\|\-–—:]/)[0].trim()
--- a/lib/openai.ts
+++ b/lib/openai.ts
@@ -1,5 +1,6 @@
 import OpenAI from 'openai'
 import type { ProductAnalysis, ScrapedContent, Opportunity } from './types'
+import { logServer } from "@/lib/server-logger";

 const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY
@@ -111,7 +112,13 @@ export async function findOpportunities(analysis: ProductAnalysis): Promise<Oppo
        }
      }
    } catch (e) {
-      console.error('Search failed:', e)
+      await logServer({
+        level: "error",
+        message: "Search failed",
+        labels: ["openai", "search", "error"],
+        payload: { error: String(e) },
+        source: "lib/openai",
+      });
    }
  }

@@ -213,7 +220,13 @@ async function searchGoogle(query: string, num: number): Promise<SearchResult[]>
      const results = await searchSerper(query, num)
      if (results.length > 0) return results
    } catch (e) {
-      console.error('Serper search failed:', e)
+      await logServer({
+        level: "error",
+        message: "Serper search failed",
+        labels: ["openai", "serper", "error"],
+        payload: { error: String(e) },
+        source: "lib/openai",
+      });
    }
  }

@@ -234,6 +247,13 @@ async function searchSerper(query: string, num: number): Promise<SearchResult[]>
  if (!response.ok) throw new Error('Serper API error')

  const data = await response.json()
+  await logServer({
+    level: "info",
+    message: "Serper response received",
+    labels: ["openai", "serper", "response"],
+    payload: { query, num, data },
+    source: "lib/openai",
+  });
  return (data.organic || []).map((r: any) => ({
    title: r.title,
    url: r.link,
--- a/lib/query-generator.ts
+++ b/lib/query-generator.ts
@@ -5,6 +5,7 @@ import type {
  SearchStrategy,
  PlatformId
 } from './types'
+import { logServer } from "@/lib/server-logger";

 export function getDefaultPlatforms(): Record<PlatformId, { name: string; icon: string; rateLimit: number; enabled: boolean; searchTemplate: string }> {
  return {
@@ -77,9 +78,13 @@ export function generateSearchQueries(
  
  const deduped = sortAndDedupeQueries(queries)
  const limited = deduped.slice(0, config.maxResults || 50)
-  console.info(
-    `[opportunities] queries: generated=${queries.length} deduped=${deduped.length} limited=${limited.length}`
-  )
+  void logServer({
+    level: "info",
+    message: "Search queries generated",
+    labels: ["query-generator", "queries"],
+    payload: { generated: queries.length, deduped: deduped.length, limited: limited.length },
+    source: "lib/query-generator",
+  });
  return limited
 }

--- a/lib/scraper.ts
+++ b/lib/scraper.ts
@@ -1,5 +1,6 @@
 import puppeteer from 'puppeteer'
 import type { ScrapedContent } from './types'
+import { logServer } from "@/lib/server-logger";

 export class ScrapingError extends Error {
  constructor(message: string, public code: string) {
@@ -94,7 +95,13 @@ export async function scrapeWebsite(url: string): Promise<ScrapedContent> {
    }

  } catch (error: any) {
-    console.error('Scraping error:', error)
+    await logServer({
+      level: "error",
+      message: "Scraping error",
+      labels: ["scraper", "error"],
+      payload: { url: validatedUrl, error: String(error) },
+      source: "lib/scraper",
+    });
    
    if (error.message?.includes('ERR_NAME_NOT_RESOLVED') || error.message?.includes('net::ERR')) {
      throw new ScrapingError(
--- a/lib/search-executor.ts
+++ b/lib/search-executor.ts
@@ -1,4 +1,6 @@
 import type { GeneratedQuery, Opportunity, EnhancedProductAnalysis } from './types'
+import { logServer } from "@/lib/server-logger";
+import { appendSerperAgeModifiers, SerperAgeFilter } from "@/lib/serper-date-filters";

 interface SearchResult {
  title: string
@@ -10,6 +12,7 @@ interface SearchResult {

 export async function executeSearches(
  queries: GeneratedQuery[],
+  filters?: SerperAgeFilter,
  onProgress?: (progress: { current: number; total: number; platform: string }) => void
 ): Promise<SearchResult[]> {
  const results: SearchResult[] = []
@@ -24,11 +27,17 @@ export async function executeSearches(
  let completed = 0
  
  for (const [platform, platformQueries] of byPlatform) {
-    console.log(`Searching ${platform}: ${platformQueries.length} queries`)
+    await logServer({
+      level: "info",
+      message: "Searching platform",
+      labels: ["search-executor", "platform", "start"],
+      payload: { platform, queries: platformQueries.length },
+      source: "lib/search-executor",
+    });
    
    for (const query of platformQueries) {
      try {
-        const searchResults = await executeSingleSearch(query)
+        const searchResults = await executeSingleSearch(query, filters)
        results.push(...searchResults)
        
        completed++
@@ -37,7 +46,13 @@ export async function executeSearches(
        // Rate limiting - 1 second between requests
        await delay(1000)
      } catch (err) {
-        console.error(`Search failed for ${platform}:`, err)
+        await logServer({
+          level: "error",
+          message: "Search failed for platform",
+          labels: ["search-executor", "platform", "error"],
+          payload: { platform, error: String(err) },
+          source: "lib/search-executor",
+        });
      }
    }
  }
@@ -45,15 +60,15 @@ export async function executeSearches(
  return results
 }

-async function executeSingleSearch(query: GeneratedQuery): Promise<SearchResult[]> {
+async function executeSingleSearch(query: GeneratedQuery, filters?: SerperAgeFilter): Promise<SearchResult[]> {
  if (!process.env.SERPER_API_KEY) {
    throw new Error('SERPER_API_KEY is not configured.')
  }

-  return searchWithSerper(query)
+  return searchWithSerper(query, filters)
 }

-async function searchWithSerper(query: GeneratedQuery): Promise<SearchResult[]> {
+async function searchWithSerper(query: GeneratedQuery, filters?: SerperAgeFilter): Promise<SearchResult[]> {
  const response = await fetch('https://google.serper.dev/search', {
    method: 'POST',
    headers: {
@@ -61,7 +76,7 @@ async function searchWithSerper(query: GeneratedQuery): Promise<SearchResult[]>
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ 
-      q: query.query, 
+      q: appendSerperAgeModifiers(query.query, filters),
      num: 5,
      gl: 'us',
      hl: 'en'
@@ -73,6 +88,13 @@ async function searchWithSerper(query: GeneratedQuery): Promise<SearchResult[]>
  }

  const data = await response.json()
+  await logServer({
+    level: "info",
+    message: "Serper response received",
+    labels: ["search-executor", "serper", "response"],
+    payload: { query: query.query, platform: query.platform, data },
+    source: "lib/search-executor",
+  });
  
  return (data.organic || []).map((r: any) => ({
    title: r.title,
--- a/lib/serper-date-filters.ts
+++ b/lib/serper-date-filters.ts
@@ -0,0 +1,28 @@
+export type SerperAgeFilter = {
+  maxAgeDays?: number
+  minAgeDays?: number
+}
+
+const normalizeDays = (value?: number) => {
+  if (typeof value !== 'number') return undefined
+  if (!Number.isFinite(value)) return undefined
+  if (value <= 0) return undefined
+  return Math.floor(value)
+}
+
+export function appendSerperAgeModifiers(query: string, filters?: SerperAgeFilter): string {
+  if (!filters) return query
+  const modifiers: string[] = []
+  const normalizedMax = normalizeDays(filters.maxAgeDays)
+  const normalizedMin = normalizeDays(filters.minAgeDays)
+
+  if (typeof normalizedMax === 'number') {
+    modifiers.push(`newer_than:${normalizedMax}d`)
+  }
+  if (typeof normalizedMin === 'number') {
+    modifiers.push(`older_than:${normalizedMin}d`)
+  }
+
+  if (modifiers.length === 0) return query
+  return `${query} ${modifiers.join(' ')}`
+}
--- a/lib/server-logger.ts
+++ b/lib/server-logger.ts
@@ -0,0 +1,57 @@
+import { fetchMutation } from "convex/nextjs";
+import { api } from "@/convex/_generated/api";
+import type { Id } from "@/convex/_generated/dataModel";
+
+type LogLevel = "debug" | "info" | "warn" | "error";
+
+type LogParams = {
+  level: LogLevel;
+  message: string;
+  labels: string[];
+  payload?: unknown;
+  source?: string;
+  requestId?: string;
+  projectId?: Id<"projects">;
+  token?: string;
+};
+
+function writeConsole(level: LogLevel, message: string, payload?: unknown) {
+  const tag = `[${level}]`;
+  if (payload === undefined) {
+    if (level === "error") {
+      console.error(tag, message);
+      return;
+    }
+    if (level === "warn") {
+      console.warn(tag, message);
+      return;
+    }
+    console.log(tag, message);
+    return;
+  }
+  if (level === "error") {
+    console.error(tag, message, payload);
+    return;
+  }
+  if (level === "warn") {
+    console.warn(tag, message, payload);
+    return;
+  }
+  console.log(tag, message, payload);
+}
+
+export async function logServer({
+  token,
+  ...args
+}: LogParams): Promise<void> {
+  writeConsole(args.level, args.message, args.payload);
+  try {
+    if (token) {
+      await fetchMutation(api.logs.createLog, args, { token });
+      return;
+    }
+    await fetchMutation(api.logs.createLog, args);
+  } catch (error) {
+    console.error("[logger] Failed to write log", error);
+  }
+}
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -26,6 +26,8 @@ export interface SearchConfig {
  platforms: PlatformConfig[]
  strategies: SearchStrategy[]
  maxResults: number
+  minAgeDays?: number
+  maxAgeDays?: number
  timeFilter?: 'past-day' | 'past-week' | 'past-month' | 'past-year' | 'all'
 }