import { NextRequest, NextResponse } from 'next/server' import { convexAuthNextjsToken, isAuthenticatedNextjs } from "@convex-dev/auth/nextjs/server"; import { fetchMutation, fetchQuery } from "convex/nextjs"; import { api } from "@/convex/_generated/api"; import { z } from 'zod' import { scrapeWebsite, ScrapingError } from '@/lib/scraper' import { performDeepAnalysis } from '@/lib/analysis-pipeline' import { logServer } from "@/lib/server-logger"; const bodySchema = z.object({ url: z.string().min(1), jobId: z.optional(z.string()) }) export async function POST(request: NextRequest) { let jobId: string | undefined let timeline: { key: string label: string status: "pending" | "running" | "completed" | "failed" detail?: string }[] = [] try { const requestId = request.headers.get("x-request-id") ?? undefined; if (!(await isAuthenticatedNextjs())) { const redirectUrl = new URL("/auth", request.url); const referer = request.headers.get("referer"); const nextPath = referer ? new URL(referer).pathname + new URL(referer).search : "/"; redirectUrl.searchParams.set("next", nextPath); return NextResponse.redirect(redirectUrl); } const body = await request.json() const parsed = bodySchema.parse(body) const { url } = parsed jobId = parsed.jobId const token = await convexAuthNextjsToken(); timeline = [ { key: "scrape", label: "Scrape website", status: "pending" }, { key: "features", label: "Pass 1: Features", status: "pending" }, { key: "competitors", label: "Pass 2: Competitors", status: "pending" }, { key: "keywords", label: "Pass 3: Keywords", status: "pending" }, { key: "problems", label: "Pass 4: Problems & Personas", status: "pending" }, { key: "useCases", label: "Pass 5: Use cases", status: "pending" }, { key: "dorkQueries", label: "Pass 6: Dork queries", status: "pending" }, { key: "finalize", label: "Finalize analysis", status: "pending" }, ] const updateTimeline = async ({ key, status, detail, progress, finalStatus, }: { key: string status: "pending" | "running" | "completed" | "failed" detail?: string progress?: number finalStatus?: "running" | "completed" | "failed" }) => { if (!jobId) return timeline = timeline.map((item) => item.key === key ? { ...item, status, detail: detail ?? item.detail } : item ) await fetchMutation( api.analysisJobs.update, { jobId: jobId as any, status: finalStatus || "running", progress, stage: key, timeline, }, { token } ) } if (jobId) { await updateTimeline({ key: "scrape", status: "running", progress: 10 }) } if (!process.env.OPENAI_API_KEY) { if (jobId) { await fetchMutation( api.analysisJobs.update, { jobId: jobId as any, status: "failed", error: "OpenAI API key not configured", timeline: timeline.map((item) => item.status === "running" ? { ...item, status: "failed" } : item ), }, { token } ); } return NextResponse.json( { error: 'OpenAI API key not configured' }, { status: 500 } ) } await logServer({ level: "info", message: "Scraping website", labels: ["api", "analyze", "scrape"], payload: { url }, requestId, source: "api/analyze", }); const scrapedContent = await scrapeWebsite(url) await logServer({ level: "info", message: "Scrape complete", labels: ["api", "analyze", "scrape"], payload: { headings: scrapedContent.headings.length, paragraphs: scrapedContent.paragraphs.length, }, requestId, source: "api/analyze", }); if (jobId) { await updateTimeline({ key: "scrape", status: "completed", detail: `${scrapedContent.headings.length} headings, ${scrapedContent.paragraphs.length} paragraphs`, progress: 20, }) } await logServer({ level: "info", message: "Starting enhanced analysis", labels: ["api", "analyze", "analysis"], requestId, source: "api/analyze", }); const progressMap: Record = { features: 35, competitors: 50, keywords: 65, problems: 78, useCases: 88, dorkQueries: 95, } const analysis = await performDeepAnalysis(scrapedContent, async (update) => { await updateTimeline({ key: update.key, status: update.status, detail: update.detail, progress: progressMap[update.key] ?? 80, }) }) await logServer({ level: "info", message: "Analysis complete", labels: ["api", "analyze", "analysis"], payload: { features: analysis.features.length, keywords: analysis.keywords.length, dorkQueries: analysis.dorkQueries.length, }, requestId, source: "api/analyze", }); if (jobId) { await updateTimeline({ key: "finalize", status: "running", progress: 98, }) } if (jobId) { await updateTimeline({ key: "finalize", status: "completed", progress: 100, finalStatus: "completed", }) } let persisted = false if (jobId) { try { const job = await fetchQuery( api.analysisJobs.getById, { jobId: jobId as any }, { token } ) if (job?.dataSourceId && job.projectId) { const existing = await fetchQuery( api.analyses.getLatestByDataSource, { dataSourceId: job.dataSourceId as any }, { token } ) if (!existing || existing.createdAt < job.createdAt) { await fetchMutation( api.analyses.createAnalysis, { projectId: job.projectId as any, dataSourceId: job.dataSourceId as any, analysis, }, { token } ) } await fetchMutation( api.dataSources.updateDataSourceStatus, { dataSourceId: job.dataSourceId as any, analysisStatus: "completed", lastError: undefined, lastAnalyzedAt: Date.now(), }, { token } ) persisted = true } } catch (persistError) { await logServer({ level: "error", message: "Failed to persist analysis", labels: ["api", "analyze", "persist", "error"], payload: { error: String(persistError) }, requestId, source: "api/analyze", }); } } return NextResponse.json({ success: true, data: analysis, persisted, stats: { features: analysis.features.length, keywords: analysis.keywords.length, personas: analysis.personas.length, useCases: analysis.useCases.length, competitors: analysis.competitors.length, dorkQueries: analysis.dorkQueries.length } }) } catch (error: any) { await logServer({ level: "error", message: "Analysis error", labels: ["api", "analyze", "error"], payload: { message: error?.message, stack: error?.stack, }, requestId: request.headers.get("x-request-id") ?? undefined, source: "api/analyze", }); if (jobId) { try { const token = await convexAuthNextjsToken(); await fetchMutation( api.analysisJobs.update, { jobId: jobId as any, status: "failed", error: error.message || "Analysis failed", timeline: timeline.map((item) => item.status === "running" ? { ...item, status: "failed" } : item ), }, { token } ); try { const job = await fetchQuery( api.analysisJobs.getById, { jobId: jobId as any }, { token } ) if (job?.dataSourceId) { await fetchMutation( api.dataSources.updateDataSourceStatus, { dataSourceId: job.dataSourceId as any, analysisStatus: "failed", lastError: error.message || "Analysis failed", lastAnalyzedAt: Date.now(), }, { token } ) } } catch { // Best-effort data source update only. } } catch { // Best-effort job update only. } } if (error instanceof ScrapingError) { return NextResponse.json( { error: error.message, code: error.code, needsManualInput: true }, { status: 400 } ) } if (error.name === 'ZodError') { return NextResponse.json( { error: 'Invalid URL provided' }, { status: 400 } ) } return NextResponse.json( { error: error.message || 'Failed to analyze website' }, { status: 500 } ) } }