From c47614bc668c703730120a9dbb27609cfa6a48f3 Mon Sep 17 00:00:00 2001 From: Matiss Jurevics Date: Tue, 3 Feb 2026 20:35:03 +0000 Subject: [PATCH] feat: Implement data source management and analysis flow, allowing users to add and analyze websites for project opportunities. --- app/(app)/dashboard/page.tsx | 118 +++++++++++++++++++++++ app/(app)/opportunities/page.tsx | 77 ++++++++++++++- app/api/opportunities/route.ts | 59 +++++------- app/onboarding/page.tsx | 7 ++ components/app-sidebar.tsx | 126 ++++++++++++++++++++++++ convex/analyses.ts | 22 +++++ convex/dataSources.ts | 66 ++++++++++--- convex/projects.ts | 159 +++++++++++++++++++++++++++++++ convex/schema.ts | 7 +- 9 files changed, 587 insertions(+), 54 deletions(-) diff --git a/app/(app)/dashboard/page.tsx b/app/(app)/dashboard/page.tsx index 0cd6ffd..49bfbad 100644 --- a/app/(app)/dashboard/page.tsx +++ b/app/(app)/dashboard/page.tsx @@ -1,20 +1,36 @@ "use client" import { useQuery } from "convex/react" +import { useState } from "react" import { api } from "@/convex/_generated/api" import { useProject } from "@/components/project-context" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Badge } from "@/components/ui/badge" +import { Alert, AlertDescription } from "@/components/ui/alert" +import { Button } from "@/components/ui/button" +import { useMutation } from "convex/react" export default function Page() { const { selectedProjectId } = useProject() const projects = useQuery(api.projects.getProjects) + const dataSources = useQuery( + api.dataSources.getProjectDataSources, + selectedProjectId ? { projectId: selectedProjectId as any } : "skip" + ) + const searchContext = useQuery( + api.projects.getSearchContext, + selectedProjectId ? { projectId: selectedProjectId as any } : "skip" + ) + const updateDataSourceStatus = useMutation(api.dataSources.updateDataSourceStatus) + const createAnalysis = useMutation(api.analyses.createAnalysis) + const [reanalyzingId, setReanalyzingId] = useState(null) const analysis = useQuery( api.analyses.getLatestByProject, selectedProjectId ? { projectId: selectedProjectId as any } : "skip" ) const selectedProject = projects?.find((project) => project._id === selectedProjectId) + const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || [] const isLoading = selectedProjectId && analysis === undefined if (!selectedProjectId && projects && projects.length === 0) { @@ -47,6 +63,53 @@ export default function Page() { ) } + const handleReanalyze = async (source: any) => { + if (!selectedProjectId) return + + setReanalyzingId(source._id) + await updateDataSourceStatus({ + dataSourceId: source._id, + analysisStatus: "pending", + lastError: undefined, + lastAnalyzedAt: undefined, + }) + + try { + const response = await fetch("/api/analyze", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url: source.url }), + }) + + const data = await response.json() + + if (!response.ok) { + await updateDataSourceStatus({ + dataSourceId: source._id, + analysisStatus: "failed", + lastError: data.error || "Analysis failed", + lastAnalyzedAt: Date.now(), + }) + return + } + + await createAnalysis({ + projectId: selectedProjectId as any, + dataSourceId: source._id, + analysis: data.data, + }) + + await updateDataSourceStatus({ + dataSourceId: source._id, + analysisStatus: "completed", + lastError: undefined, + lastAnalyzedAt: Date.now(), + }) + } finally { + setReanalyzingId(null) + } + } + return (
@@ -60,6 +123,14 @@ export default function Page() {

{analysis.description}

+ {searchContext?.missingSources?.length > 0 && ( + + + Some selected sources don't have analysis yet. Run onboarding or re-analyze them for best results. + + + )} +
@@ -93,6 +164,53 @@ export default function Page() {
+ + + Data Sources + + + {dataSources && dataSources.length > 0 ? ( + dataSources.map((source: any) => ( +
+ {source.name || source.url} +
+ + {selectedSourceIds.includes(source._id) ? "active" : "inactive"} + + + {source.analysisStatus} + + +
+
+ )) + ) : ( +

No data sources yet. Add a source during onboarding.

+ )} + + + + {searchContext?.context && ( + + + Aggregated Context + + +
Keywords: {searchContext.context.keywords.length}
+
Problems: {searchContext.context.problemsSolved.length}
+
Competitors: {searchContext.context.competitors.length}
+
Use Cases: {searchContext.context.useCases.length}
+ + + )} +
diff --git a/app/(app)/opportunities/page.tsx b/app/(app)/opportunities/page.tsx index b7b163f..ee6d738 100644 --- a/app/(app)/opportunities/page.tsx +++ b/app/(app)/opportunities/page.tsx @@ -89,6 +89,7 @@ export default function OpportunitiesPage() { const [replyText, setReplyText] = useState('') const [stats, setStats] = useState(null) const [searchError, setSearchError] = useState('') + const [missingSources, setMissingSources] = useState([]) const [statusFilter, setStatusFilter] = useState('all') const [intentFilter, setIntentFilter] = useState('all') const [minScore, setMinScore] = useState(0) @@ -97,6 +98,7 @@ export default function OpportunitiesPage() { const [notesInput, setNotesInput] = useState('') const [tagsInput, setTagsInput] = useState('') + const projects = useQuery(api.projects.getProjects) const latestAnalysis = useQuery( api.analyses.getLatestByProject, selectedProjectId ? { projectId: selectedProjectId as any } : 'skip' @@ -137,12 +139,21 @@ export default function OpportunitiesPage() { })) as Opportunity[] }, [savedOpportunities, opportunities]) + const selectedSources = useQuery( + api.dataSources.getProjectDataSources, + selectedProjectId ? { projectId: selectedProjectId as any } : "skip" + ) + + const selectedProject = projects?.find((project: any) => project._id === selectedProjectId) + const selectedSourceIds = selectedProject?.dorkingConfig?.selectedSourceIds || [] + const activeSources = selectedSources?.filter((source: any) => + selectedSourceIds.includes(source._id) + ) || [] + useEffect(() => { const stored = localStorage.getItem('productAnalysis') if (stored) { setAnalysis(JSON.parse(stored)) - } else { - router.push('/onboarding') } fetch('/api/opportunities') @@ -160,6 +171,18 @@ export default function OpportunitiesPage() { }) }, [router]) + useEffect(() => { + if (!analysis && latestAnalysis) { + setAnalysis(latestAnalysis as any) + } + }, [analysis, latestAnalysis]) + + useEffect(() => { + if (!analysis && latestAnalysis === null) { + router.push('/onboarding') + } + }, [analysis, latestAnalysis, router]) + const togglePlatform = (platformId: string) => { setPlatforms(prev => prev.map(p => p.id === platformId ? { ...p, enabled: !p.enabled } : p @@ -193,7 +216,7 @@ export default function OpportunitiesPage() { const response = await fetch('/api/opportunities', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ analysis, config }) + body: JSON.stringify({ projectId: selectedProjectId, config }) }) if (response.redirected) { @@ -202,7 +225,11 @@ export default function OpportunitiesPage() { } const data = await response.json() - + + if (!response.ok) { + throw new Error(data.error || 'Failed to search for opportunities') + } + if (data.success) { const mapped = data.data.opportunities.map((opp: Opportunity) => ({ ...opp, @@ -211,6 +238,7 @@ export default function OpportunitiesPage() { setOpportunities(mapped) setGeneratedQueries(data.data.queries) setStats(data.data.stats) + setMissingSources(data.data.missingSources || []) await upsertOpportunities({ projectId: selectedProjectId as any, @@ -353,7 +381,11 @@ export default function OpportunitiesPage() {
))} +
@@ -205,6 +288,49 @@ export function AppSidebar({ ...props }: React.ComponentProps) { avatar: "" }} /> + + + + Add Data Source + + Add a website to analyze for this project. + + +
+
+ + setSourceUrl(event.target.value)} + disabled={isSubmittingSource} + /> +
+
+ + setSourceName(event.target.value)} + disabled={isSubmittingSource} + /> +
+ {sourceError && ( +
{sourceError}
+ )} +
+ + +
+
+ + ) } diff --git a/convex/analyses.ts b/convex/analyses.ts index 81e357f..e14e9ce 100644 --- a/convex/analyses.ts +++ b/convex/analyses.ts @@ -19,6 +19,28 @@ export const getLatestByProject = query({ }, }); +export const getLatestByDataSource = query({ + args: { dataSourceId: v.id("dataSources") }, + handler: async (ctx, { dataSourceId }) => { + const userId = await getAuthUserId(ctx); + if (!userId) return null; + + const dataSource = await ctx.db.get(dataSourceId); + if (!dataSource) return null; + + const project = await ctx.db.get(dataSource.projectId); + if (!project || project.userId !== userId) return null; + + return await ctx.db + .query("analyses") + .withIndex("by_dataSource_createdAt", (q) => + q.eq("dataSourceId", dataSourceId) + ) + .order("desc") + .first(); + }, +}); + export const createAnalysis = mutation({ args: { projectId: v.id("projects"), diff --git a/convex/dataSources.ts b/convex/dataSources.ts index 7d4fa86..e01dd18 100644 --- a/convex/dataSources.ts +++ b/convex/dataSources.ts @@ -52,24 +52,68 @@ export const addDataSource = mutation({ } } - const sourceId = await ctx.db.insert("dataSources", { - projectId: projectId!, // Assert exists - type: args.type, - url: args.url, - name: args.name, - analysisStatus: "pending", - // analysisResults not set initially - }); + const existing = await ctx.db + .query("dataSources") + .withIndex("by_project_url", (q) => + q.eq("projectId", projectId!).eq("url", args.url) + ) + .first(); + + const sourceId = existing + ? existing._id + : await ctx.db.insert("dataSources", { + projectId: projectId!, // Assert exists + type: args.type, + url: args.url, + name: args.name, + analysisStatus: "pending", + lastAnalyzedAt: undefined, + lastError: undefined, + // analysisResults not set initially + }); // Auto-select this source in the project config const project = await ctx.db.get(projectId!); if (project) { const currentSelected = project.dorkingConfig.selectedSourceIds; - await ctx.db.patch(projectId!, { - dorkingConfig: { selectedSourceIds: [...currentSelected, sourceId] } - }); + if (!currentSelected.includes(sourceId)) { + await ctx.db.patch(projectId!, { + dorkingConfig: { selectedSourceIds: [...currentSelected, sourceId] } + }); + } } return { sourceId, projectId: projectId! }; }, }); + +export const updateDataSourceStatus = mutation({ + args: { + dataSourceId: v.id("dataSources"), + analysisStatus: v.union( + v.literal("pending"), + v.literal("completed"), + v.literal("failed") + ), + lastError: v.optional(v.string()), + lastAnalyzedAt: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const userId = await getAuthUserId(ctx); + if (!userId) throw new Error("Unauthorized"); + + const dataSource = await ctx.db.get(args.dataSourceId); + if (!dataSource) throw new Error("Data source not found"); + + const project = await ctx.db.get(dataSource.projectId); + if (!project || project.userId !== userId) { + throw new Error("Project not found or unauthorized"); + } + + await ctx.db.patch(args.dataSourceId, { + analysisStatus: args.analysisStatus, + lastError: args.lastError, + lastAnalyzedAt: args.lastAnalyzedAt, + }); + }, +}); diff --git a/convex/projects.ts b/convex/projects.ts index 4b57282..98b5db0 100644 --- a/convex/projects.ts +++ b/convex/projects.ts @@ -67,3 +67,162 @@ export const toggleDataSourceConfig = mutation({ }); }, }); + +export const getSearchContext = query({ + args: { projectId: v.id("projects") }, + handler: async (ctx, args) => { + const userId = await getAuthUserId(ctx); + if (!userId) return { context: null, selectedSourceIds: [], missingSources: [] }; + + const project = await ctx.db.get(args.projectId); + if (!project || project.userId !== userId) { + return { context: null, selectedSourceIds: [], missingSources: [] }; + } + + const selectedSourceIds = project.dorkingConfig.selectedSourceIds || []; + if (selectedSourceIds.length === 0) { + return { context: null, selectedSourceIds, missingSources: [] }; + } + + const analyses = []; + const missingSources: { sourceId: string; reason: string }[] = []; + + for (const sourceId of selectedSourceIds) { + const dataSource = await ctx.db.get(sourceId); + if (!dataSource || dataSource.projectId !== project._id) { + missingSources.push({ sourceId: sourceId as string, reason: "not_found" }); + continue; + } + + const latest = await ctx.db + .query("analyses") + .withIndex("by_dataSource_createdAt", (q) => q.eq("dataSourceId", sourceId)) + .order("desc") + .first(); + + if (!latest) { + missingSources.push({ sourceId: sourceId as string, reason: "no_analysis" }); + continue; + } + + analyses.push(latest); + } + + if (analyses.length === 0) { + return { context: null, selectedSourceIds, missingSources }; + } + + const merged = mergeAnalyses(analyses, project.name); + + return { context: merged, selectedSourceIds, missingSources }; + }, +}); + +function normalizeKey(value: string) { + return value.trim().toLowerCase(); +} + +function severityRank(severity: "high" | "medium" | "low") { + if (severity === "high") return 3; + if (severity === "medium") return 2; + return 1; +} + +function mergeAnalyses(analyses: any[], fallbackName: string) { + const keywordMap = new Map(); + const problemMap = new Map(); + const competitorMap = new Map(); + const personaMap = new Map(); + const useCaseMap = new Map(); + const featureMap = new Map(); + + let latestScrapedAt = analyses[0].scrapedAt; + + for (const analysis of analyses) { + if (analysis.scrapedAt > latestScrapedAt) { + latestScrapedAt = analysis.scrapedAt; + } + + for (const keyword of analysis.keywords || []) { + const key = normalizeKey(keyword.term); + if (!keywordMap.has(key)) { + keywordMap.set(key, keyword); + } else if (keywordMap.get(key)?.type !== "differentiator" && keyword.type === "differentiator") { + keywordMap.set(key, keyword); + } + } + + for (const problem of analysis.problemsSolved || []) { + const key = normalizeKey(problem.problem); + const existing = problemMap.get(key); + if (!existing || severityRank(problem.severity) > severityRank(existing.severity)) { + problemMap.set(key, problem); + } + } + + for (const competitor of analysis.competitors || []) { + const key = normalizeKey(competitor.name); + if (!competitorMap.has(key)) { + competitorMap.set(key, competitor); + } + } + + for (const persona of analysis.personas || []) { + const key = normalizeKey(`${persona.name}:${persona.role}`); + if (!personaMap.has(key)) { + personaMap.set(key, persona); + } + } + + for (const useCase of analysis.useCases || []) { + const key = normalizeKey(useCase.scenario); + if (!useCaseMap.has(key)) { + useCaseMap.set(key, useCase); + } + } + + for (const feature of analysis.features || []) { + const key = normalizeKey(feature.name); + if (!featureMap.has(key)) { + featureMap.set(key, feature); + } + } + } + + const keywords = Array.from(keywordMap.values()) + .sort((a, b) => { + const aDiff = a.type === "differentiator" ? 0 : 1; + const bDiff = b.type === "differentiator" ? 0 : 1; + if (aDiff !== bDiff) return aDiff - bDiff; + return a.term.length - b.term.length; + }) + .slice(0, 80); + + const problemsSolved = Array.from(problemMap.values()) + .sort((a, b) => severityRank(b.severity) - severityRank(a.severity)) + .slice(0, 15); + + const competitors = Array.from(competitorMap.values()).slice(0, 10); + const personas = Array.from(personaMap.values()).slice(0, 6); + const useCases = Array.from(useCaseMap.values()).slice(0, 10); + const features = Array.from(featureMap.values()).slice(0, 20); + + const base = analyses[0]; + + return { + productName: base.productName || fallbackName, + tagline: base.tagline || "", + description: base.description || "", + category: base.category || "", + positioning: base.positioning || "", + features, + problemsSolved, + personas, + keywords, + useCases, + competitors, + dorkQueries: [], + scrapedAt: latestScrapedAt, + analysisVersion: "aggregated", + }; +} diff --git a/convex/schema.ts b/convex/schema.ts index d7e577d..416f3b0 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -22,6 +22,8 @@ const schema = defineSchema({ v.literal("completed"), v.literal("failed") ), + lastAnalyzedAt: v.optional(v.number()), + lastError: v.optional(v.string()), analysisResults: v.optional( v.object({ features: v.array(v.string()), @@ -31,7 +33,7 @@ const schema = defineSchema({ }) ), metadata: v.optional(v.any()), - }), + }).index("by_project_url", ["projectId", "url"]), analyses: defineTable({ projectId: v.id("projects"), dataSourceId: v.id("dataSources"), @@ -121,7 +123,8 @@ const schema = defineSchema({ })), scrapedAt: v.string(), }) - .index("by_project_createdAt", ["projectId", "createdAt"]), + .index("by_project_createdAt", ["projectId", "createdAt"]) + .index("by_dataSource_createdAt", ["dataSourceId", "createdAt"]), opportunities: defineTable({ projectId: v.id("projects"), analysisId: v.optional(v.id("analyses")),