feat: Implement analysis job tracking with progress timeline and enhanced data source status management.

convex/_generated/api.d.ts

@@ -8,10 +8,15 @@
  * @module
  */
 
+import type * as analyses from "../analyses.js";
+import type * as analysisJobs from "../analysisJobs.js";
 import type * as auth from "../auth.js";
 import type * as dataSources from "../dataSources.js";
 import type * as http from "../http.js";
+import type * as opportunities from "../opportunities.js";
 import type * as projects from "../projects.js";
+import type * as searchJobs from "../searchJobs.js";
+import type * as users from "../users.js";
 
 import type {
   ApiFromModules,
@@ -20,10 +25,15 @@ import type {
 } from "convex/server";
 
 declare const fullApi: ApiFromModules<{
+  analyses: typeof analyses;
+  analysisJobs: typeof analysisJobs;
   auth: typeof auth;
   dataSources: typeof dataSources;
   http: typeof http;
+  opportunities: typeof opportunities;
   projects: typeof projects;
+  searchJobs: typeof searchJobs;
+  users: typeof users;
 }>;
 
 /**

convex/analysisJobs.ts

@@ -0,0 +1,128 @@
+import { mutation, query } from "./_generated/server";
+import { v } from "convex/values";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const create = mutation({
+  args: {
+    projectId: v.id("projects"),
+    dataSourceId: v.optional(v.id("dataSources")),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const now = Date.now();
+    return await ctx.db.insert("analysisJobs", {
+      projectId: args.projectId,
+      dataSourceId: args.dataSourceId,
+      status: "pending",
+      progress: 0,
+      stage: undefined,
+      timeline: [],
+      createdAt: now,
+      updatedAt: now,
+    });
+  },
+});
+
+export const update = mutation({
+  args: {
+    jobId: v.id("analysisJobs"),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("completed"),
+      v.literal("failed")
+    ),
+    progress: v.optional(v.number()),
+    stage: v.optional(v.string()),
+    timeline: v.optional(v.array(v.object({
+      key: v.string(),
+      label: v.string(),
+      status: v.union(
+        v.literal("pending"),
+        v.literal("running"),
+        v.literal("completed"),
+        v.literal("failed")
+      ),
+      detail: v.optional(v.string()),
+    }))),
+    error: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) throw new Error("Job not found");
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const patch: Record<string, unknown> = {
+      status: args.status,
+      updatedAt: Date.now(),
+    };
+    if (args.progress !== undefined) patch.progress = args.progress;
+    if (args.error !== undefined) patch.error = args.error;
+    if (args.stage !== undefined) patch.stage = args.stage;
+    if (args.timeline !== undefined) patch.timeline = args.timeline;
+
+    await ctx.db.patch(args.jobId, patch);
+  },
+});
+
+export const getById = query({
+  args: {
+    jobId: v.id("analysisJobs"),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return null;
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) return null;
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) return null;
+
+    return job;
+  },
+});
+
+export const listByProject = query({
+  args: {
+    projectId: v.id("projects"),
+    status: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return [];
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) return [];
+
+    if (args.status) {
+      return await ctx.db
+        .query("analysisJobs")
+        .withIndex("by_project_status", (q) =>
+          q.eq("projectId", args.projectId).eq("status", args.status!)
+        )
+        .order("desc")
+        .collect();
+    }
+
+    return await ctx.db
+      .query("analysisJobs")
+      .withIndex("by_project_createdAt", (q) => q.eq("projectId", args.projectId))
+      .order("desc")
+      .collect();
+  },
+});

convex/dataSources.ts

@@ -19,6 +19,22 @@ export const getProjectDataSources = query({
     },
 });
 
+export const getById = query({
+    args: { dataSourceId: v.id("dataSources") },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) return null;
+
+        const dataSource = await ctx.db.get(args.dataSourceId);
+        if (!dataSource) return null;
+
+        const project = await ctx.db.get(dataSource.projectId);
+        if (!project || project.userId !== userId) return null;
+
+        return dataSource;
+    },
+});
+
 export const addDataSource = mutation({
     args: {
         projectId: v.optional(v.id("projects")), // Optional, if not provided, use default
@@ -30,6 +46,14 @@ export const addDataSource = mutation({
         const userId = await getAuthUserId(ctx);
         if (!userId) throw new Error("Unauthorized");
 
+        let normalizedUrl = args.url.trim();
+        if (normalizedUrl.startsWith("manual:")) {
+            // Keep manual sources as-is.
+        } else if (!normalizedUrl.startsWith("http")) {
+            normalizedUrl = `https://${normalizedUrl}`;
+        }
+        normalizedUrl = normalizedUrl.replace(/\/+$/, "");
+
         let projectId = args.projectId;
 
         // Use default project if not provided
@@ -55,7 +79,7 @@ export const addDataSource = mutation({
         const existing = await ctx.db
             .query("dataSources")
             .withIndex("by_project_url", (q) =>
-                q.eq("projectId", projectId!).eq("url", args.url)
+                q.eq("projectId", projectId!).eq("url", normalizedUrl)
             )
             .first();
 
@@ -64,7 +88,7 @@ export const addDataSource = mutation({
             : await ctx.db.insert("dataSources", {
                   projectId: projectId!, // Assert exists
                   type: args.type,
-                  url: args.url,
+                  url: normalizedUrl,
                   name: args.name,
                   analysisStatus: "pending",
                   lastAnalyzedAt: undefined,
@@ -83,7 +107,7 @@ export const addDataSource = mutation({
             }
         }
 
-        return { sourceId, projectId: projectId! };
+        return { sourceId, projectId: projectId!, isExisting: Boolean(existing) };
     },
 });
 
@@ -117,3 +141,46 @@ export const updateDataSourceStatus = mutation({
         });
     },
 });
+
+export const remove = mutation({
+    args: { dataSourceId: v.id("dataSources") },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) throw new Error("Unauthorized");
+
+        const dataSource = await ctx.db.get(args.dataSourceId);
+        if (!dataSource) throw new Error("Data source not found");
+
+        const project = await ctx.db.get(dataSource.projectId);
+        if (!project || project.userId !== userId) {
+            throw new Error("Project not found or unauthorized");
+        }
+
+        const updatedSelected = project.dorkingConfig.selectedSourceIds.filter(
+            (id) => id !== args.dataSourceId
+        );
+        await ctx.db.patch(project._id, {
+            dorkingConfig: { selectedSourceIds: updatedSelected },
+        });
+
+        const analyses = await ctx.db
+            .query("analyses")
+            .withIndex("by_dataSource_createdAt", (q) =>
+                q.eq("dataSourceId", args.dataSourceId)
+            )
+            .collect();
+        for (const analysis of analyses) {
+            await ctx.db.delete(analysis._id);
+        }
+
+        const analysisJobs = await ctx.db
+            .query("analysisJobs")
+            .filter((q) => q.eq(q.field("dataSourceId"), args.dataSourceId))
+            .collect();
+        for (const job of analysisJobs) {
+            await ctx.db.delete(job._id);
+        }
+
+        await ctx.db.delete(args.dataSourceId);
+    },
+});

convex/projects.ts

@@ -32,8 +32,17 @@ export const createProject = mutation({
         const userId = await getAuthUserId(ctx);
         if (!userId) throw new Error("Unauthorized");
 
-        // If setting as default, unset other defaults? For now assume handled by UI or logic
-        // Actually simplicity: just create.
+        if (args.isDefault) {
+            const existingDefaults = await ctx.db
+                .query("projects")
+                .withIndex("by_owner", (q) => q.eq("userId", userId))
+                .collect();
+            for (const project of existingDefaults) {
+                if (project.isDefault) {
+                    await ctx.db.patch(project._id, { isDefault: false });
+                }
+            }
+        }
 
         return await ctx.db.insert("projects", {
             userId,
@@ -44,6 +53,36 @@ export const createProject = mutation({
     },
 });
 
+export const updateProject = mutation({
+    args: { projectId: v.id("projects"), name: v.string(), isDefault: v.boolean() },
+    handler: async (ctx, args) => {
+        const userId = await getAuthUserId(ctx);
+        if (!userId) throw new Error("Unauthorized");
+
+        const project = await ctx.db.get(args.projectId);
+        if (!project || project.userId !== userId) {
+            throw new Error("Project not found or unauthorized");
+        }
+
+        if (args.isDefault) {
+            const existingDefaults = await ctx.db
+                .query("projects")
+                .withIndex("by_owner", (q) => q.eq("userId", userId))
+                .collect();
+            for (const item of existingDefaults) {
+                if (item.isDefault && item._id !== args.projectId) {
+                    await ctx.db.patch(item._id, { isDefault: false });
+                }
+            }
+        }
+
+        await ctx.db.patch(args.projectId, {
+            name: args.name,
+            isDefault: args.isDefault,
+        });
+    },
+});
+
 export const toggleDataSourceConfig = mutation({
     args: { projectId: v.id("projects"), sourceId: v.id("dataSources"), selected: v.boolean() },
     handler: async (ctx, args) => {

convex/schema.ts

@@ -147,6 +147,50 @@ const schema = defineSchema({
         .index("by_project_status", ["projectId", "status"])
         .index("by_project_createdAt", ["projectId", "createdAt"])
         .index("by_project_url", ["projectId", "url"]),
+    analysisJobs: defineTable({
+        projectId: v.id("projects"),
+        dataSourceId: v.optional(v.id("dataSources")),
+        status: v.union(
+            v.literal("pending"),
+            v.literal("running"),
+            v.literal("completed"),
+            v.literal("failed")
+        ),
+        progress: v.optional(v.number()),
+        stage: v.optional(v.string()),
+        timeline: v.optional(v.array(v.object({
+            key: v.string(),
+            label: v.string(),
+            status: v.union(
+                v.literal("pending"),
+                v.literal("running"),
+                v.literal("completed"),
+                v.literal("failed")
+            ),
+            detail: v.optional(v.string()),
+        }))),
+        error: v.optional(v.string()),
+        createdAt: v.number(),
+        updatedAt: v.number(),
+    })
+        .index("by_project_status", ["projectId", "status"])
+        .index("by_project_createdAt", ["projectId", "createdAt"]),
+    searchJobs: defineTable({
+        projectId: v.id("projects"),
+        status: v.union(
+            v.literal("pending"),
+            v.literal("running"),
+            v.literal("completed"),
+            v.literal("failed")
+        ),
+        config: v.optional(v.any()),
+        progress: v.optional(v.number()),
+        error: v.optional(v.string()),
+        createdAt: v.number(),
+        updatedAt: v.number(),
+    })
+        .index("by_project_status", ["projectId", "status"])
+        .index("by_project_createdAt", ["projectId", "createdAt"]),
 });
 
 export default schema;

convex/searchJobs.ts

@@ -0,0 +1,92 @@
+import { mutation, query } from "./_generated/server";
+import { v } from "convex/values";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const create = mutation({
+  args: {
+    projectId: v.id("projects"),
+    config: v.optional(v.any()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    const now = Date.now();
+    return await ctx.db.insert("searchJobs", {
+      projectId: args.projectId,
+      status: "pending",
+      config: args.config,
+      progress: 0,
+      createdAt: now,
+      updatedAt: now,
+    });
+  },
+});
+
+export const update = mutation({
+  args: {
+    jobId: v.id("searchJobs"),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("completed"),
+      v.literal("failed")
+    ),
+    progress: v.optional(v.number()),
+    error: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) throw new Error("Unauthorized");
+
+    const job = await ctx.db.get(args.jobId);
+    if (!job) throw new Error("Job not found");
+
+    const project = await ctx.db.get(job.projectId);
+    if (!project || project.userId !== userId) {
+      throw new Error("Project not found or unauthorized");
+    }
+
+    await ctx.db.patch(args.jobId, {
+      status: args.status,
+      progress: args.progress,
+      error: args.error,
+      updatedAt: Date.now(),
+    });
+  },
+});
+
+export const listByProject = query({
+  args: {
+    projectId: v.id("projects"),
+    status: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return [];
+
+    const project = await ctx.db.get(args.projectId);
+    if (!project || project.userId !== userId) return [];
+
+    if (args.status) {
+      return await ctx.db
+        .query("searchJobs")
+        .withIndex("by_project_status", (q) =>
+          q.eq("projectId", args.projectId).eq("status", args.status!)
+        )
+        .order("desc")
+        .collect();
+    }
+
+    return await ctx.db
+      .query("searchJobs")
+      .withIndex("by_project_createdAt", (q) => q.eq("projectId", args.projectId))
+      .order("desc")
+      .collect();
+  },
+});

convex/users.ts

@@ -0,0 +1,11 @@
+import { query } from "./_generated/server";
+import { getAuthUserId } from "@convex-dev/auth/server";
+
+export const getCurrent = query({
+  args: {},
+  handler: async (ctx) => {
+    const userId = await getAuthUserId(ctx);
+    if (!userId) return null;
+    return await ctx.db.get(userId);
+  },
+});