Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import { describe, expect, it } from "vitest";
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { schemaTools } from "../tools/schemaTools.js";
import { architectTools } from "../tools/architectTools.js";
import { devSetupTools } from "../tools/devSetupTools.js";

const audit = schemaTools.find(tool => tool.name === "convex_audit_schema")!;
const scan = architectTools.find(tool => tool.name === "convex_scan_capabilities")!;
const setup = devSetupTools.find(tool => tool.name === "convex_audit_dev_setup")!;
function project() { return mkdtempSync(join(homedir(), "project-")); }
function backend(root: string, relative: string, tables = 2) {
const dir = join(root, relative);
mkdirSync(join(dir, "_generated"), { recursive: true });
writeFileSync(join(dir, "schema.ts"), `import { defineSchema, defineTable } from "convex/server";
import { v } from "convex/values";
export default defineSchema({ ${Array.from({ length: tables }, (_, i) => `table${i}: defineTable({ title: v.string() })`).join(",")} });`);
writeFileSync(join(dir, "queries.ts"), 'export const list = query({ args: {}, returns: v.array(v.string()), handler: async (ctx) => [] });');
return dir;
}
function configure(root: string, value: unknown) {
writeFileSync(join(root, "convex.json"), JSON.stringify(value));
}
async function expectSource(root: string, source: string, tables = 2) {
const result = await audit.handler({ projectDir: root });
expect(result).not.toHaveProperty("error");
expect(result).toMatchObject({ summary: { schemaFile: join(source, "schema.ts"), tables } });
const capabilities = await scan.handler({ projectDir: root });
expect(capabilities).not.toHaveProperty("error");
expect(capabilities).toMatchObject({ mode: "directory", convexDir: source, totalFiles: 2 });
}
async function expectUnavailable(root: string) {
for (const tool of [audit, scan]) {
const result = await tool.handler({ projectDir: root });
expect(result).toMatchObject({ error: expect.stringMatching(/directory|config/i) });
expect(result).not.toHaveProperty("summary");
}
}

describe("backend resolution for developers and coding agents", () => {
it.each(["convex", "src/convex", "backend/convex"])("audits a developer's conventional %s layout without a config override", async relative => {
const root = project(); const source = backend(root, relative);
await expectSource(root, source);
});

it("uses the configured backend even when a stale conventional tree would produce a different audit", async () => {
const root = project();
backend(root, "convex", 4); backend(root, "src/convex", 5); backend(root, "backend/convex", 6);
const source = backend(root, "services/source code", 3);
configure(root, { functions: "services/source code/" });
await expectSource(root, source, 3);
expect(await setup.handler({ projectDir: root })).toMatchObject({ checks: expect.arrayContaining([
expect.objectContaining({ area: "convex_json", status: "pass" }),
expect.objectContaining({ area: "initialization", status: "pass" }),
]) });
});

it("preserves conventional discovery when a valid config has no functions override", async () => {
const root = project(); const source = backend(root, "backend/convex");
configure(root, {});
await expectSource(root, source);
});

it("refuses a missing configured directory instead of reassuring the developer about a stale tree", async () => {
const root = project(); backend(root, "convex");
configure(root, { functions: "backend/missing" });
await expectUnavailable(root);
const result = await setup.handler({ projectDir: root });
expect(result).toMatchObject({ checks: expect.arrayContaining([expect.objectContaining({ area: "convex_json", status: "fail" })]) });
expect(result).not.toMatchObject({ checks: expect.arrayContaining([expect.objectContaining({ area: "initialization", status: "pass" })]) });
});

it("treats a partially written config as an unavailable project, then recovers on the next call", async () => {
const root = project(); backend(root, "convex", 4);
const source = backend(root, "actual/source");
writeFileSync(join(root, "convex.json"), '{"functions":');
await expectUnavailable(root);
configure(root, { functions: "actual/source" });
await expectSource(root, source);
});

it.each([null, [], { functions: null }, { functions: 12 }, { functions: "" }, { functions: " " }].map(value => [value]))("rejects an invalid config value %j without falling back", async config => {
const root = project(); backend(root, "convex"); configure(root, config);
await expectUnavailable(root);
});

it("reports a configured regular file as unavailable instead of crashing during directory enumeration", async () => {
const root = project(); backend(root, "convex");
writeFileSync(join(root, "not-a-directory"), "reviewed text");
configure(root, { functions: "not-a-directory" });
await expectUnavailable(root);
});

it("skips a regular file at a conventional candidate and finds the actual source directory", async () => {
const root = project(); writeFileSync(join(root, "convex"), "not a directory");
const source = backend(root, "src/convex");
await expectSource(root, source);
});

it("rejects an oversized config before reading an unbounded document or scanning the wrong tree", async () => {
const root = project(); backend(root, "convex"); backend(root, "actual/source");
configure(root, { functions: "actual/source", description: "x".repeat(70_000) });
await expectUnavailable(root);
});

it("keeps twenty-four simultaneous project audits isolated across twelve successive configuration changes", async () => {
const projects = Array.from({ length: 24 }, () => {
const root = project(); backend(root, "convex", 5);
return { root, a: backend(root, "source/a", 2), b: backend(root, "source/b", 3) };
});
for (let round = 0; round < 12; round++) {
const side = round % 2 === 0 ? "a" : "b";
for (const item of projects) configure(item.root, { functions: `source/${side}` });
await Promise.all(projects.map(item => expectSource(item.root, item[side], side === "a" ? 2 : 3)));
}
}, 30_000);
});
7 changes: 5 additions & 2 deletions packages/convex-mcp-nodebench/src/__tests__/tools.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { findConvexDir } from "../project.js";
import { describe, it, expect, beforeAll } from "vitest";
import { resolve } from "node:path";
import { schemaTools } from "../tools/schemaTools.js";
Expand Down Expand Up @@ -76,7 +77,7 @@ describe("Function Tools", () => {
expect(result.summary).toBeDefined();
expect(typeof result.summary.totalFunctions).toBe("number");
console.log(`Function audit: ${result.summary.totalFunctions} functions, ${result.summary.totalIssues} issues (${result.summary.critical} critical)`);
});
}, 30_000); // scans the whole backend/convex tree, like the 30s/60s repo scans above

it("convex_check_function_refs runs against nodebench-ai", async () => {
const tool = functionTools.find((t) => t.name === "convex_check_function_refs")!;
Expand Down Expand Up @@ -588,7 +589,9 @@ describe("Architect Tools", () => {
});

it("convex_scan_capabilities scans a single file", async () => {
const schemaPath = resolve(PROJECT_DIR, "convex", "schema.ts");
const convexDir = findConvexDir(PROJECT_DIR);
expect(convexDir).not.toBeNull();
const schemaPath = resolve(convexDir!, "schema.ts");
const tool = architectTools.find((t) => t.name === "convex_scan_capabilities")!;
const result = (await tool.handler({ filePath: schemaPath })) as any;
expect(result).toBeDefined();
Expand Down
40 changes: 40 additions & 0 deletions packages/convex-mcp-nodebench/src/project.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { closeSync, openSync, readSync, statSync } from "node:fs";
import { isAbsolute, join, resolve } from "node:path";

const MAX_CONFIG_BYTES = 64 * 1024;

function directory(path: string): string | null {
try { return statSync(path).isDirectory() ? path : null; }
catch { return null; }
}

/** Resolve the project being audited; an invalid explicit config never selects a stale tree. */
export function findConvexDir(projectDir: string): string | null {
let config: unknown;
try {
const fd = openSync(join(projectDir, "convex.json"), "r");
try {
const buffer = Buffer.alloc(MAX_CONFIG_BYTES + 1);
const bytes = readSync(fd, buffer, 0, buffer.length, 0);
if (bytes > MAX_CONFIG_BYTES) return null;
config = JSON.parse(buffer.toString("utf8", 0, bytes));
} finally { closeSync(fd); }
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") return null;
}

if (config !== undefined) {
if (typeof config !== "object" || config === null || Array.isArray(config)) return null;
if (Object.hasOwn(config, "functions")) {
const functions = (config as Record<string, unknown>).functions;
if (typeof functions !== "string" || !functions.trim() || isAbsolute(functions)) return null;
return directory(resolve(projectDir, functions));
}
}

for (const relative of ["convex", "src/convex", "backend/convex"]) {
const found = directory(resolve(projectDir, relative));
if (found) return found;
}
return null;
}
11 changes: 2 additions & 9 deletions packages/convex-mcp-nodebench/src/tools/actionAuditTools.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { findConvexDir } from "../project.js";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { join, resolve } from "node:path";
import { getDb, genId } from "../db.js";
Expand All @@ -6,14 +7,6 @@ import type { McpTool } from "../types.js";

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
const candidates = [join(projectDir, "convex"), join(projectDir, "src", "convex"), join(projectDir, "backend", "convex")];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

function collectTsFiles(dir: string): string[] {
const results: string[] = [];
if (!existsSync(dir)) return results;
Expand Down Expand Up @@ -191,7 +184,7 @@ export const actionAuditTools: McpTool[] = [
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) {
return { error: "No convex/ directory found" };
return { error: "No configured Convex functions directory found" };
}

const { issues, stats } = auditActions(convexDir);
Expand Down
30 changes: 3 additions & 27 deletions packages/convex-mcp-nodebench/src/tools/architectTools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* - convex_generate_plan: Build a plan for missing signatures
*/

import { findConvexDir } from "../project.js";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { join, resolve } from "node:path";
import { getDb, genId } from "../db.js";
Expand Down Expand Up @@ -129,31 +130,6 @@ function analyzePatterns(content: string, patterns: PatternCategory): Record<str

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
// convex.json "functions" overrides the default directory, same as the Convex CLI.
const configPath = join(projectDir, "convex.json");
if (existsSync(configPath)) {
try {
const config = JSON.parse(readFileSync(configPath, "utf8")) as { functions?: string };
if (typeof config.functions === "string" && config.functions.length > 0) {
const configured = join(projectDir, config.functions);
if (existsSync(configured)) return configured;
}
} catch {
// fall through to the conventional candidates
}
}
const candidates = [
join(projectDir, "convex"),
join(projectDir, "src", "convex"),
join(projectDir, "backend", "convex"),
];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

function collectTsFiles(dir: string): string[] {
const results: string[] = [];
if (!existsSync(dir)) return results;
Expand Down Expand Up @@ -236,7 +212,7 @@ export const architectTools: McpTool[] = [
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) {
return { error: "No convex/ directory found" };
return { error: "No configured Convex functions directory found" };
}

const files = collectTsFiles(convexDir);
Expand Down Expand Up @@ -358,7 +334,7 @@ export const architectTools: McpTool[] = [
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) {
return { error: "No convex/ directory found" };
return { error: "No configured Convex functions directory found" };
}
const files = collectTsFiles(convexDir);
content = files.map(f => readFileSync(f, "utf-8")).join("\n// ── file boundary ──\n");
Expand Down
11 changes: 2 additions & 9 deletions packages/convex-mcp-nodebench/src/tools/authorizationTools.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { findConvexDir } from "../project.js";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { join, resolve } from "node:path";
import { getDb, genId } from "../db.js";
Expand All @@ -6,14 +7,6 @@ import type { McpTool } from "../types.js";

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
const candidates = [join(projectDir, "convex"), join(projectDir, "src", "convex"), join(projectDir, "backend", "convex")];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

function collectTsFiles(dir: string): string[] {
const results: string[] = [];
if (!existsSync(dir)) return results;
Expand Down Expand Up @@ -211,7 +204,7 @@ export const authorizationTools: McpTool[] = [
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) {
return { error: "No convex/ directory found" };
return { error: "No configured Convex functions directory found" };
}

const { issues, stats } = auditAuthorization(convexDir);
Expand Down
11 changes: 2 additions & 9 deletions packages/convex-mcp-nodebench/src/tools/componentTools.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
import { findConvexDir } from "../project.js";
import { readFileSync, existsSync } from "node:fs";
import { join, resolve } from "node:path";
import { getQuickRef } from "./toolRegistry.js";
import type { McpTool } from "../types.js";

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
const candidates = [join(projectDir, "convex"), join(projectDir, "src", "convex"), join(projectDir, "backend", "convex")];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

interface ComponentInfo {
importName: string;
packageName: string;
Expand Down Expand Up @@ -103,7 +96,7 @@ export const componentTools: McpTool[] = [
handler: async (args: { projectDir: string }) => {
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) return { error: "No convex/ directory found" };
if (!convexDir) return { error: "No configured Convex functions directory found" };

const configPath = join(convexDir, "convex.config.ts");
if (!existsSync(configPath)) {
Expand Down
11 changes: 2 additions & 9 deletions packages/convex-mcp-nodebench/src/tools/cronTools.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
import { findConvexDir } from "../project.js";
import { readFileSync, existsSync } from "node:fs";
import { join, resolve } from "node:path";
import { getQuickRef } from "./toolRegistry.js";
import type { McpTool } from "../types.js";

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
const candidates = [join(projectDir, "convex"), join(projectDir, "src", "convex"), join(projectDir, "backend", "convex")];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

interface CronEntry {
name: string;
type: "interval" | "daily" | "weekly" | "monthly" | "cron";
Expand Down Expand Up @@ -122,7 +115,7 @@ export const cronTools: McpTool[] = [
handler: async (args: { projectDir: string }) => {
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) return { error: "No convex/ directory found" };
if (!convexDir) return { error: "No configured Convex functions directory found" };

const cronsPath = join(convexDir, "crons.ts");
if (!existsSync(cronsPath)) {
Expand Down
11 changes: 2 additions & 9 deletions packages/convex-mcp-nodebench/src/tools/dataModelingTools.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { findConvexDir } from "../project.js";
import { readFileSync, existsSync } from "node:fs";
import { join, resolve } from "node:path";
import { getDb, genId } from "../db.js";
Expand All @@ -6,14 +7,6 @@ import type { McpTool } from "../types.js";

// ── Helpers ──────────────────────────────────────────────────────────

function findConvexDir(projectDir: string): string | null {
const candidates = [join(projectDir, "convex"), join(projectDir, "src", "convex"), join(projectDir, "backend", "convex")];
for (const c of candidates) {
if (existsSync(c)) return c;
}
return null;
}

// ── Data Modeling Audit ─────────────────────────────────────────────

interface ModelingIssue {
Expand Down Expand Up @@ -229,7 +222,7 @@ export const dataModelingTools: McpTool[] = [
const projectDir = resolve(args.projectDir);
const convexDir = findConvexDir(projectDir);
if (!convexDir) {
return { error: "No convex/ directory found" };
return { error: "No configured Convex functions directory found" };
}

const { issues, stats } = auditDataModeling(convexDir);
Expand Down
Loading
Loading