diff --git a/docs/testing.md b/docs/testing.md index 554bf9fcc1..2517d62c91 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -65,6 +65,18 @@ API dependency, and independent fixture compiler oracles keep their existing TypeScript versions. `npm run build:cached` compiles the CLI/MCP runtime and its packaged skill. +The build also records `dist/mcp-catalog.json` (compact inputs) and +`dist/mcp-catalog-full.json` from the canonical SDK registrations and binds +each to the runtime with its own SHA-256 digest. These are +runtime artifacts, separate from the test catalog below. Default target-free +MCP startup serves its schemas without constructing analysis validators; +the first tool or guided-workflow request materializes the canonical runtime +on that same SDK server. Explicit compact/full input selections preserve +their advertised schemas throughout materialization. Initial-target, +HTTP and output-schema-off startup keep the eager path; source runs without +recorded artifacts also use eager composition. Compare memory both after discovery and after a completed +request; report diagnostic GC separately from normal resident memory. + Turbo's strict task environment preserves caller-selected `GOMAXPROCS`, `GOMEMLIMIT`, and `UV_THREADPOOL_SIZE` for resource-constrained builds and checks. These execution controls do not change artifact content, so changing diff --git a/experiments/analysis-efficiency/managed-cli-runtime.mjs b/experiments/analysis-efficiency/managed-cli-runtime.mjs index f7655e5dd2..628547eb2a 100644 --- a/experiments/analysis-efficiency/managed-cli-runtime.mjs +++ b/experiments/analysis-efficiency/managed-cli-runtime.mjs @@ -6,8 +6,9 @@ import { join, relative } from "node:path"; import { fileURLToPath } from "node:url"; const sha = (value) => createHash("sha256").update(value).digest("hex"); -/** Bind a trial to its source checkout and the compiled CLI actually launched. */ -export const captureRuntimeBinding = async (runtime) => { +/** Bind a trial to its source checkout and compiled runtime, including deferred assets. */ +export const captureRuntimeBinding = async (runtime, kind = "cli") => { + assert.ok(kind === "cli" || kind === "mcp", "Unknown runtime entry"); const root = await realpath(runtime); const git = (args) => execFileSync("git", args, { cwd: root, maxBuffer: 16 * 1024 * 1024 }); @@ -50,10 +51,16 @@ export const captureRuntimeBinding = async (runtime) => { files.push(relative(root, path)); } }; - await visit(join(root, "dist/cli-runtime")); + // MCP materializes additional workers and reads build-owned JSON after + // discovery. Bind every compiled JS/JSON asset, rather than only its entry. + await visit(join(root, kind === "mcp" ? "dist" : "dist/cli-runtime")); assert.ok( - files.includes(join("dist", "cli-runtime", "cli.js")), - "Missing compiled CLI entry", + files.includes( + kind === "mcp" + ? join("dist", "mcp", "main.js") + : join("dist", "cli-runtime", "cli.js"), + ), + "Missing compiled runtime entry", ); const hash = createHash("sha256"); let bytes = 0; @@ -63,15 +70,23 @@ export const captureRuntimeBinding = async (runtime) => { hash.update(JSON.stringify([name, value.length, sha(value)]) + "\n"); } return { - binding_version: 1, + binding_version: kind === "mcp" ? 2 : 1, root, node: process.version, git_revision: gitRevision, source_diff_sha256: sha(sourceDiff), untracked_source_sha256: sha(JSON.stringify(untrackedEntries)), - compiled_cli_sha256: hash.digest("hex"), - compiled_cli_files: files.length, - compiled_cli_bytes: bytes, + ...(kind === "mcp" + ? { + compiled_runtime_sha256: hash.digest("hex"), + compiled_runtime_files: files.length, + compiled_runtime_bytes: bytes, + } + : { + compiled_cli_sha256: hash.digest("hex"), + compiled_cli_files: files.length, + compiled_cli_bytes: bytes, + }), }; }; @@ -82,6 +97,10 @@ export const assertRuntimeBinding = (actual, expected, mode) => { if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { console.log( - JSON.stringify(await captureRuntimeBinding(process.argv[2]), null, 2), + JSON.stringify( + await captureRuntimeBinding(process.argv[2], process.argv[3]), + null, + 2, + ), ); } diff --git a/experiments/analysis-efficiency/mcp-input-catalog.md b/experiments/analysis-efficiency/mcp-input-catalog.md index 7f5ca3f47f..a926169a41 100644 --- a/experiments/analysis-efficiency/mcp-input-catalog.md +++ b/experiments/analysis-efficiency/mcp-input-catalog.md @@ -226,3 +226,8 @@ keeps default valid calls and typed refusals while checking full producer constraints separately. Five isolated suites passed 14 tests across call paths, ADB start, browser scenarios, source selection and managed analysis. The first browser attempt failed and remains recorded; no performance cohort changed. + +The subsequent [startup audit](mcp-startup.md) extends this harness with +source/bundle bindings and a post-materialization directory check. Its new +measurements are a separate cohort; reproduce this report at its recorded +revision when comparing the original timing and model-session receipts. diff --git a/experiments/analysis-efficiency/mcp-input-catalog.mjs b/experiments/analysis-efficiency/mcp-input-catalog.mjs index 45672b4f86..c6b3e98e13 100644 --- a/experiments/analysis-efficiency/mcp-input-catalog.mjs +++ b/experiments/analysis-efficiency/mcp-input-catalog.mjs @@ -7,6 +7,10 @@ import { parseArgs } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { + captureRuntimeBinding, + assertRuntimeBinding, +} from "./managed-cli-runtime.mjs"; const { values } = parseArgs({ options: { @@ -15,26 +19,68 @@ const { values } = parseArgs({ oracle: { type: "string" }, output: { type: "string" }, profile: { type: "string", default: "default" }, + phase: { type: "string", default: "analysis" }, + cache: { type: "string", default: "cold" }, + "cache-root": { type: "string" }, + bindings: { type: "string" }, + role: { type: "string" }, }, }); -for (const key of ["runtime", "artifact", "oracle", "output"]) +assert.ok(["analysis", "discovery"].includes(values.phase)); +assert.ok(["cold", "warm"].includes(values.cache)); +for (const key of values.phase === "analysis" + ? ["runtime", "artifact", "oracle", "output"] + : ["runtime", "output"]) assert.ok( values[key] && resolve(values[key]) === values[key], `${key} must be absolute`, ); assert.ok(["default", "full", "compact"].includes(values.profile)); +if (values.cache === "warm") + assert.ok( + values["cache-root"] && + resolve(values["cache-root"]) === values["cache-root"], + ); await mkdir(values.output, { recursive: false }); const hash = (value) => createHash("sha256").update(value).digest("hex"); -const oracle = JSON.parse(await readFile(values.oracle, "utf8")); -const artifactSha = hash(await readFile(values.artifact)); -assert.equal(artifactSha, oracle.artifact_sha256); +let oracle; +let artifactSha; +if (values.phase === "analysis") { + oracle = JSON.parse(await readFile(values.oracle, "utf8")); + artifactSha = hash(await readFile(values.artifact)); + assert.equal(artifactSha, oracle.artifact_sha256); +} +let expectedBinding; +let bindingBefore; +let auditBeforeMs; +let measurementInputsSha256; +const producerSources = {}; +if (values.bindings !== undefined) { + assert.ok(["baseline", "candidate"].includes(values.role)); + const inputBytes = await readFile(values.bindings); + measurementInputsSha256 = hash(inputBytes); + const inputs = JSON.parse(inputBytes.toString("utf8")); + for (const name of ["mcp-input-catalog.mjs", "managed-cli-runtime.mjs"]) { + producerSources[name] = hash( + await readFile(new URL(name, import.meta.url)), + ); + assert.equal(producerSources[name], inputs.harness_sources_sha256[name]); + } + expectedBinding = inputs.runtime_bindings[values.role]; + const start = performance.now(); + bindingBefore = await captureRuntimeBinding(values.runtime, "mcp"); + auditBeforeMs = performance.now() - start; + assertRuntimeBinding(bindingBefore, expectedBinding, values.role); +} const environment = { PATH: process.env.PATH, HOME: values.output, REA_LOG_LEVEL: "silent", UV_THREADPOOL_SIZE: "1", NODE_OPTIONS: "--max-old-space-size=2048", - NODE_DISABLE_COMPILE_CACHE: "1", + ...(values.cache === "cold" + ? { NODE_DISABLE_COMPILE_CACHE: "1" } + : { NODE_COMPILE_CACHE: values["cache-root"] }), ...(values.profile === "default" ? {} : { REA_MCP_INPUT_SCHEMA_PROFILE: values.profile }), @@ -46,14 +92,29 @@ const transport = new StdioClientTransport({ env: environment, stderr: "pipe", }); +const protocolRequests = {}; +const send = transport.send.bind(transport); +transport.send = (...args) => { + const method = args[0].method; + if (typeof method === "string") + protocolRequests[method] = (protocolRequests[method] ?? 0) + 1; + return send(...args); +}; const client = new Client({ name: "rea-catalog-task", version: "1" }); const report = { profile: values.profile, + phase: values.phase, + cache: values.cache, + cache_root: values["cache-root"] ?? null, + role: values.role, + measurement_inputs_sha256: measurementInputsSha256, + producer_sources_sha256: producerSources, artifact_sha256: artifactSha, calls: [], checks: [], status: "incomplete", cleanup_verified: false, + protocol_requests: protocolRequests, }; const transcript = []; let stderr = ""; @@ -63,6 +124,15 @@ transport.stderr?.on("data", (value) => { const began = performance.now(); try { await client.connect(transport); + report.protocol_era = client.getProtocolEra(); + await writeFile( + join(values.output, "server.json"), + JSON.stringify({ + info: client.getServerVersion(), + capabilities: client.getServerCapabilities(), + instructions: client.getInstructions(), + }), + ); const catalog = await client.listTools(); assert.equal(catalog.nextCursor, undefined); const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); @@ -73,97 +143,145 @@ try { ajv.validateSchema(tool.inputSchema), `Invalid schema: ${tool.name}`, ); - await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog)); + { + const encoded = JSON.stringify(catalog); + await writeFile(join(values.output, "catalog.json"), encoded); + report.catalog_sha256 = hash(encoded); + } report.tool_names = [...definitions.keys()]; - const call = async (name, args) => { - const definition = definitions.get(name); - assert.ok(definition, `Missing named tool ${name}`); - const start = performance.now(); - const reply = await client.callTool( - { name, arguments: args }, - { toolDefinition: definition }, + if (values.phase === "discovery") { + const prompts = await client.listPrompts(); + assert.equal(prompts.nextCursor, undefined); + await writeFile( + join(values.output, "prompts.json"), + JSON.stringify(prompts), ); - report.calls.push({ - name, - elapsed_ms: performance.now() - start, - is_error: reply.isError === true, - }); - transcript.push({ name, arguments: args, reply }); - return reply; - }; - const input = { - path: values.artifact, - view: { - collection: "packages", - names: ["serde"], - fields: ["name", "version", "kind"], - }, - }; - const first = await call("inspect_rust_binary", input); - assert.notEqual(first.isError, true); - const selected = first.structuredContent; - assert.ok(selected?.normalized_result?.page); - assert.deepEqual(selected.normalized_result.page.rows, oracle.serde_rows); - assert.equal(selected.subject.digest.sha256, artifactSha); - const parentId = selected.normalized_result.parent_evidence_id; - for (let index = 0; index < 2; index++) { - const reply = await call("inspect_analysis_view", { - source: { kind: "retained-evidence", evidence_id: parentId }, + report.prompts_sha256 = hash(JSON.stringify(prompts)); + report.answer = { + tools: report.tool_names, + prompts: prompts.prompts.map((prompt) => prompt.name), + }; + report.status = "passed"; + } else { + const call = async (name, args) => { + const definition = definitions.get(name); + assert.ok(definition, `Missing named tool ${name}`); + const start = performance.now(); + const reply = await client.callTool( + { name, arguments: args }, + { toolDefinition: definition }, + ); + report.calls.push({ + name, + elapsed_ms: performance.now() - start, + is_error: reply.isError === true, + }); + transcript.push({ name, arguments: args, reply }); + return reply; + }; + const input = { + path: values.artifact, view: { - kind: "rust", collection: "packages", names: ["serde"], fields: ["name", "version", "kind"], }, + }; + const first = await call("inspect_rust_binary", input); + assert.notEqual(first.isError, true); + const selected = first.structuredContent; + assert.ok(selected?.normalized_result?.page); + assert.deepEqual(selected.normalized_result.page.rows, oracle.serde_rows); + assert.equal(selected.subject.digest.sha256, artifactSha); + const parentId = selected.normalized_result.parent_evidence_id; + for (let index = 0; index < 2; index++) { + const reply = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { + kind: "rust", + collection: "packages", + names: ["serde"], + fields: ["name", "version", "kind"], + }, + }); + assert.notEqual(reply.isError, true); + assert.deepEqual( + reply.structuredContent.normalized_result.page.rows, + oracle.serde_rows, + ); + assert.equal( + reply.structuredContent.normalized_result.parent_evidence_id, + parentId, + ); + } + const malformed = await call("inspect_rust_binary", { + path: values.artifact, + view: { collection: "packages", offset: -1 }, }); - assert.notEqual(reply.isError, true); + assert.equal(malformed.isError, true); + report.checks.push( + "Independent package rows and digest; repeated retained identity; canonical negative-offset refusal", + ); + const exported = await call("export_evidence_bundle", { + path: join(values.output, "evidence.json"), + }); + assert.notEqual(exported.isError, true); + const bundle = JSON.parse( + await readFile(join(values.output, "evidence.json"), "utf8"), + ); + const parent = bundle.records.find( + (record) => record.evidence_id === parentId, + ); + assert.ok(parent, "Complete original Evidence is retained"); + assert.equal(parent.subject.digest.sha256, artifactSha); assert.deepEqual( - reply.structuredContent.normalized_result.page.rows, - oracle.serde_rows, + parent.normalized_result.auditable_metadata.packages, + oracle.packages, ); - assert.equal( - reply.structuredContent.normalized_result.parent_evidence_id, - parentId, + assert.deepEqual( + parent.normalized_result.auditable_metadata.dependency_edges, + oracle.edges, ); + report.parent_evidence_id = parentId; + report.parent_semantic_sha256 = hash(JSON.stringify(parent)); + report.answer = oracle.serde_rows; + assert.deepEqual( + await client.listTools(), + catalog, + "Advertisement changed after canonical materialization", + ); + report.checks.push( + "Catalog remains identical after canonical materialization", + ); + report.status = "passed"; } - const malformed = await call("inspect_rust_binary", { - path: values.artifact, - view: { collection: "packages", offset: -1 }, - }); - assert.equal(malformed.isError, true); - report.checks.push( - "Independent package rows and digest; repeated retained identity; canonical negative-offset refusal", - ); - const exported = await call("export_evidence_bundle", { - path: join(values.output, "evidence.json"), - }); - assert.notEqual(exported.isError, true); - const bundle = JSON.parse( - await readFile(join(values.output, "evidence.json"), "utf8"), - ); - const parent = bundle.records.find( - (record) => record.evidence_id === parentId, - ); - assert.ok(parent, "Complete original Evidence is retained"); - assert.equal(parent.subject.digest.sha256, artifactSha); - assert.deepEqual( - parent.normalized_result.auditable_metadata.packages, - oracle.packages, - ); - assert.deepEqual( - parent.normalized_result.auditable_metadata.dependency_edges, - oracle.edges, - ); - report.parent_evidence_id = parentId; - report.parent_semantic_sha256 = hash(JSON.stringify(parent)); - report.answer = oracle.serde_rows; - report.status = "passed"; } finally { await client.close(); await transport.close(); report.cleanup_verified = transport.pid === null; assert.ok(report.cleanup_verified); report.elapsed_ms = performance.now() - began; + if (expectedBinding !== undefined) { + const start = performance.now(); + const after = await captureRuntimeBinding(values.runtime, "mcp"); + report.runtime_audit_ms = { + before: auditBeforeMs, + after: performance.now() - start, + }; + assertRuntimeBinding(after, expectedBinding, values.role); + assert.deepEqual(after, bindingBefore); + assert.equal( + hash(await readFile(values.bindings)), + measurementInputsSha256, + ); + for (const [name, digest] of Object.entries(producerSources)) + assert.equal( + hash(await readFile(new URL(name, import.meta.url))), + digest, + ); + report.runtime_binding = after; + report.runtime_binding_verified = true; + } await writeFile( join(values.output, "report.json"), JSON.stringify(report, null, 2) + "\n", diff --git a/experiments/analysis-efficiency/mcp-startup-costs.py b/experiments/analysis-efficiency/mcp-startup-costs.py new file mode 100644 index 0000000000..33392d4bd5 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-startup-costs.py @@ -0,0 +1,118 @@ +"""Verify paired SDK startup/analysis receipts and account for the whole task.""" +import argparse +import hashlib +import importlib.util +import json +import statistics +from pathlib import Path + +import tiktoken + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('root', type=Path) +parser.add_argument('--output', type=Path, required=True) +args = parser.parse_args() +root = args.root +input_bytes = (root / 'measurement-inputs.json').read_bytes() +inputs = json.loads(input_bytes) +expected = inputs['runtime_bindings'] +sha = lambda value: hashlib.sha256(value).hexdigest() +compact = lambda value: json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) +encoding = tiktoken.get_encoding('cl100k_base') +count = lambda value: len(encoding.encode(value, disallowed_special=())) +for role in ['baseline', 'candidate']: + assert expected[role]['git_revision'] == inputs[f'{role}_revision'] + assert expected[role]['source_diff_sha256'] == sha(b'') + assert expected[role]['untracked_source_sha256'] == sha(b'[]') +assert expected['baseline']['root'] != expected['candidate']['root'] +assert expected['baseline']['compiled_runtime_sha256'] != expected['candidate']['compiled_runtime_sha256'] +spec = importlib.util.spec_from_file_location('catalog_costs', Path(__file__).with_name('mcp-input-catalog-costs.py')) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +catalogs = {} +identities = {} +rows = [] +for case in inputs['cases']: + directory = root / case['case'] + report = json.loads((directory / 'report.json').read_text()) + assert report['status'] == 'passed' and report['cleanup_verified'] + assert report['runtime_binding_verified'] + assert report['measurement_inputs_sha256'] == sha(input_bytes) + for name, producer_digest in report['producer_sources_sha256'].items(): + assert producer_digest == inputs['harness_sources_sha256'][name] + assert report['runtime_binding'] == expected[case['role']] + for key in ['role', 'cache', 'profile', 'phase']: + assert report[key] == case[key] + raw = (directory / 'catalog.json').read_bytes() + catalog_digest = sha(raw) + assert catalog_digest == report['catalog_sha256'] + profile = case['profile'] + if profile not in catalogs: + catalog = json.loads(raw) + definitions = [{key: tool[key] for key in ['name', 'description', 'inputSchema']} for tool in catalog['tools']] + catalogs[profile] = {'sha256': catalog_digest, 'tool_count': len(catalog['tools']), + 'input_catalog_tokens': count(compact(definitions)), + 'raw_catalog_tokens': count(raw.decode()), 'raw_catalog_bytes': len(raw)} + assert catalog_digest == catalogs[profile]['sha256'], 'Directory differs across roles or phases' + transcript = json.loads((directory / 'transcript.json').read_text()) + server = json.loads((directory / 'server.json').read_text()) + tokens = {'question_and_recipe': count(inputs['questions'][case['phase']] + '\n' + inputs['recipes'][case['phase']]), + 'server_instructions': count(server.get('instructions') or ''), + 'input_catalog_once': catalogs[profile]['input_catalog_tokens'], + 'arguments': sum(count(compact({'name': call['name'], 'arguments': call['arguments']})) for call in transcript), + 'text_replies': sum(count(block['text']) for call in transcript for block in call['reply']['content'] if block['type'] == 'text')} + requests = report['protocol_requests'] + assert requests['initialize'] == 1 + if case['phase'] == 'analysis': + assert len(transcript) == 5 and requests['tools/call'] == 5 and requests['tools/list'] == 2 + assert report['artifact_sha256'] == inputs['artifacts'][case['artifact']]['sha256'] + identity = (report['parent_evidence_id'], report['parent_semantic_sha256'], sha(compact(report['answer']).encode())) + assert identities.setdefault(case['artifact'], identity) == identity + assert sum(call['is_error'] for call in report['calls']) == 1 + else: + assert not transcript and requests.get('tools/call', 0) == 0 + assert requests['tools/list'] == 1 and requests['prompts/list'] == 1 + prompts = (directory / 'prompts.json').read_bytes() + assert sha(prompts) == report['prompts_sha256'] + assert identities.setdefault('prompts', report['prompts_sha256']) == report['prompts_sha256'] + tokens['prompt_directory'] = count(prompts.decode()) + tokens['whole_task'] = sum(tokens.values()) + resource = module.resource(directory) + for name, guard_digest in resource['guard_sources_sha256'].items(): + assert guard_digest == inputs['harness_sources_sha256'][name] + del resource['guard_sources_sha256'] + row = {**case, 'runtime_binding_sha256': sha(compact(report['runtime_binding']).encode()), 'runtime_audit_ms': report['runtime_audit_ms'], + 'catalog_sha256': catalog_digest, 'parent_evidence_id': report.get('parent_evidence_id'), + 'parent_semantic_sha256': report.get('parent_semantic_sha256'), + 'answer_sha256': sha(compact(report['answer']).encode()), 'tokens_estimated': tokens, + 'raw_catalog_result_tokens_all_lists': catalogs[profile]['raw_catalog_tokens'] * requests['tools/list'], + 'raw_catalog_result_bytes_all_lists': catalogs[profile]['raw_catalog_bytes'] * requests['tools/list'], + 'actual_model_usage': None, 'protocol_era': report['protocol_era'], 'protocol_requests': requests, 'calls': report['calls'], + **resource, + 'receipt_sha256': {name: sha((directory / name).read_bytes()) for name in ['report.json', 'catalog.json', 'server.json', 'transcript.json']}} + assert row['catalog_sha256'] == row['receipt_sha256']['catalog.json'] == catalogs[profile]['sha256'] + rows.append(row) +assert len(rows) == 72 +summary = [] +for phase in ['discovery', 'analysis']: + for cache in ['cold', 'warm']: + for profile in ['default', 'full']: + for artifact in ([None] if phase == 'discovery' else ['ripgrep', 'zellij']): + item = {'phase': phase, 'cache': cache, 'profile': profile, 'artifact': artifact} + for role in ['baseline', 'candidate']: + selected = [row for row in rows if all(row[key] == item[key] for key in ['phase', 'cache', 'profile', 'artifact']) and row['role'] == role] + assert len(selected) == 3 + item[role] = {metric: {'median': statistics.median(row[metric] for row in selected), + 'min': min(row[metric] for row in selected), 'max': max(row[metric] for row in selected)} + for metric in ['wall_seconds', 'cpu_seconds', 'kernel_family_peak_bytes_including_page_cache']} + item[role]['tokens_estimated'] = [row['tokens_estimated']['whole_task'] for row in selected] + summary.append(item) +result = {'status': 'passed', 'tokenizer': f'tiktoken=={tiktoken.__version__}/cl100k_base; estimates, not model usage', + 'aggregation_sources_sha256': {path.name: sha(path.read_bytes()) for path in [Path(__file__), Path(spec.origin)]}, + 'context_scope': 'Question, explicit recipe, server instructions, one complete name/description/input-schema directory, every tool argument and text reply, plus prompt directory for discovery. This projection assumes the client retains a single unchanged directory. Raw catalog result costs include every observed tools/list result separately, with output schemas and without JSON-RPC envelopes. SDK initialization/framing, reasoning and actual model usage are unmeasured. The analysis task includes retained-read, rejection, export/oracle and post-materialization directory checks.', + 'cache_scope': 'Cold disables server Node compile cache. Warm enables a prepopulated role/profile-specific cache after one complete analysis priming run. Fresh server/client processes and HOME each trial. Driver compile cache disabled in both cohorts. Cache hit rate and OS page cache are unmeasured; source/runtime audits warm compiled-file pages before startup.', + 'resource_scope': 'Whole SDK driver, resource observer and owned REA process family. Includes before/after source-and-all-compiled-JS/JSON binding, SDK schema checks, extra validation calls, bundle export and independent oracle checking. Kernel cgroup peak includes file cache. Build/download/oracle preparation and cache priming excluded. Retained V8 heap and forced-GC diagnostics were not separately measured.', + 'measurement_inputs': inputs, 'catalogs': catalogs, 'summary': summary, 'records': rows, + 'warmup_receipts': {name: sha((root / name / 'report.json').read_bytes()) for name in [f'warmup-{profile}-{role}' for profile in ['default', 'full'] for role in ['baseline', 'candidate']]}} +args.output.write_text(json.dumps(result, indent=2) + '\n') +print(json.dumps(summary, indent=2)) diff --git a/experiments/analysis-efficiency/mcp-startup.json b/experiments/analysis-efficiency/mcp-startup.json new file mode 100644 index 0000000000..03f21c91d5 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-startup.json @@ -0,0 +1,6172 @@ +{ + "status": "passed", + "tokenizer": "tiktoken==0.14.0/cl100k_base; estimates, not model usage", + "aggregation_sources_sha256": { + "mcp-startup-costs.py": "043eeb5014601a000dc13c23db710e67892867145fd49fdfbcfee24b9d4ef918", + "mcp-input-catalog-costs.py": "ad32e6726f892be16cc7d352118f7e9f9f0a85649d9edd1049b5d73483df8078" + }, + "context_scope": "Question, explicit recipe, server instructions, one complete name/description/input-schema directory, every tool argument and text reply, plus prompt directory for discovery. This projection assumes the client retains a single unchanged directory. Raw catalog result costs include every observed tools/list result separately, with output schemas and without JSON-RPC envelopes. SDK initialization/framing, reasoning and actual model usage are unmeasured. The analysis task includes retained-read, rejection, export/oracle and post-materialization directory checks.", + "cache_scope": "Cold disables server Node compile cache. Warm enables a prepopulated role/profile-specific cache after one complete analysis priming run. Fresh server/client processes and HOME each trial. Driver compile cache disabled in both cohorts. Cache hit rate and OS page cache are unmeasured; source/runtime audits warm compiled-file pages before startup.", + "resource_scope": "Whole SDK driver, resource observer and owned REA process family. Includes before/after source-and-all-compiled-JS/JSON binding, SDK schema checks, extra validation calls, bundle export and independent oracle checking. Kernel cgroup peak includes file cache. Build/download/oracle preparation and cache priming excluded. Retained V8 heap and forced-GC diagnostics were not separately measured.", + "measurement_inputs": { + "baseline_revision": "c465ff5a814b8e1a386fc547cbcd94df5c35f086", + "candidate_revision": "04900abbcd3c3438fce4af932ff28400fe744a7d", + "runtime_bindings": { + "baseline": { + "binding_version": 2, + "root": "/home/yann/yann/rea-4424-main-control-20261022", + "node": "v24.18.0", + "git_revision": "c465ff5a814b8e1a386fc547cbcd94df5c35f086", + "source_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "untracked_source_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "compiled_runtime_sha256": "4719eb8f317c72855fc06a0ace5cb73eccd71146a4b03e14b0a0f19206c2db67", + "compiled_runtime_files": 1980, + "compiled_runtime_bytes": 32014298 + }, + "candidate": { + "binding_version": 2, + "root": "/home/yann/yann/rea-perf-mcp-startup-20261022", + "node": "v24.18.0", + "git_revision": "04900abbcd3c3438fce4af932ff28400fe744a7d", + "source_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "untracked_source_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "compiled_runtime_sha256": "186038c068ea62da11b6e92cf15f67e2a41291680998a39815435d3b31db2da3", + "compiled_runtime_files": 1989, + "compiled_runtime_bytes": 38602472 + } + }, + "artifacts": { + "ripgrep": { + "path": "/tmp/rea-perf-rust-20261021/ripgrep", + "sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "bytes": 3416168, + "oracle": "/tmp/rea-perf-mcp-catalog-20261021/ripgrep-oracle.json", + "oracle_sha256": "2b90d656b671f43d042a9efbc33f155e47ba8860a6c76a6b33bb60f54d2e2f8e" + }, + "zellij": { + "path": "/tmp/rea-perf-rust-20261021/zellij", + "sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "bytes": 17151864, + "oracle": "/tmp/rea-perf-mcp-catalog-20261021/zellij-oracle.json", + "oracle_sha256": "ceffe670551105d159004591a8e060588a022b19231e43d9de6a9a2e6f514783" + } + }, + "cases": [ + { + "case": "pair-cold-default-discovery-directory-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-default-discovery-directory-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-default-discovery-directory-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-default-discovery-directory-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-default-discovery-directory-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-default-discovery-directory-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-ripgrep-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-ripgrep-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-ripgrep-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-ripgrep-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-ripgrep-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-ripgrep-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-zellij-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-zellij-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-zellij-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-default-analysis-zellij-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-zellij-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-default-analysis-zellij-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-cold-full-discovery-directory-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-full-discovery-directory-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-full-discovery-directory-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-full-discovery-directory-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-full-discovery-directory-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-full-discovery-directory-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-ripgrep-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-ripgrep-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-ripgrep-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-ripgrep-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-ripgrep-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-ripgrep-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-zellij-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-zellij-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-zellij-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-cold-full-analysis-zellij-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-zellij-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-cold-full-analysis-zellij-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-default-discovery-directory-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-default-discovery-directory-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-default-discovery-directory-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-default-discovery-directory-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-default-discovery-directory-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-default-discovery-directory-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-ripgrep-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-ripgrep-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-ripgrep-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-ripgrep-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-ripgrep-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-ripgrep-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-zellij-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-zellij-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-zellij-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-default-analysis-zellij-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-zellij-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-default-analysis-zellij-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-full-discovery-directory-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-full-discovery-directory-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-full-discovery-directory-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-full-discovery-directory-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-full-discovery-directory-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-full-discovery-directory-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-ripgrep-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-ripgrep-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-ripgrep-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-ripgrep-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-ripgrep-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-ripgrep-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-zellij-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-zellij-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-zellij-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate" + }, + { + "case": "pair-warm-full-analysis-zellij-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-zellij-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline" + }, + { + "case": "pair-warm-full-analysis-zellij-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate" + } + ], + "acquisition": [ + { + "package": "ripgrep-14.1.1-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/ripgrep-14.1.1-r0.apk", + "package_sha256": "213dd0b047163f6627cc2fdd6fbfe205009476eb4fc48c352a3f5b9ab647f6d4", + "package_bytes": 1422830, + "member": "usr/bin/rg", + "bytes": 3416168, + "sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "aports_revision": "4e899d119e4c983a938436d6f397a509521225d1" + }, + { + "package": "zellij-0.40.0-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/zellij-0.40.0-r0.apk", + "package_sha256": "575075945415b221857563e19c5f4c661af9f98bbba54f0994c7385c0e432432", + "package_bytes": 6722129, + "member": "usr/bin/zellij", + "bytes": 17151864, + "sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "aports_revision": "3be26a3a26e76e1bf3eff784bffe3956d29d77a0" + } + ], + "independent_oracles": { + "ripgrep": { + "artifact_sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "section_sha256": "02b3cb036955be58bf4bd51710fcdec96a1f1a771e1bb7d1397d604557c2a6a7", + "json_sha256": "ea478aa80f91aa8f5159aba648ddfdda642fa2c98759e54e65623a19065634be", + "serde_rows": [ + { + "index": 36, + "name": "serde", + "version": "1.0.210", + "kind": null + } + ] + }, + "zellij": { + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "section_sha256": "d0cabaf40a2875c519f68788547e484865463beeacf56bb01defa74253b4ba72", + "json_sha256": "9bcf8d694440a7e064189616aa40b69046e32ee1c70683ef2e0778150b9163ef", + "serde_rows": [ + { + "index": 294, + "name": "serde", + "version": "1.0.137", + "kind": null + } + ] + } + }, + "harness_sources_sha256": { + "mcp-input-catalog.mjs": "a19530514d49f98ff550e05fbb69661b3b48e808189efe81eab6032425322adc", + "managed-cli-runtime.mjs": "48c2a5fd617dccc10b8d36b01bc5c3d149758e1225008c931759e17ccbe83db4", + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "questions": { + "analysis": "Which serde package versions and package kinds occur in the selected Rust binary? Preserve complete source Evidence and verify retained reads.", + "discovery": "Discover the complete REA tool and prompt directory." + }, + "recipes": { + "analysis": "Initialize; list tools; inspect_rust_binary selecting serde name/version/kind; read the exact retained parent twice; verify canonical rejection of a negative offset; export and independently check all packages and edges; confirm the catalog remains identical; close.", + "discovery": "Initialize; list the complete tools and prompts; validate input schema dialect and unique names; close." + }, + "orchestrator_sha256": "5d42540a5d7120cd5eeab4d3fa1d40b44d47fc5f96e9d0fa675e2472cd0cdda6" + }, + "catalogs": { + "default": { + "sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "tool_count": 213, + "input_catalog_tokens": 49670, + "raw_catalog_tokens": 756224, + "raw_catalog_bytes": 2935459 + }, + "full": { + "sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "tool_count": 213, + "input_catalog_tokens": 205744, + "raw_catalog_tokens": 912495, + "raw_catalog_bytes": 3588103 + } + }, + "summary": [ + { + "phase": "discovery", + "cache": "cold", + "profile": "default", + "artifact": null, + "baseline": { + "wall_seconds": { + "median": 5.661, + "min": 5.257, + "max": 6.573 + }, + "cpu_seconds": { + "median": 5.402669, + "min": 5.106671, + "max": 6.396994 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 586420224, + "min": 583921664, + "max": 588685312 + }, + "tokens_estimated": [50840, 50840, 50840] + }, + "candidate": { + "wall_seconds": { + "median": 2.236, + "min": 2.235, + "max": 2.237 + }, + "cpu_seconds": { + "median": 2.049533, + "min": 2.013812, + "max": 2.051896 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 260640768, + "min": 259063808, + "max": 265650176 + }, + "tokens_estimated": [50840, 50840, 50840] + } + }, + { + "phase": "discovery", + "cache": "cold", + "profile": "full", + "artifact": null, + "baseline": { + "wall_seconds": { + "median": 5.256, + "min": 5.16, + "max": 5.257 + }, + "cpu_seconds": { + "median": 5.039555, + "min": 4.941541, + "max": 5.039595 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 682528768, + "min": 678322176, + "max": 686043136 + }, + "tokens_estimated": [206914, 206914, 206914] + }, + "candidate": { + "wall_seconds": { + "median": 2.337, + "min": 2.336, + "max": 2.437 + }, + "cpu_seconds": { + "median": 2.174622, + "min": 2.141803, + "max": 2.201697 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 366006272, + "min": 362889216, + "max": 366518272 + }, + "tokens_estimated": [206914, 206914, 206914] + } + }, + { + "phase": "discovery", + "cache": "warm", + "profile": "default", + "artifact": null, + "baseline": { + "wall_seconds": { + "median": 5.056, + "min": 4.954, + "max": 5.058 + }, + "cpu_seconds": { + "median": 4.799916, + "min": 4.776322, + "max": 4.818559 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 600616960, + "min": 600211456, + "max": 600797184 + }, + "tokens_estimated": [50840, 50840, 50840] + }, + "candidate": { + "wall_seconds": { + "median": 2.236, + "min": 2.135, + "max": 2.236 + }, + "cpu_seconds": { + "median": 2.014862, + "min": 1.986985, + "max": 2.025594 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 263593984, + "min": 256937984, + "max": 264220672 + }, + "tokens_estimated": [50840, 50840, 50840] + } + }, + { + "phase": "discovery", + "cache": "warm", + "profile": "full", + "artifact": null, + "baseline": { + "wall_seconds": { + "median": 4.957, + "min": 4.952, + "max": 5.054 + }, + "cpu_seconds": { + "median": 4.753724, + "min": 4.713301, + "max": 4.806174 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 705363968, + "min": 701562880, + "max": 707497984 + }, + "tokens_estimated": [206914, 206914, 206914] + }, + "candidate": { + "wall_seconds": { + "median": 2.338, + "min": 2.336, + "max": 2.34 + }, + "cpu_seconds": { + "median": 2.138658, + "min": 2.106354, + "max": 2.151271 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 366624768, + "min": 366333952, + "max": 372981760 + }, + "tokens_estimated": [206914, 206914, 206914] + } + }, + { + "phase": "analysis", + "cache": "cold", + "profile": "default", + "artifact": "ripgrep", + "baseline": { + "wall_seconds": { + "median": 8.177, + "min": 8.177, + "max": 8.382 + }, + "cpu_seconds": { + "median": 7.94501, + "min": 7.907789, + "max": 8.122413 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 890929152, + "min": 885346304, + "max": 891183104 + }, + "tokens_estimated": [54027, 54027, 54027] + }, + "candidate": { + "wall_seconds": { + "median": 8.078, + "min": 7.069, + "max": 8.279 + }, + "cpu_seconds": { + "median": 7.899241, + "min": 6.847499, + "max": 8.095031 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 830971904, + "min": 821231616, + "max": 845656064 + }, + "tokens_estimated": [54027, 54027, 54027] + } + }, + { + "phase": "analysis", + "cache": "cold", + "profile": "default", + "artifact": "zellij", + "baseline": { + "wall_seconds": { + "median": 8.276, + "min": 8.178, + "max": 8.279 + }, + "cpu_seconds": { + "median": 8.040952, + "min": 7.96754, + "max": 8.079063 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 887341056, + "min": 864006144, + "max": 919195648 + }, + "tokens_estimated": [53947, 53947, 53947] + }, + "candidate": { + "wall_seconds": { + "median": 7.276, + "min": 7.27, + "max": 7.472 + }, + "cpu_seconds": { + "median": 7.104698, + "min": 7.053558, + "max": 7.223805 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 858734592, + "min": 858263552, + "max": 862248960 + }, + "tokens_estimated": [53947, 53947, 53947] + } + }, + { + "phase": "analysis", + "cache": "cold", + "profile": "full", + "artifact": "ripgrep", + "baseline": { + "wall_seconds": { + "median": 8.18, + "min": 8.078, + "max": 8.286 + }, + "cpu_seconds": { + "median": 8.000206, + "min": 7.8649, + "max": 8.087954 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 887574528, + "min": 883621888, + "max": 894648320 + }, + "tokens_estimated": [210101, 210101, 210101] + }, + "candidate": { + "wall_seconds": { + "median": 7.483, + "min": 7.474, + "max": 7.576 + }, + "cpu_seconds": { + "median": 7.311571, + "min": 7.251707, + "max": 7.364513 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 832987136, + "min": 830832640, + "max": 833130496 + }, + "tokens_estimated": [210101, 210101, 210101] + } + }, + { + "phase": "analysis", + "cache": "cold", + "profile": "full", + "artifact": "zellij", + "baseline": { + "wall_seconds": { + "median": 8.484, + "min": 8.382, + "max": 8.687 + }, + "cpu_seconds": { + "median": 8.284427, + "min": 8.205329, + "max": 8.457721 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 891723776, + "min": 886362112, + "max": 893550592 + }, + "tokens_estimated": [210021, 210021, 210021] + }, + "candidate": { + "wall_seconds": { + "median": 7.776, + "min": 7.58, + "max": 8.692 + }, + "cpu_seconds": { + "median": 7.513651, + "min": 7.38469, + "max": 8.46357 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 852561920, + "min": 842448896, + "max": 866377728 + }, + "tokens_estimated": [210021, 210021, 210021] + } + }, + { + "phase": "analysis", + "cache": "warm", + "profile": "default", + "artifact": "ripgrep", + "baseline": { + "wall_seconds": { + "median": 7.976, + "min": 7.779, + "max": 8.187 + }, + "cpu_seconds": { + "median": 7.738718, + "min": 7.595732, + "max": 7.936629 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 867004416, + "min": 866107392, + "max": 894771200 + }, + "tokens_estimated": [54027, 54027, 54027] + }, + "candidate": { + "wall_seconds": { + "median": 6.879, + "min": 6.868, + "max": 7.996 + }, + "cpu_seconds": { + "median": 6.678343, + "min": 6.666864, + "max": 7.809846 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 841535488, + "min": 828764160, + "max": 843214848 + }, + "tokens_estimated": [54027, 54027, 54027] + } + }, + { + "phase": "analysis", + "cache": "warm", + "profile": "default", + "artifact": "zellij", + "baseline": { + "wall_seconds": { + "median": 8.285, + "min": 8.081, + "max": 8.583 + }, + "cpu_seconds": { + "median": 8.019149, + "min": 7.888906, + "max": 8.393267 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 924995584, + "min": 896237568, + "max": 925614080 + }, + "tokens_estimated": [53947, 53947, 53947] + }, + "candidate": { + "wall_seconds": { + "median": 7.472, + "min": 7.071, + "max": 7.573 + }, + "cpu_seconds": { + "median": 7.254133, + "min": 6.85764, + "max": 7.324238 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 850833408, + "min": 839819264, + "max": 864182272 + }, + "tokens_estimated": [53947, 53947, 53947] + } + }, + { + "phase": "analysis", + "cache": "warm", + "profile": "full", + "artifact": "ripgrep", + "baseline": { + "wall_seconds": { + "median": 8.074, + "min": 7.976, + "max": 8.282 + }, + "cpu_seconds": { + "median": 7.889965, + "min": 7.768468, + "max": 8.06946 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 893456384, + "min": 892899328, + "max": 908144640 + }, + "tokens_estimated": [210101, 210101, 210101] + }, + "candidate": { + "wall_seconds": { + "median": 7.476, + "min": 7.271, + "max": 7.575 + }, + "cpu_seconds": { + "median": 7.264899, + "min": 7.011412, + "max": 7.372084 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 842661888, + "min": 839393280, + "max": 847982592 + }, + "tokens_estimated": [210101, 210101, 210101] + } + }, + { + "phase": "analysis", + "cache": "warm", + "profile": "full", + "artifact": "zellij", + "baseline": { + "wall_seconds": { + "median": 8.382, + "min": 8.279, + "max": 8.582 + }, + "cpu_seconds": { + "median": 8.150593, + "min": 8.123721, + "max": 8.332648 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 895754240, + "min": 894513152, + "max": 896585728 + }, + "tokens_estimated": [210021, 210021, 210021] + }, + "candidate": { + "wall_seconds": { + "median": 7.479, + "min": 7.474, + "max": 7.577 + }, + "cpu_seconds": { + "median": 7.292747, + "min": 7.231771, + "max": 7.373364 + }, + "kernel_family_peak_bytes_including_page_cache": { + "median": 870350848, + "min": 867786752, + "max": 882143232 + }, + "tokens_estimated": [210021, 210021, 210021] + } + } + ], + "records": [ + { + "case": "pair-cold-default-discovery-directory-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 506.37848699999995, + "after": 383.48330799999985 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 6.573, + "cpu_seconds": 6.396994, + "kernel_family_peak_bytes_including_page_cache": 586420224, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "c7668ba37a7331930951010568ee9c8c50921e4f6f0fd8142612742087151682", + "receipt_sha256": { + "report.json": "fb7ab0c6dba3e5d7178955914d464d1a5b8a1bf7c550b22e71bf8c30d4546278", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-discovery-directory-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 393.868538, + "after": 388.9534389999999 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.237, + "cpu_seconds": 2.051896, + "kernel_family_peak_bytes_including_page_cache": 265650176, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "df2a582c9fb0a457c91a7fc5f38170e8fd802efc380b30fea3e5b4d7814fa529", + "receipt_sha256": { + "report.json": "2cb6fdb5a5467acaf86f8643c73acb1f258583b14a2053bd7fdeab490a5c2eef", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-discovery-directory-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 408.59600800000004, + "after": 373.9158789999999 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.236, + "cpu_seconds": 2.049533, + "kernel_family_peak_bytes_including_page_cache": 260640768, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "0ed30068080e71715ea3f57530add80446d64015f339f31e3af55c60e96e6a25", + "receipt_sha256": { + "report.json": "c2ec173344072f7eeac2b55dbafa201d86159dba94525a10884d799352857a04", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-discovery-directory-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 392.12916799999994, + "after": 395.4477969999998 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.661, + "cpu_seconds": 5.402669, + "kernel_family_peak_bytes_including_page_cache": 588685312, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "d1d1b6d8d40014bf9a28075a9cf550150c53164fe6a5dd8899a8e87ea5409852", + "receipt_sha256": { + "report.json": "72124bca106ee6484e6ca3d5d7a33369b6b9b091157ce9a89948fbbba092ce8d", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-discovery-directory-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 395.983488, + "after": 383.51651799999945 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.257, + "cpu_seconds": 5.106671, + "kernel_family_peak_bytes_including_page_cache": 583921664, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "2e9b150e2a69174623bad00bb1c94438c178f4e1fa256c74311901084fba7d63", + "receipt_sha256": { + "report.json": "8f0a1230d583b47f6eecd04d4053c8db60516ddb318bf96580869edd11d2282e", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-discovery-directory-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 398.496408, + "after": 374.36179800000014 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.235, + "cpu_seconds": 2.013812, + "kernel_family_peak_bytes_including_page_cache": 259063808, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "d8229a516e78a474d91903d37a92eb86b1d636ab6dc7c3f3ff1389d18f890104", + "receipt_sha256": { + "report.json": "1696adbd07d42271b59369609e3d2403f4181b6f8bd0cdc39e24f212ceab8ec7", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 388.44587900000005, + "after": 384.4485880000002 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 231.50504900000033, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2466.3926060000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.4670500000002, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.448030000000472, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.455298999999286, + "is_error": false + } + ], + "wall_seconds": 8.382, + "cpu_seconds": 8.122413, + "kernel_family_peak_bytes_including_page_cache": 891183104, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "821ff65d255e67a66810ea43d8c410c0f7040938cbfce1339483272277869bfd", + "receipt_sha256": { + "report.json": "5d13b34fdb734cb701b1f1bf4ac10e6d28605e610bfdc558ccd02b7660ee5f33", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "5339ea61a140c1651b9c0722df5a3e857a6d6e096f262195290256d21a281d94" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 399.96640700000006, + "after": 373.44509800000014 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2656.1985050000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2883.0528130000002, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 53.92411000000084, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 7.510129999999663, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 23.69365999999991, + "is_error": false + } + ], + "wall_seconds": 8.279, + "cpu_seconds": 8.095031, + "kernel_family_peak_bytes_including_page_cache": 845656064, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "35cbb9eceb49d203ab8191fea0d94d5583caff270164158adef2f0401c47f962", + "receipt_sha256": { + "report.json": "e13f3dc68b7e37869c126307e5bc9b570cf6eb92952a512e3fd87bed55bd661e", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "bbed26a043f35cac222d04aa1e48cca681e2f75c1b1f7505985e5b8ef64b0e16" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 390.77658700000006, + "after": 372.5078680000006 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2198.601397, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2157.017327, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 78.7398489999996, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.584418999999798, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 20.19941000000017, + "is_error": false + } + ], + "wall_seconds": 7.069, + "cpu_seconds": 6.847499, + "kernel_family_peak_bytes_including_page_cache": 830971904, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "646ebb6b433a1318ac041d26a04cd3ee606a8536f6038276f161401af26ee2c2", + "receipt_sha256": { + "report.json": "f571c9f4e5e39576c9fd59167edbc7699d62794f1f298ca561a65ac4f0dffb1b", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "811f2044dd4818bda84e3346866aee9911adce1dc04d3d1d8fc48693eee441d3" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 399.96877700000005, + "after": 378.63447799999994 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 235.36689800000022, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2301.830806, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.607820000000174, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.062179999999898, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 25.012810000000172, + "is_error": false + } + ], + "wall_seconds": 8.177, + "cpu_seconds": 7.94501, + "kernel_family_peak_bytes_including_page_cache": 885346304, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "05f1fe6e4ee56fbf966ddaf42dfa7e3adab10ad6a93ea332ff6e9107df50de4e", + "receipt_sha256": { + "report.json": "ea40226a59beb69a6f32aaf011cd0b96e706e8a612cfeaab522aa5b237440042", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "fb8310b446516dcac1302c1b888d2d9d75d456d523a7b73bf01d57f508ee3de7" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 395.456087, + "after": 387.4231770000006 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 217.739098, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2328.010265, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.894239999999627, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.41475999999966, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 24.567889999999352, + "is_error": false + } + ], + "wall_seconds": 8.177, + "cpu_seconds": 7.907789, + "kernel_family_peak_bytes_including_page_cache": 890929152, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "3b0952e942e50b2ce0c233097a6cb6f6076e6e6092996996ed2bba53cc64f7ba", + "receipt_sha256": { + "report.json": "c17423e18d14076675f28d5c253c6cfdd8d7c6594fa93183faef69ce4406b0e8", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "dbd127d13cb5d2328242587f3fbad282babf6644ea28b8fed12104e27bbb0ec1" + } + }, + { + "case": "pair-cold-default-analysis-ripgrep-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 392.1519870000001, + "after": 374.93381800000043 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2532.116923, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2948.28848, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 10.22776999999951, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.834399999999732, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 25.64838999999938, + "is_error": false + } + ], + "wall_seconds": 8.078, + "cpu_seconds": 7.899241, + "kernel_family_peak_bytes_including_page_cache": 821231616, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "c0a00be8db0031e72ba2250623302f31751a8b423ed65335eb519f51de25d28c", + "receipt_sha256": { + "report.json": "f027377ed2e589a86213f9d1708aa04d05c1e97def74d65819d5d316accbd326", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "6350b40c9b4bf6b3b0bda7ee01e584c5aa27b5b2c23f1da2a925d79bdfcc0995" + } + }, + { + "case": "pair-cold-default-analysis-zellij-1-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 380.05093700000003, + "after": 368.26613799999996 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 389.6055860000006, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2251.2189150000004, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.268479999999727, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.4712799999997515, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.36559000000034, + "is_error": false + } + ], + "wall_seconds": 8.279, + "cpu_seconds": 8.040952, + "kernel_family_peak_bytes_including_page_cache": 864006144, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "82105544ae0f621fc040d10391e1b10f2a71086011f79ef96cfe05e8ef27b760", + "receipt_sha256": { + "report.json": "99f11edf3a0676b5924983927c26f0a7234b4536641a5aacb05ed2336dd13ad0", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "26cf65bb76b419efe5151301ed837fb4417c084ef2e52ff38d740c98008735a8" + } + }, + { + "case": "pair-cold-default-analysis-zellij-1-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 396.72964700000006, + "after": 384.7991270000002 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2374.6167530000002, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2304.7313240000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 12.59936000000016, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.993188999999802, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.42609999999968, + "is_error": false + } + ], + "wall_seconds": 7.27, + "cpu_seconds": 7.104698, + "kernel_family_peak_bytes_including_page_cache": 858734592, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "03372a596886853b90968c4fc9491a4bbf415b223c63d94a944e88f313d9fd6a", + "receipt_sha256": { + "report.json": "e88b369c8a9736102edd4f8406bcb7df4cd814d1a6f5868b2f85142dafaa7cbe", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "6b1925b000a6f1f223cef05e4c6cb48388b95ab1f407d6bdeb5b28e4cbfb447d" + } + }, + { + "case": "pair-cold-default-analysis-zellij-2-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 383.530408, + "after": 380.42205699999977 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2361.3623230000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2270.282083, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.830868999999439, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.383009999999558, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.06808000000001, + "is_error": false + } + ], + "wall_seconds": 7.276, + "cpu_seconds": 7.053558, + "kernel_family_peak_bytes_including_page_cache": 858263552, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b114701046a6d5da6cdad4b64c88b0c2bce1e04d44d40d385cb98f732aa46ec2", + "receipt_sha256": { + "report.json": "97c29f32822008f92e0464e2e7d12288486e54a3f8803cb5a96f2714a2a1c230", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "703879dca8faa155f1fdccdacb0dcee80a44a60068ff5cce9d33feff8414e8ad" + } + }, + { + "case": "pair-cold-default-analysis-zellij-2-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 387.22730600000006, + "after": 380.939026 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 372.1123379999999, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2314.2110820000007, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.857170000000224, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.183610000000044, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.596669000000475, + "is_error": false + } + ], + "wall_seconds": 8.178, + "cpu_seconds": 7.96754, + "kernel_family_peak_bytes_including_page_cache": 887341056, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "4c4e7fb9f5e85bb506473e64070f08b6554ff87e9b56adcb2c64601d791a3d22", + "receipt_sha256": { + "report.json": "e767b4fe87aac935c325d6328b77b110bc1cb6e73a93e1b479dcc0eee31e2709", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "6ee687998ecf1abcaf1126a106d86351416710f700ebddc43a669f0accb54e5e" + } + }, + { + "case": "pair-cold-default-analysis-zellij-3-baseline", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 392.189227, + "after": 378.3951070000003 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 361.38807799999995, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2265.5642610000004, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.350690000000213, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.178849999999329, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.60490000000027, + "is_error": false + } + ], + "wall_seconds": 8.276, + "cpu_seconds": 8.079063, + "kernel_family_peak_bytes_including_page_cache": 919195648, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "8eb380a496944a5c1a2162ae8aacbb1921c1710f5c05d0ee1aa5ac08b2e9ed78", + "receipt_sha256": { + "report.json": "5f6b836bcd43f28e1a1e9a17a1fcc47b1a376a5d391cb4fea57a0dbbaa671baf", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "dc07e0fec3b71d21f60090d68c56e8c237102f5f53d63cca0312e375e90e9999" + } + }, + { + "case": "pair-cold-default-analysis-zellij-3-candidate", + "cache": "cold", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 411.829166, + "after": 384.4884069999998 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2413.6837109999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2322.668941, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 11.807480000000396, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 7.356169999999111, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 32.13658899999973, + "is_error": false + } + ], + "wall_seconds": 7.472, + "cpu_seconds": 7.223805, + "kernel_family_peak_bytes_including_page_cache": 862248960, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "6bf25029d1ac93bd04bbdffe2a959b9d2542e8b74cfa1083ca5cc615d0d351c7", + "receipt_sha256": { + "report.json": "61b127859f0b058768295380fe3bc035509b98b2f3ef0b5b2bae70ac3648d808", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "735efa21da33b0234db3819e9c2271a8aecb29dffee9b75a3229f9064c634cf5" + } + }, + { + "case": "pair-cold-full-discovery-directory-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 398.755417, + "after": 381.78022599999986 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.257, + "cpu_seconds": 5.039595, + "kernel_family_peak_bytes_including_page_cache": 686043136, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "071f77920018c98368ee7573e06b0442dfbd6f656edd2aba331235f8404f05f6", + "receipt_sha256": { + "report.json": "171ae75d92ea4b0f6f3cfa9c43a660bc6ef38213e7b2491175dc479a2588dda5", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-discovery-directory-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 388.91068700000005, + "after": 388.7231469999999 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.437, + "cpu_seconds": 2.201697, + "kernel_family_peak_bytes_including_page_cache": 362889216, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "85b0ffff4f47833c418d9f3fefa057e479d4447663fa81777df36a79b78f7e2f", + "receipt_sha256": { + "report.json": "e2b94eb8f149f06aec634578aa97016b9512ecab98198bc66bf8aa320dc7527a", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-discovery-directory-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 392.689287, + "after": 375.5382760000002 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.337, + "cpu_seconds": 2.141803, + "kernel_family_peak_bytes_including_page_cache": 366006272, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "ffec34115a74cfd3ae85edf60f4995c2524a7110a60ccc25792905cd9ad70329", + "receipt_sha256": { + "report.json": "98e147170a654b7f1e530b05f9104e4eb53e85f602f720dba50aebbe7bd7f8e0", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-discovery-directory-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 376.18074700000005, + "after": 371.696586 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.16, + "cpu_seconds": 4.941541, + "kernel_family_peak_bytes_including_page_cache": 678322176, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "73fb02fadf0eca36a860898c3b1d67d62eb2c0b56638c84bffebe48832dbcc6b", + "receipt_sha256": { + "report.json": "d6f9a0f9703dba6ac6a12d77215c6565a9fef6e04fa95b24284e6101d51f3965", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-discovery-directory-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 397.05393599999996, + "after": 383.94540699999925 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.256, + "cpu_seconds": 5.039555, + "kernel_family_peak_bytes_including_page_cache": 682528768, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "e94bad751fe24587812fc4fff7aa471c91cfc21481a4faa7da47ba2d38f1e14d", + "receipt_sha256": { + "report.json": "6479290a9c61dd5ed825c93aa86c5be1c40add30bb89b08b3b37601a7697c594", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-discovery-directory-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 401.086936, + "after": 363.90771599999994 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.336, + "cpu_seconds": 2.174622, + "kernel_family_peak_bytes_including_page_cache": 366518272, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "4e186b9ed1c0f0f72be305bbbae32f3889af307f775e7430aebaf45bb8a2f9ff", + "receipt_sha256": { + "report.json": "9f3ccc333fe2127c5f6dad58c54fae19ffac7d033061eae50864d9bc310965e8", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 390.21384700000004, + "after": 392.29759600000034 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 219.64897800000017, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2195.1449299999995, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 84.64082900000085, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.840389999999388, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.56847999999991, + "is_error": false + } + ], + "wall_seconds": 8.286, + "cpu_seconds": 8.087954, + "kernel_family_peak_bytes_including_page_cache": 894648320, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "c3d3f42167024b24491960ad879970a28c46e1c35fb98eefa1dc9706b714255d", + "receipt_sha256": { + "report.json": "1b3e5ca01f7d0918b825dc54f138ac39e703c76ee165db4062d2e1836ab4a77f", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "b1dc03149f6a24e26bafe636bed86ccd59d914b1d41c2c655734a588f028ecc6" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 414.19911599999995, + "after": 382.60509600000023 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2228.02056, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2190.5758189999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 72.02308900000025, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 6.812190999999984, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 25.61801899999955, + "is_error": false + } + ], + "wall_seconds": 7.576, + "cpu_seconds": 7.364513, + "kernel_family_peak_bytes_including_page_cache": 830832640, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "cb95809408678209af4ae651fa45bf188dca4ef90d9a7347a28b5e7b9bb42c04", + "receipt_sha256": { + "report.json": "2ec937bf0fea0bf1219d891a7c1fa94dd8442cec92e8a1430c5bc3cad82e9812", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "ab204801a26bd06de23b4f347fdb452c30ea8bdb264147f66a0e838dbcd1503c" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 387.43815700000005, + "after": 397.392296 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2272.420779, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2150.4206000000004, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 69.52158899999995, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.2822299999997995, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 18.803679999999986, + "is_error": false + } + ], + "wall_seconds": 7.474, + "cpu_seconds": 7.251707, + "kernel_family_peak_bytes_including_page_cache": 833130496, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b54c0b5c81f102981e056b6dafd6755f6b0b79d76530d646601104602f91b6c5", + "receipt_sha256": { + "report.json": "86bd2ad471fa2cedbc4219a03dd5175f46ae98773b4ce8b882a5df37f66a1084", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "c7e7e9c0ea472b9194ddc11697c4e31ade5154d7739149b2a8d43bf155505e99" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 391.875397, + "after": 385.31773599999997 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 222.41961700000047, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2118.6533789999994, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 75.26427899999999, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.1229990000001635, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 19.23714999999993, + "is_error": false + } + ], + "wall_seconds": 8.18, + "cpu_seconds": 8.000206, + "kernel_family_peak_bytes_including_page_cache": 887574528, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "2a79aa8b13ce99a07d48458584100d260d39e2ed77a39ffce3b7783f2bdb8705", + "receipt_sha256": { + "report.json": "1d7e4a1e1ee5fadc73853d6fa0a9a0079074a1a3e7359135af6a0bb75d8476b6", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "888ac9aa4760c7fc00ec3b3f3d35d7bf0c55d5aae3ffca62353aa330d49b8e66" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 385.66209599999996, + "after": 384.93557600000076 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 219.4574480000001, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2087.11531, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 68.6765489999998, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.761439999999311, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 18.226380000000063, + "is_error": false + } + ], + "wall_seconds": 8.078, + "cpu_seconds": 7.8649, + "kernel_family_peak_bytes_including_page_cache": 883621888, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "d0e53b3991411463031a234eaa178c95fb063c7adfa3946a72d2a4ede136ce6e", + "receipt_sha256": { + "report.json": "a3ed4be444d411650b83ac93e94e092d3c155c27f9cf38125d8193c8945aa9a3", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "3417b64694c374e68552f6ac354ee4ced777262536f3375e4beea5a3127ca391" + } + }, + { + "case": "pair-cold-full-analysis-ripgrep-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 395.39197599999994, + "after": 377.38734600000043 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2300.851867, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2178.248269, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 65.45291899999938, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.002019999999902, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 26.409888999999566, + "is_error": false + } + ], + "wall_seconds": 7.483, + "cpu_seconds": 7.311571, + "kernel_family_peak_bytes_including_page_cache": 832987136, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "13f2517f60fbca6e17c5d651a6c37078f5645ec03f4cbfb17f34680106b3314a", + "receipt_sha256": { + "report.json": "750e4154a0b1b36c98a549ee886eb20c04962e24b9afbdf78281eef2c6dccb1c", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "bbec0a2cdb24051b80a697b37c5751f147f368e9fb8d62438ebea77e40cd9bfb" + } + }, + { + "case": "pair-cold-full-analysis-zellij-1-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 380.24241600000005, + "after": 392.5702259999998 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 348.6446270000006, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2148.8615090000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 74.70148900000004, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.575309000000743, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 24.902938999999606, + "is_error": false + } + ], + "wall_seconds": 8.382, + "cpu_seconds": 8.205329, + "kernel_family_peak_bytes_including_page_cache": 893550592, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "95087ae47246162c06b639daca498250646fbd0c384870923d4ca5266d63889f", + "receipt_sha256": { + "report.json": "e04d5c06e52dddd25736e14c7b9796f65e0a8c493ddbd7b4bc3d72bd0e9bf802", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "48482171f87d65ec5f20f0e134ec912ca1af0b66408489f1da36d95103de72c5" + } + }, + { + "case": "pair-cold-full-analysis-zellij-1-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 410.031915, + "after": 389.4158950000001 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2862.03309, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2546.3957229999996, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 76.73263999999926, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.32445000000007, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.056540000000496, + "is_error": false + } + ], + "wall_seconds": 8.692, + "cpu_seconds": 8.46357, + "kernel_family_peak_bytes_including_page_cache": 852561920, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "af07ffa500104e0405bb7438652a88816baf24d30f563707c60db392f68faa73", + "receipt_sha256": { + "report.json": "57b977958feb829d0715334164503d228159b9e0c411838dab39c3c9291f4be5", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "e9349c9a74d7379e44ec46e90a08550d5cc1a4538521e0f58c9eb58f705f9011" + } + }, + { + "case": "pair-cold-full-analysis-zellij-2-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 391.23202599999996, + "after": 389.8210560000007 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2448.2431840000004, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2244.5923359999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 10.242549000000508, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.451670000000377, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.74160000000029, + "is_error": false + } + ], + "wall_seconds": 7.776, + "cpu_seconds": 7.513651, + "kernel_family_peak_bytes_including_page_cache": 866377728, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "cd8df9aae8a1c76eaf647674cb2b66e41b364efb968b10ba54875c05413b0cc1", + "receipt_sha256": { + "report.json": "7162934761877b89b4bf91f4aa542f88b6241c3b5b60957910aa6f25f8d2f152", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "9f15d9d74c29c9fd07c6542f7f7a39f9bad7f2b9710c6135a95cc6ad1302a0d5" + } + }, + { + "case": "pair-cold-full-analysis-zellij-2-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 395.588075, + "after": 414.52098599999954 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 360.5417070000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2273.702996, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 78.97393900000043, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.887289999999666, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.09741000000031, + "is_error": false + } + ], + "wall_seconds": 8.687, + "cpu_seconds": 8.457721, + "kernel_family_peak_bytes_including_page_cache": 886362112, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "1092c9bb2a1591000a1dd8e298b3fe1a3e8cf0dbd753d8fa860e97e496754d4b", + "receipt_sha256": { + "report.json": "052cfa7f01fdff745b5dd2086017873f447c21ec8057602034274986d065628f", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "095c09a509c597afe9ff516e8dcfb67c0925ed72b80b9442f20169c4059e98f4" + } + }, + { + "case": "pair-cold-full-analysis-zellij-3-baseline", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 412.84193600000003, + "after": 385.5208850000008 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 343.735866, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2145.391627, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 77.26140899999973, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.778510000000097, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.472999999999956, + "is_error": false + } + ], + "wall_seconds": 8.484, + "cpu_seconds": 8.284427, + "kernel_family_peak_bytes_including_page_cache": 891723776, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "20aa853dfff2be0e448f14408edc263ad4dd1d911fd4c717216a49a3040319af", + "receipt_sha256": { + "report.json": "46bfacd24eb6746e4edfd0e0b4b4c6dfc19f04a06a3098f5737596691bf137d9", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "df7f3d7a57578f2aedbb46c48b491344a1c6fc23a685a763e291f7f82da41905" + } + }, + { + "case": "pair-cold-full-analysis-zellij-3-candidate", + "cache": "cold", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 391.267917, + "after": 406.9656050000003 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2357.0330240000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2176.264926, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 75.93191000000024, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.537600000000566, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.541110000000117, + "is_error": false + } + ], + "wall_seconds": 7.58, + "cpu_seconds": 7.38469, + "kernel_family_peak_bytes_including_page_cache": 842448896, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b7353bde74f8dd5175f89d8de95c094068133b0100656900d3731491b42e280d", + "receipt_sha256": { + "report.json": "76427c7fe04d1eac179c9f19dc84a14921af18b9178822bc021d7ecb5f6fcd53", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "45271691abe46caefd39dca50544361bb162f4531261c90fb3e2773bafb46d51" + } + }, + { + "case": "pair-warm-default-discovery-directory-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 403.081856, + "after": 376.17297599999984 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 4.954, + "cpu_seconds": 4.776322, + "kernel_family_peak_bytes_including_page_cache": 600211456, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "18a713e883abbf5ad48c7daf1af53559d935af4d9bb7ff8f12762e67b607f52c", + "receipt_sha256": { + "report.json": "12810d4060588f4910db4f4edb76eaaa27bf34a38fd01b8cf5bf882fee97dac7", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-discovery-directory-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 390.832706, + "after": 379.16815599999995 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.236, + "cpu_seconds": 2.025594, + "kernel_family_peak_bytes_including_page_cache": 263593984, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "745fbf3ff0d01fbf080674be5ce30b68b42c6f7da36ab221fbb8ef9af71f7429", + "receipt_sha256": { + "report.json": "687725420da9e5be55f4bf9b4aa3cfde2484504a5aeedd18dfdbc3f0866cca4f", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-discovery-directory-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 383.63070600000003, + "after": 382.6648849999999 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.135, + "cpu_seconds": 1.986985, + "kernel_family_peak_bytes_including_page_cache": 256937984, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "ade56628303a06afb1288fd5b3d57c63a08426140174057b7d2d7282c1f76aea", + "receipt_sha256": { + "report.json": "3a8e3a51638cb2dbdcb99bc64a3f7994b9ee921ef5a44920b42a7d1d19abba37", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-discovery-directory-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 393.205315, + "after": 380.4509760000001 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.056, + "cpu_seconds": 4.818559, + "kernel_family_peak_bytes_including_page_cache": 600616960, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "e607e13574c60d7b94ff483d2b715c4da702175c89d37c70faad76e389a680a4", + "receipt_sha256": { + "report.json": "b22e0c966dab798782a66effcf6dee87d67011798fb420466b86d247c735f03b", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-discovery-directory-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 406.89577499999996, + "after": 387.98794500000076 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.058, + "cpu_seconds": 4.799916, + "kernel_family_peak_bytes_including_page_cache": 600797184, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "6a52bf7eae2262d73a35bba632e015a1a8ae6ef219c0eaa853c30a849694871c", + "receipt_sha256": { + "report.json": "942ec5006d2ffe0fd00e197998f8970177f6b270d0e312b53761ee1b93a729ac", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-discovery-directory-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 398.64401499999997, + "after": 381.42463599999996 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 50840 + }, + "raw_catalog_result_tokens_all_lists": 756224, + "raw_catalog_result_bytes_all_lists": 2935459, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.236, + "cpu_seconds": 2.014862, + "kernel_family_peak_bytes_including_page_cache": 264220672, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "c38f9061296230b8d0a28437254f0005980eb3f5fa5f8d2be09acf04b05b28f6", + "receipt_sha256": { + "report.json": "7bc7748af13a49b5001b7e47b49dc88d604a36e273a415de66b0ab56c73db147", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 387.767655, + "after": 379.7490859999998 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 225.03145700000005, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2380.8913629999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.764369999999872, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.4737599999998565, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.742949999999837, + "is_error": false + } + ], + "wall_seconds": 7.976, + "cpu_seconds": 7.738718, + "kernel_family_peak_bytes_including_page_cache": 894771200, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "53421e8c3cdf43e992e2c6439d619fb4f9ccdd512c1c5fa0c41d1766b98b4b2b", + "receipt_sha256": { + "report.json": "4c3db4d6ad4d23156388fd55d4c979a280537ee3105ef232bcdb8a3982e3c4c9", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "671fc1b9944a846a36af46e3cbdfcc8868946f2b57cdb96525b3972ef7fb8001" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 408.372505, + "after": 386.729875 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2099.549725, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2104.5337849999996, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 65.43502900000021, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 17.50449099999969, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.982688999999482, + "is_error": false + } + ], + "wall_seconds": 6.868, + "cpu_seconds": 6.678343, + "kernel_family_peak_bytes_including_page_cache": 841535488, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "2088f30be4b1c78f0dfd2c8aa12aaed881c160a5c15480e5ade4f38c9fb686f7", + "receipt_sha256": { + "report.json": "826bbfc87d387b822a47e857cd88d946a8e3c1812e2e04056b9856ab20c465ac", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "386bdca7d837af5a4cce41a37fe1f5b34f0329d9be3ec27c45661ddcfe1d74f4" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 400.508395, + "after": 376.9724860000006 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2118.167875, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2106.773865, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 68.18083800000022, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 18.242310999999972, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.297319000000243, + "is_error": false + } + ], + "wall_seconds": 6.879, + "cpu_seconds": 6.666864, + "kernel_family_peak_bytes_including_page_cache": 843214848, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "8234dbac5dd23124a266df5f82c08800bfd6513abd7fe23e42e55a843944ef2f", + "receipt_sha256": { + "report.json": "3b430c7621263f2510f262a5ca0338d2685bacfd9a0eb58bfbe061c75b0ffc79", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "11ac6ea7a054df842fb88723373ba612862fcfe3eb25d8d32f56032ed491777f" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 407.702875, + "after": 383.3668749999997 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 246.735498, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2404.1977500000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.430578999999852, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.017899999999827, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 23.877619999999297, + "is_error": false + } + ], + "wall_seconds": 8.187, + "cpu_seconds": 7.936629, + "kernel_family_peak_bytes_including_page_cache": 867004416, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "bc9fc1fdf0fe3faa99afd2358b690d9113c18d62ce201cb917d72fb5f8e98681", + "receipt_sha256": { + "report.json": "2b4e6d1911f9bce9ace04c5851b7823d054c9680d33e05d20b8ca33d65f89a5e", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "34961f5ec09ab398279e0443421720fe764d6b255b54354127bb51a363046557" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 380.92482599999994, + "after": 377.0981240000001 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 227.10112800000024, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2292.3257919999996, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.259479999999712, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.59599900000012, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.949650000000474, + "is_error": false + } + ], + "wall_seconds": 7.779, + "cpu_seconds": 7.595732, + "kernel_family_peak_bytes_including_page_cache": 866107392, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "944ca93665e3ab07ea8e88620868f91dd86dcea2ca458cfe54b54cc6bd3d3054", + "receipt_sha256": { + "report.json": "7a15e38112eede65a8803d4665b5ab6a3d64ef4e28c34f3b10af6833a2a30434", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "cc7986bdb8d609528d1f2a789975e97d1cf6f99aec308fa1f2ddd5a3aba71077" + } + }, + { + "case": "pair-warm-default-analysis-ripgrep-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 404.94066499999997, + "after": 384.30944499999987 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3636, + "whole_task": 54027 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 3076.403112, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2290.932191, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.966039999999339, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.51187100000061, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 25.755310000000463, + "is_error": false + } + ], + "wall_seconds": 7.996, + "cpu_seconds": 7.809846, + "kernel_family_peak_bytes_including_page_cache": 828764160, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "88bc145b7257534a9efd03a274ce82ae7c4780e1403d60db71a3ed60299d1152", + "receipt_sha256": { + "report.json": "e682b9abf240d910ad80228175e823f440d2a1ec63b8ca7fe906d6749d353dd0", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "c62dd19a9fc8369194d2105cf489fd1f7824450297908ae6c57e1d9d12435862" + } + }, + { + "case": "pair-warm-default-analysis-zellij-1-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 386.008776, + "after": 374.2090049999997 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 374.20225500000015, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2383.8425500000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.88184899999942, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.684290000000146, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 31.454289999999673, + "is_error": false + } + ], + "wall_seconds": 8.285, + "cpu_seconds": 8.019149, + "kernel_family_peak_bytes_including_page_cache": 896237568, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b302c6e5f60fa416dfde98ed529b41059006357c7b36ca0f146893208d5c3815", + "receipt_sha256": { + "report.json": "a7de9b48a13781be5575c19e335a303f57b06e1c499e1284ae6bc2ffc0f78c2b", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "354f2c389d79de40cc498ab64e27ef344caf9c2af775b61ce6028d06a138c857" + } + }, + { + "case": "pair-warm-default-analysis-zellij-1-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 380.286046, + "after": 410.21765499999947 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2485.776508, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2453.0577980000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.046299999999974, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.8915909999996074, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.994689000000108, + "is_error": false + } + ], + "wall_seconds": 7.573, + "cpu_seconds": 7.324238, + "kernel_family_peak_bytes_including_page_cache": 839819264, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "3b7939dec7168408177bb407367ffb23069e06d5dc935ae432b8cc64d97b5ab5", + "receipt_sha256": { + "report.json": "549a5e1d1fcac4029745e46015f83af5b6710ef4e057391a34f3893e9ba2bcb4", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "b0f7872e64a2a1eff46a12c7d24a4513dead515fd67141a81fab5754367e70e7" + } + }, + { + "case": "pair-warm-default-analysis-zellij-2-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 398.92517499999997, + "after": 408.2714839999999 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2322.8452, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2445.6882590000005, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.4074099999998, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.95593000000008, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.136469000000034, + "is_error": false + } + ], + "wall_seconds": 7.472, + "cpu_seconds": 7.254133, + "kernel_family_peak_bytes_including_page_cache": 864182272, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "386035ee116c37373c21dc3045d46f12779fb95cd15d2a6d6772223a6dfd0114", + "receipt_sha256": { + "report.json": "78ea94141206e7ecf7e09b2711d2ea7800d43bc93287eed6ef95903cbe501a5c", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "14fa55e7e78c7c1f18e039efd57b36c7739e4f6ff8b5fd3f85dcd33d6bfb80ac" + } + }, + { + "case": "pair-warm-default-analysis-zellij-2-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 404.892255, + "after": 379.3739249999999 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 391.238875, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2313.1076789999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.723709999999301, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.59737000000041, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.452470000000176, + "is_error": false + } + ], + "wall_seconds": 8.081, + "cpu_seconds": 7.888906, + "kernel_family_peak_bytes_including_page_cache": 925614080, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "268c8df2d6076fc6806410d4cc181babb263022e8a04b869b93e6f9df7964b13", + "receipt_sha256": { + "report.json": "9ae3393958f8abb52ce978b3954edf8e814e4c1737c8bb3a0b754b76044b5ab0", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "287d59a709d22a2b6040fae45059974594ecb2f6e280d8e41854ea0cd1be5dea" + } + }, + { + "case": "pair-warm-default-analysis-zellij-3-baseline", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 412.723535, + "after": 377.9653050000006 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 338.0532659999999, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2807.3306520000006, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.587730000000192, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.3115600000000995, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.31768000000011, + "is_error": false + } + ], + "wall_seconds": 8.583, + "cpu_seconds": 8.393267, + "kernel_family_peak_bytes_including_page_cache": 924995584, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "ecadaff390c8d18a564ad074958a5be834c929331cfcaacf029b3df8d37b5323", + "receipt_sha256": { + "report.json": "f52ccaacd6c378ea68d92e529ababee148d688f69e17f155451facd67b7e3e99", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "d32e9ff5b7777a7375e9c479d61181cd98c8fab8dd7707ca58a47a4b31fc9c7b" + } + }, + { + "case": "pair-warm-default-analysis-zellij-3-candidate", + "cache": "warm", + "profile": "default", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 375.60634400000004, + "after": 370.27791499999967 + }, + "catalog_sha256": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 49670, + "arguments": 296, + "text_replies": 3556, + "whole_task": 53947 + }, + "raw_catalog_result_tokens_all_lists": 1512448, + "raw_catalog_result_bytes_all_lists": 5870918, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2177.872971, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2346.1349790000004, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.553550000000541, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.0773699999999735, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 27.78998000000047, + "is_error": false + } + ], + "wall_seconds": 7.071, + "cpu_seconds": 6.85764, + "kernel_family_peak_bytes_including_page_cache": 850833408, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "4187c7df93c249dfb953cf765a9d13929cbd359ed2d30f7caea27d5bda951534", + "receipt_sha256": { + "report.json": "b8a0b82267a51fa7b2f4b7a613f979e069e5a870ca6480d408ecf315615a397b", + "catalog.json": "86cb371b867b72312c8be9b4f6e8823a5e4cd7496c2490851840a715d004f8d8", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "e30cfb3f22c060b845520c3484dd6391fcb88aa54f017ad13a01492714e449b1" + } + }, + { + "case": "pair-warm-full-discovery-directory-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 389.09490500000004, + "after": 374.77101500000026 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 5.054, + "cpu_seconds": 4.806174, + "kernel_family_peak_bytes_including_page_cache": 705363968, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "9d4e6dfde9a78f1a0e04d8b23eecfb39151df90dcf4a5031011805f3a153123d", + "receipt_sha256": { + "report.json": "0103356c631db6326e553ef1a8226ad40542833e5e731a1be7042ec06c95bceb", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-discovery-directory-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 399.343645, + "after": 374.28959499999996 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.336, + "cpu_seconds": 2.138658, + "kernel_family_peak_bytes_including_page_cache": 366624768, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "c24e77daee61b7f5c4a870f77874473638b8d175bf5552fbcfd8738d0bfcdfb9", + "receipt_sha256": { + "report.json": "619303df11c8cf6a074e75d1ac179070bde898d6b5aa9547dc800f54b3b893a4", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-discovery-directory-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 412.103304, + "after": 382.0371650000002 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.338, + "cpu_seconds": 2.151271, + "kernel_family_peak_bytes_including_page_cache": 366333952, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "e6e36bfb174be0e048ed10f0a91a2f55a4a5cf749d8243c7f55ac1c130ee2e3c", + "receipt_sha256": { + "report.json": "ff83d1dd809ec1c463849314e3d1d9918b7635aafb114254f6582048f4d990af", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-discovery-directory-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 399.10454500000003, + "after": 417.8960450000004 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 4.957, + "cpu_seconds": 4.713301, + "kernel_family_peak_bytes_including_page_cache": 701562880, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "ee47f6264ef90fbc54281baece1947eb815bbd0e61b12ae30528bc83b9b075b3", + "receipt_sha256": { + "report.json": "0d42c5756a5cedeede261ed120eb10ca6731fd9953e82c6e2fe7ab46e6bd4bcd", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-discovery-directory-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 389.04046400000004, + "after": 393.44539499999973 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 4.952, + "cpu_seconds": 4.753724, + "kernel_family_peak_bytes_including_page_cache": 707497984, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "e082f5c76df3a12887824f112fe7aff802bde098b58e85adf1005f31f59629d9", + "receipt_sha256": { + "report.json": "687f2de62dda32f3089a9571c5ebd1137ce51e9fc74e948dcf17a20af0fddd99", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-discovery-directory-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "discovery", + "artifact": null, + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 388.74498400000004, + "after": 368.15286400000014 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": null, + "parent_semantic_sha256": null, + "answer_sha256": "763f5252a8277e07583989fe7fa589c93d9e13ee408dad6697c699e102178421", + "tokens_estimated": { + "question_and_recipe": 29, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 0, + "text_replies": 0, + "prompt_directory": 787, + "whole_task": 206914 + }, + "raw_catalog_result_tokens_all_lists": 912495, + "raw_catalog_result_bytes_all_lists": 3588103, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 1, + "prompts/list": 1 + }, + "calls": [], + "wall_seconds": 2.34, + "cpu_seconds": 2.106354, + "kernel_family_peak_bytes_including_page_cache": 372981760, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "bf1e4f03fb353838d88126114c858af4da4c4e9a67fe211c1c5346add8361880", + "receipt_sha256": { + "report.json": "8feb0ba4e6ad5f06d98c1e9fd067151b5dcba23622baea666511689a9e04891f", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 376.80948500000005, + "after": 380.4271150000004 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 218.14719700000023, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2273.172039, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 70.32287800000086, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.320519999999306, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 26.772320000000036, + "is_error": false + } + ], + "wall_seconds": 8.074, + "cpu_seconds": 7.889965, + "kernel_family_peak_bytes_including_page_cache": 893456384, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "5a941cd2e5f793429b357bd64edec3d9688def6f97bb1f1dc49c89f14337543a", + "receipt_sha256": { + "report.json": "1fe0fb333c9c75b674dc6adbaf6566143e6759631442d09f1e0c089c994504cb", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "caccc40d37b47dc2b14961fdabc8c47d668b0abae3d1537254764fd573e59270" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 417.998235, + "after": 367.1838950000001 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2262.5982489999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2166.510759, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 73.45156999999926, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.514960000000428, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 26.11811900000066, + "is_error": false + } + ], + "wall_seconds": 7.476, + "cpu_seconds": 7.264899, + "kernel_family_peak_bytes_including_page_cache": 847982592, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b0614ad7929ec0d2ea4addfd916126ed1d18aa71a9111b413aa616690f3454cd", + "receipt_sha256": { + "report.json": "6ea8e9f17e8716357a3e11023eeeb558cace92f544e9df4d412fab3cc4904a32", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "21c4730eb75d8fd8c02c25076c56e439049c8cb589ffce95c42a5e114ec0809d" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 390.038024, + "after": 387.91532499999994 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2091.263771, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2156.6897690000005, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 66.09377800000038, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.600711000000047, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.42678899999919, + "is_error": false + } + ], + "wall_seconds": 7.271, + "cpu_seconds": 7.011412, + "kernel_family_peak_bytes_including_page_cache": 839393280, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "2fe0fe8c6a6de88d5f102b06ce1db37bb0bde9fb3bb7aa32857aa7dd3ad1bd87", + "receipt_sha256": { + "report.json": "908788962b3a911693e608dce8221c30ff04c7823067486c00d7a61b12a8099a", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "3411b8dd4541bd06c05a44e950c3914e89768c08bdc63f15b849872d324a53b3" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 389.31388400000003, + "after": 407.5675449999999 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 222.64202699999987, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2118.45407, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 71.2047279999997, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.947869999999966, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 25.791919999999664, + "is_error": false + } + ], + "wall_seconds": 7.976, + "cpu_seconds": 7.768468, + "kernel_family_peak_bytes_including_page_cache": 892899328, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "7487a06d9efbe2612adaea8dbdad6bfd7acac1c6a9f4151b5786cb5fde3de3af", + "receipt_sha256": { + "report.json": "ccbc4961f5c054dfbbb49572d84906362135527abbf32955c72a2f666c3411bd", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "d5032cc82acad9b0dd0677519b6ca104dfb0ea9f1894a9bcb7b1a8727a54d699" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 413.92575400000004, + "after": 378.4948539999996 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 223.63232599999992, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2126.89873, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 79.2121589999997, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.129460000000108, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.48841900000025, + "is_error": false + } + ], + "wall_seconds": 8.282, + "cpu_seconds": 8.06946, + "kernel_family_peak_bytes_including_page_cache": 908144640, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "7b33308461c5451d5ab16a5ce58ae925953238071d49e563fd2f79cd88388025", + "receipt_sha256": { + "report.json": "1f24845784720aa028bfa79ff3744247da124726474ca48c72996c317b158947", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "9c0b1ec72be6e614d7fff2d21109a87197c8e8b602d7135bde6efe027a6f8b1c" + } + }, + { + "case": "pair-warm-full-analysis-ripgrep-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "ripgrep", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 409.46834399999995, + "after": 409.9364939999996 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "parent_semantic_sha256": "3561d500cece1ac5d49a02f1594148a9435d60f8f60acbadb9fae392faddcdfa", + "answer_sha256": "261c094fe6ba6c576502ecdc0c40cbc5951bccef28fe3e1189e4f9e38549232e", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3636, + "whole_task": 210101 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2095.75842, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2355.2385050000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.538099999999758, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.057960000000094, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.19480999999996, + "is_error": false + } + ], + "wall_seconds": 7.575, + "cpu_seconds": 7.372084, + "kernel_family_peak_bytes_including_page_cache": 842661888, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "59cc47c87f6ee6b72d9f1cf6c49acd6be7e6fe32b01841d66cd4337b477af80a", + "receipt_sha256": { + "report.json": "cc13f27057c49471582a551b399afda764c4715961dc384c75a3320e20fc7b4d", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "e16b2f4ddd60c4967d8f21887549a9bd770ed41d464e21e50f203dc1ebbcb427" + } + }, + { + "case": "pair-warm-full-analysis-zellij-1-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 407.323875, + "after": 381.71681399999943 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 346.2757539999993, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2258.0875880000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 76.21503800000028, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.03556000000026, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.250050000000556, + "is_error": false + } + ], + "wall_seconds": 8.582, + "cpu_seconds": 8.332648, + "kernel_family_peak_bytes_including_page_cache": 895754240, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "d26f30a93c3b43d5a41244595d64fbe41397dfa47a006a097ea9016ae21fc30a", + "receipt_sha256": { + "report.json": "a69c274b8795dd124430bc0359022d34fd4e14bea3f5d9f74079e96f42c3a695", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "02e41168c0b3f9d8cc1d64aa8ee3f4c301b769b7617907d0194a6d0c1e3619e2" + } + }, + { + "case": "pair-warm-full-analysis-zellij-1-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 1, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 390.661104, + "after": 382.28986499999974 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2268.767597, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2219.6971880000006, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.998830000000453, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.803291000000172, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.50260900000012, + "is_error": false + } + ], + "wall_seconds": 7.479, + "cpu_seconds": 7.231771, + "kernel_family_peak_bytes_including_page_cache": 867786752, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "cce885599297773a56d0d2aea7ba0e9b6a01adb336dbf3ab1d1520a99e2997e3", + "receipt_sha256": { + "report.json": "ed90d3435d1d59ed2aefd1f84d78e933e6855c7c8ee3b4fe25965f986f43b25a", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "7fbdb8dccc9fec2410fd4ad714980e9cf238d175d206accf5f283dff24e4a494" + } + }, + { + "case": "pair-warm-full-analysis-zellij-2-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 392.751284, + "after": 385.325065 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2232.147907, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2273.7027160000002, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 10.00869000000057, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.875379000000066, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.659529000000475, + "is_error": false + } + ], + "wall_seconds": 7.474, + "cpu_seconds": 7.292747, + "kernel_family_peak_bytes_including_page_cache": 870350848, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "25e38c99097c4eaa375cc00d370498ea0a1ebdbea1c2de66da957bb7843ea536", + "receipt_sha256": { + "report.json": "20e36c19a72776d87b0e06c51ccf8d14afa133dd4fd059b6eb69df32856a8ddb", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "a1562421d674d3413945f41661c5ef76db94037ab95b7755402fdeeb94094a0c" + } + }, + { + "case": "pair-warm-full-analysis-zellij-2-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 2, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 393.055544, + "after": 393.32170399999995 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 340.5052649999998, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2204.1794469999995, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 67.60613000000012, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.84136000000035, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 29.628348999999616, + "is_error": false + } + ], + "wall_seconds": 8.279, + "cpu_seconds": 8.123721, + "kernel_family_peak_bytes_including_page_cache": 896585728, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "9982d566cffd7544563c820e8268a31377630e1fc54d3c5ff558ad143150ba46", + "receipt_sha256": { + "report.json": "8fc867d36077e3739b8c7ef10214e89a345a0ce5f4839db0d6ad79ff0cc2bb6b", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "27120d5475d2a48f97227ac234a7300f5f9c0fb6b9774b5d9bb46dd73cf4e78a" + } + }, + { + "case": "pair-warm-full-analysis-zellij-3-baseline", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "baseline", + "runtime_binding_sha256": "02cdb813b3539e3e552106ec3091d9ac1d910a2d7cf22666dd1e785edfca0abd", + "runtime_audit_ms": { + "before": 396.262505, + "after": 389.0651739999994 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 335.02966600000036, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2298.8260359999995, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.15252000000055, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.975699999999961, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.19293000000016, + "is_error": false + } + ], + "wall_seconds": 8.382, + "cpu_seconds": 8.150593, + "kernel_family_peak_bytes_including_page_cache": 894513152, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "705a44390c318501894eaf791dce31f9dfed6263aa6335ea9b2680ac7469888d", + "receipt_sha256": { + "report.json": "92fe819b6854b3bbb96a8a0fa854cb66fc77eafc5ee4eb25691be0b5893ec762", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "4e7f940fc0ef5cbb3d62441318f1c008960ee9bc64a76de02c7f92646af0a75e" + } + }, + { + "case": "pair-warm-full-analysis-zellij-3-candidate", + "cache": "warm", + "profile": "full", + "phase": "analysis", + "artifact": "zellij", + "pair": 3, + "role": "candidate", + "runtime_binding_sha256": "b64bc6df16b83e7504bb624898a8c3dac6cc445d64528d80d061554fe5e98db2", + "runtime_audit_ms": { + "before": 393.959254, + "after": 394.55480500000067 + }, + "catalog_sha256": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "answer_sha256": "6facddf755f558b0c9989cf8db5fb0efd8271f954021d1d9521d686b08185eae", + "tokens_estimated": { + "question_and_recipe": 71, + "server_instructions": 354, + "input_catalog_once": 205744, + "arguments": 296, + "text_replies": 3556, + "whole_task": 210021 + }, + "raw_catalog_result_tokens_all_lists": 1824990, + "raw_catalog_result_bytes_all_lists": 7176206, + "actual_model_usage": null, + "protocol_era": "legacy", + "protocol_requests": { + "initialize": 1, + "notifications/initialized": 1, + "tools/list": 2, + "tools/call": 5 + }, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 2218.2269770000003, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2345.2631739999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.635131000000001, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.114990000000034, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.30637900000056, + "is_error": false + } + ], + "wall_seconds": 7.577, + "cpu_seconds": 7.373364, + "kernel_family_peak_bytes_including_page_cache": 882143232, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "log_sha256": "b30714f1a6d9f33caa638c612de6e097aa3d9844dec4514db9044b72ae1fecaa", + "receipt_sha256": { + "report.json": "e8c07a543a6cb7b7345498fb14d7ccb1b24068926481cc98f20fedce184f0089", + "catalog.json": "7dd864451baa2fce4abfcfcd456bcf1ec90cd29e5c4531e346968223a9c72015", + "server.json": "ef711c3e11bf620b0b0de35e6c413325dbfa3fd3c4c851cb2dd8689d6b59ff6c", + "transcript.json": "65d77de008a4bff811da32e9e26b3426f65ffb1c9581eb2ca2d3cc85c06374b7" + } + } + ], + "warmup_receipts": { + "warmup-default-baseline": "fc854779ce6265a2f5955bb4cff6fdddc410a41bf12f940658efaebf39ca129e", + "warmup-default-candidate": "84da9b7497b6cfca59dc094894f9ea525f48d50541d3c8c682ffbca9f00597c1", + "warmup-full-baseline": "a4ddd633c96cf9e582585d0851e0d4d738055e4b21ebbc3af471b8c33a65948e", + "warmup-full-candidate": "fd3048e4449975a6af000100bf58bbee16e86b2f2aef72f7b69b4564b57b25fb" + } +} diff --git a/experiments/analysis-efficiency/mcp-startup.md b/experiments/analysis-efficiency/mcp-startup.md new file mode 100644 index 0000000000..a22c79afa4 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-startup.md @@ -0,0 +1,61 @@ +# MCP discovery and complete Rust tasks + +Target-free stdio discovery previously constructed the canonical analysis runtime before the client could inspect its directory. The candidate records SDK advertisements at build time for default/explicit compact and explicit full inputs. Its first tool, prompt or completion request materializes canonical validators and handlers on the same SDK server. HTTP, output-schema-off, initial-target and unrecorded source runs retain eager composition. + +Pinned baseline: [`main@c465ff5a8`](https://github.com/morluto/rea/commit/c465ff5a814b8e1a386fc547cbcd94df5c35f086). Measured candidate: [`04900abbc`](https://github.com/leeguooooo/rea/commit/04900abbcd3c3438fce4af932ff28400fe744a7d). Both use the same main base and locked dependencies. Before/after source and all compiled JS/JSON bindings identify the exact runtime used in every trial. The input manifest and producer scripts are also hashed and checked. + +## Real tasks and independent facts + +Official Alpine v3.21 packages supply ripgrep 14.1.1-r0 (3,416,168-byte executable) and Zellij 0.40.0-r0 (17,151,864 bytes). Their download URLs, archive/executable digests and aports revisions are retained in [the measurement data](mcp-startup.json). The independent Python ELF/section/zlib/JSON oracle checks 42 packages and 88 dependency edges for ripgrep, and 422 packages and 1,063 edges for Zellij. Serde versions are 1.0.210 and 1.0.137 respectively; the selected package kind remains explicit unknown (`null`). Verification covers embedded static build metadata; execution and runtime reachability remain unverified. + +Each analysis task initializes the actual SDK client, lists and validates the input directory, selects serde name/version/kind, reads the exact retained parent twice, verifies canonical rejection of a negative offset, exports and independently checks the complete package/edge inventory, confirms unchanged advertisements after materialization, then closes. All five tool calls and both directory reads are included in the measured task. Discovery tasks initialize, list tools/prompts, validate the input directory and close without an analysis call. + +All 72 paired trials and four priming runs passed. Each artifact has identical answers, parent Evidence IDs and complete semantic hashes across roles, profiles and cache modes. Both profiles retain byte-identical directories across baseline/candidate and before/after canonical materialization. The directory contains 213 tools. Complete Evidence is retained and exported through the existing shared workflow. + +## Measured cost + +Three alternating pairs per condition; table values are medians. Min/max ranges and individual receipts are in [mcp-startup.json](mcp-startup.json). Each cell is baseline → candidate. Kernel process-family peak includes file cache, SDK driver, observer and REA children. + +| Task | Cache | Inputs | Wall seconds | Family CPU seconds | Family peak MiB | +| --------- | ----- | ----------------- | ------------: | -----------------: | --------------: | +| Directory | cold | compact (default) | 5.661 → 2.236 | 5.403 → 2.050 | 559.3 → 248.6 | +| Directory | cold | full | 5.256 → 2.337 | 5.040 → 2.175 | 650.9 → 349.1 | +| Directory | warm | compact (default) | 5.056 → 2.236 | 4.800 → 2.015 | 572.8 → 251.4 | +| Directory | warm | full | 4.957 → 2.338 | 4.754 → 2.139 | 672.7 → 349.6 | +| ripgrep | cold | compact (default) | 8.177 → 8.078 | 7.945 → 7.899 | 849.7 → 792.5 | +| zellij | cold | compact (default) | 8.276 → 7.276 | 8.041 → 7.105 | 846.2 → 819.0 | +| ripgrep | cold | full | 8.180 → 7.483 | 8.000 → 7.312 | 846.5 → 794.4 | +| zellij | cold | full | 8.484 → 7.776 | 8.284 → 7.514 | 850.4 → 813.1 | +| ripgrep | warm | compact (default) | 7.976 → 6.879 | 7.739 → 6.678 | 826.8 → 802.6 | +| zellij | warm | compact (default) | 8.285 → 7.472 | 8.019 → 7.254 | 882.1 → 811.4 | +| ripgrep | warm | full | 8.074 → 7.476 | 7.890 → 7.265 | 852.1 → 803.6 | +| zellij | warm | full | 8.382 → 7.479 | 8.151 → 7.293 | 854.3 → 830.0 | + +Discovery improves clearly in this cohort. Completed-task changes depend on the workload: default cold ripgrep is approximately unchanged. Initialization moves into the first actual analysis request; its latency must be assessed with the full workflow. + +| Default ripgrep first RPC | Baseline ms | Candidate ms | +| ------------------------- | ----------: | -----------: | +| cold | 231.5 | 2532.1 | +| warm | 227.1 | 2118.2 | + +Within each profile, advertised context remains identical. Input-directory estimates are 49,670 tokens for default compact and 205,744 for full. Whole-task estimates include the supplied question/recipe, server instructions, directory once, every tool argument and text reply, and the prompt directory in discovery tasks. This projection assumes the client retains one unchanged directory; raw catalog result costs count every observed list result separately, including output schemas. These use tiktoken 0.14.0/cl100k_base; actual model usage, reasoning and SDK initialization/framing are unmeasured. + +Linux x64 / AMD EPYC 7B12 / kernel 6.1.0-54-cloud-amd64 / Node 24.18.0, npm 11.16.0. Timed SDK tasks negotiated the legacy protocol era; auto/modern correctness is covered separately by compiled boundary tests. Trials run serially under a 1-CPU quota pinned to CPU 15, 4 GiB hard memory / 3 GiB early stop, no swap and 192 tasks. No resource stop, OOM or remaining payload process was observed. Cold disables server Node compile cache; warm uses a role/profile-specific cache populated by one complete task. Fresh processes and HOME are used each trial. Cache hit rate and OS page cache are unmeasured. Runtime audits warm compiled-file pages and are included in whole-task costs. Builds, downloads, oracle generation and cache priming are preparation outside those costs. Retained V8 heap and forced-GC measurements were not separately collected. + +## Ownership and validation + +`OwnedMcpServers` seals publication before taking its close snapshot, keeps failed exact close capabilities for retry and serves both eager and deferred paths. Deferred startup still joins servers published before an asynchronous serving handle settles. Post-snapshot and post-close factory calls are refused before allocation. Both production-boundary regressions fail with only the shared guard removed and pass restored. + +The moved transport/delivery implementation preserves current complete-error retention, escaped-preview fitting, response/string-limit recovery, target-content provenance and native Evidence continuations. Canonical parsers remain authoritative after lazy binding. Both protocol eras, input/output profile choices, invalid input/output, completion cancellation, partial startup and cleanup retry are exercised. + +Aggregation keeps catalog, producer-script and guard-script digests in distinct variables. Before publishing a trial it verifies that the catalog digest matches both its catalog-file receipt and the shared profile identity. Deliberately selecting the guard digest for that field is rejected. Regeneration preserves all measured timing, CPU, memory, context and parent-Evidence values. + +Executed local gates: 125 focused tests across 14 files; a repeated 22-case schema/profile subset after the pinned main merge; all six static-check tasks plus architecture guards; builds for both pinned runtimes; six documentation checks and five README consistency tests. HTTP coverage includes the real compiled CLI session/replacement/Host tests. Runtime-binding admission rejects a mismatched compiled graph before MCP launch; aggregation rejects a tampered trial binding. The raw receipts remain in `/tmp/rea-perf-mcp-startup-20261022`, identified by hashes in the data. Real initial-target providers, foreign OS runtime behavior and other providers remain outside this measurement. + +## Reproduction + +Use the pinned Node/npm toolchain and separate clean baseline/candidate checkouts. Build each with `npm ci` and `npm run build:cached` (the measured workspace shared one locked dependency installation). Acquire and verify the official packages listed in the data, extract the named executables and generate independent oracles with `mcp-input-catalog-oracle.py`. + +Capture each checkout with `node experiments/analysis-efficiency/managed-cli-runtime.mjs CHECKOUT mcp`. Construct `measurement-inputs.json` using the recorded schema, updating absolute checkout/artifact/oracle paths and their independently verified bindings. Keep it immutable through a cohort; producer source digests cover `mcp-input-catalog.mjs`, `managed-cli-runtime.mjs`, `resource-run.py` and `resource_payload.py`. + +Run `mcp-input-catalog.mjs` through `resource-run.py --observe-payload-exit` with `--runtime`, `--output`, `--bindings`, `--role baseline|candidate`, `--profile default|full`, `--phase discovery|analysis` and `--cache cold|warm`. Analysis also takes `--artifact` and `--oracle`; warm requires a role/profile-specific `--cache-root`. Disable the driver compile cache for both cohorts. Prime each warm role/profile once with the complete ripgrep task. Follow the 72-case order in the data, reversing role order in every second pair. Aggregate with `python mcp-startup-costs.py LAB --output mcp-startup.json` using tiktoken 0.14.0. The aggregator verifies every input/runtime/guard binding and completed receipt before producing summaries. diff --git a/package.json b/package.json index 4b34fdb6e8..9b57871e91 100644 --- a/package.json +++ b/package.json @@ -305,8 +305,8 @@ "verify:solidity:metadata": "npm run build:cached && node scripts/verify-solidity-metadata.mjs", "verify:recorded:crash": "npm run build:cached && node scripts/verify-recorded-crash.mjs", "mcp-catalog:check:built": "node scripts/generate-mcp-tool-catalog.mjs --check", - "compile": "node scripts/clean-build-output.mjs && node scripts/run-typescript.mjs -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/bundle-runtime.mjs", - "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/bundle-runtime.mjs", + "compile": "node scripts/clean-build-output.mjs && node scripts/run-typescript.mjs -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/bundle-runtime.mjs", + "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/bundle-runtime.mjs", "artifacts:mcp-catalog": "node scripts/generate-mcp-tool-catalog.mjs", "artifacts:skills": "node scripts/generate-skill-metadata.mjs", "artifacts:product-catalog": "node scripts/generate-product-catalog.mjs", diff --git a/scripts/lib/development-tests.mjs b/scripts/lib/development-tests.mjs index ee256414c5..dacd818201 100644 --- a/scripts/lib/development-tests.mjs +++ b/scripts/lib/development-tests.mjs @@ -39,6 +39,7 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => { // runtime or typecheck prerequisites. Add a consumer here when it reads one. const artifactConsumers = { "artifacts:mcp-catalog": [ + "tests/boundary/mcp/deferredRuntime.test.ts", "tests/boundary/mcp/contractPresentation.test.ts", "tests/boundary/mcp/toolSchemaValidity.test.ts", ], diff --git a/scripts/record-mcp-catalog.mjs b/scripts/record-mcp-catalog.mjs new file mode 100644 index 0000000000..9012b50633 --- /dev/null +++ b/scripts/record-mcp-catalog.mjs @@ -0,0 +1,119 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; + +// Runtime discovery is generated from the same canonical registrations, after +// default identity has been recorded and before bundling. Test catalogs stay separate. +const compiled = resolve(dirname(fileURLToPath(import.meta.url)), "../dist"); +const read = (name) => import(pathToFileURL(join(compiled, name)).href); +const [ + { createServer }, + { createBinarySession }, + { parseConfig }, + { silentLogger }, + { PROMPT_CONTRACTS }, +] = await Promise.all([ + read("server/createServer.js"), + read("composition/binary.js"), + read("config/parseConfig.js"), + read("logger.js"), + read("contracts/promptContracts.js"), +]); +const recordCatalog = async (profile) => { + // Catalog generation is independent of the host account and user configuration. + const home = await mkdtemp(join(tmpdir(), "rea-mcp-catalog-")); + const environment = { + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, "config"), + XDG_CACHE_HOME: join(home, "cache"), + XDG_STATE_HOME: join(home, "state"), + }; + const cleanup = []; + let producerFailure; + try { + const config = parseConfig(environment); + if (!config.ok) throw config.error; + const session = createBinarySession( + config.value, + silentLogger, + environment, + ); + cleanup.push(async () => { + const closed = await session.close({ retainProviderDocuments: true }); + if (!closed.ok) throw closed.error; + }); + const server = createServer({ kind: "session", session }, { environment }); + cleanup.push(() => server.close()); + const client = new Client({ name: "rea-catalog-recorder", version: "1" }); + cleanup.push(() => client.close()); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const tools = (await client.listTools()).tools; + const prompts = (await client.listPrompts()).prompts.map((prompt) => { + const contract = PROMPT_CONTRACTS.find( + (candidate) => candidate.name === prompt.name, + ); + if (contract === undefined) + throw new Error(`Missing prompt ${prompt.name}`); + return { ...prompt, parameters: contract.arguments }; + }); + const catalog = { + version: 1, + info: client.getServerVersion(), + capabilities: client.getServerCapabilities(), + instructions: client.getInstructions(), + identity: server.catalogIdentity, + tools, + prompts, + }; + const encoded = JSON.stringify(catalog); + const digest = createHash("sha256").update(encoded).digest("hex"); + await writeFile( + join( + compiled, + profile === "full" ? "mcp-catalog-full.json" : "mcp-catalog.json", + ), + encoded, + ); + return digest; + } catch (cause) { + producerFailure = { cause }; + throw cause; + } finally { + const results = await Promise.allSettled( + cleanup.reverse().map((close) => Promise.resolve().then(close)), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length === 0) { + try { + await rm(home, { recursive: true, force: true }); + } catch (cause) { + failures.push(cause); + } + } + if (failures.length > 0) + throw new AggregateError( + producerFailure === undefined + ? failures + : [producerFailure.cause, ...failures], + `Catalog cleanup failed; temporary home retained at ${home}`, + ); + } +}; + +const digests = {}; +for (const profile of ["compact", "full"]) + digests[profile] = await recordCatalog(profile); +await writeFile( + join(compiled, "recordedMcpCatalog.js"), + `/** Recorded from the compiled runtime; do not edit. */\nexport const RECORDED_MCP_CATALOG_SHA256 = ${JSON.stringify(digests)};\n`, +); diff --git a/src/config/mcpInputSchemaProfile.ts b/src/config/mcpInputSchemaProfile.ts index 1acb5b2a1b..110e53a13f 100644 --- a/src/config/mcpInputSchemaProfile.ts +++ b/src/config/mcpInputSchemaProfile.ts @@ -12,11 +12,16 @@ export const COMPACT_INPUT_SCHEMA_BUDGET_BYTES = 13 * 1024; export const mcpInputSchemaProfileSchema = z.enum(["full", "compact"]); +/** Resolve the selected advertisement at the configuration boundary. */ +export const resolveMcpInputSchemaProfile = ( + profile: McpInputSchemaProfile | undefined, +): McpInputSchemaProfile => profile ?? "compact"; + /** Select the default compact advertisement, preserving an explicit full choice. */ export const compactPresentationForMcpInputSchemaProfile = ( profile: "full" | "compact" | undefined, ): { readonly budgetBytes: number } | undefined => - profile === "full" + resolveMcpInputSchemaProfile(profile) === "full" ? undefined : { budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES }; diff --git a/src/main.ts b/src/main.ts index 4a10d8a1dd..1efe4f0788 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,19 +8,12 @@ import { realpathSync } from "node:fs"; import { serveStdio } from "@modelcontextprotocol/server/stdio"; import { parseConfig } from "./config/parseConfig.js"; -import { createBinarySession } from "./composition/binary.js"; -import { createLogger } from "./logger.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; import type { RuntimeDependencies } from "./main/types.js"; +import { runDeferred } from "./main/deferredRuntime.js"; +import { RECORDED_MCP_CATALOG_SHA256 } from "./recordedMcpCatalog.js"; import { SERVER_START_FAILED } from "./main/messages.js"; -import { openInitialTarget } from "./main/startup.js"; -import { startMcpTransport } from "./main/transport.js"; -import { - createOptionalObservationFactories, - loadOptionalObservationProviders, -} from "./composition/optionalObservationProviders.js"; -import { createToolResultDelivery } from "./server/toolResult.js"; -import { createShutdown } from "./main/shutdown.js"; + import { createExitStatus, latchSignalShutdown, @@ -53,6 +46,7 @@ const httpRuntimeDependencies = ( const base = runtimeDependencies(); return { ...base, + transport: "http", setExitCode: exit.set, serve: async (factory, options) => { // Loaded here so the default stdio entry never loads the HTTP listener. @@ -90,12 +84,7 @@ const httpRuntimeDependencies = ( }; }; -/** - * Start the long-lived MCP adapter and install idempotent shutdown handlers. - * The adapter owns its BinarySession for the process lifetime; stdin EOF on - * the stdio transport and process signals on either transport close bridge - * resources without assuming ownership of the Hopper app. - */ +/** Start discovery without constructing unused analysis validators. */ export const run = async ( dependencies: RuntimeDependencies = runtimeDependencies(), ): Promise => { @@ -105,38 +94,23 @@ export const run = async ( dependencies.writeStderr(`${projectAnalysisError(config.error).message}\n`); return 1; } - const logger = createLogger("mcp", config.value.logLevel); - const serverLogger = logger.child({ layer: "server" }); - const session = createBinarySession(config.value, logger, environment); - const opened = await openInitialTarget( - session, + const fullOutputCatalog = + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE === undefined || + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE === "full"; + if ( + RECORDED_MCP_CATALOG_SHA256 !== undefined && + dependencies.transport !== "http" && + config.value.hopperTargetPath === undefined && + fullOutputCatalog && + dependencies.createServer === undefined && + dependencies.loadOptionalProviders === undefined + ) { + return runDeferred({ ...dependencies, env: environment }, config.value); + } + return (await import("./main/eagerRuntime.js")).runEager( + { ...dependencies, env: environment }, config.value, - serverLogger, - dependencies.writeStderr, ); - if (!opened.ok) return opened.exitCode; - const transport = await startMcpTransport(dependencies, session, { - logger, - serverLogger, - environment, - delivery: createToolResultDelivery(config.value.mcpMaxResponseBytes), - loadOptionalProviders: - dependencies.loadOptionalProviders ?? - (() => - loadOptionalObservationProviders( - createOptionalObservationFactories(environment), - )), - }); - if (!transport.ok) return 1; - createShutdown({ - handle: transport.handle, - closeAndroid: transport.closeAndroid, - closeOptionalProviders: transport.closeOptionalProviders, - session, - dependencies, - serverLogger, - }); - return 0; }; /** Run the executable boundary without exposing an unexpected startup cause. */ diff --git a/src/main/OwnedMcpServers.ts b/src/main/OwnedMcpServers.ts new file mode 100644 index 0000000000..f5118f913e --- /dev/null +++ b/src/main/OwnedMcpServers.ts @@ -0,0 +1,26 @@ +/** Own the exact close capabilities of every published SDK instance. */ +export class OwnedMcpServers }> { + private readonly servers = new Map Promise>(); + private accepting = true; + + create(factory: () => Server): Server { + if (!this.accepting) throw new Error("MCP connection is closed"); + const server = factory(); + const close = server.close.bind(server); + const ownedClose = async (): Promise => { + await close(); + this.servers.delete(server); + }; + this.servers.set(server, ownedClose); + server.close = ownedClose; + return server; + } + + /** Seal publication before snapshotting; failed close capabilities stay owned. */ + closeAll(): Promise[]> { + this.accepting = false; + return Promise.allSettled( + [...this.servers.values()].map((close) => Promise.resolve().then(close)), + ); + } +} diff --git a/src/main/deferredRuntime.ts b/src/main/deferredRuntime.ts new file mode 100644 index 0000000000..e359ab69a0 --- /dev/null +++ b/src/main/deferredRuntime.ts @@ -0,0 +1,243 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { + STDIO_DEFAULT_MAX_BUFFER_SIZE, + type McpRequestContext, +} from "@modelcontextprotocol/server"; +import { + RECORDED_MCP_CATALOG_SHA256, + type RecordedMcpCatalog, +} from "../recordedMcpCatalog.js"; +import type { AppConfig } from "../config/types.js"; +import { + resolveMcpInputSchemaProfile, + type McpInputSchemaProfile, +} from "../config/mcpInputSchemaProfile.js"; +import { DeferredEvidenceMcpServer } from "../server/DeferredEvidenceMcpServer.js"; +import { ToolResultDelivery } from "../server/ToolResultDelivery.js"; +import { createLogger } from "../logger.js"; +import { reportMcpShutdownFailure } from "./shutdown.js"; +import type { McpServerHandle, RuntimeDependencies } from "./types.js"; +import { MCP_CONNECTION_LOST, SERVER_START_FAILED } from "./messages.js"; +import { OwnedMcpServers } from "./OwnedMcpServers.js"; + +/** Verify the build-owned artifact before trusting its exact typed projection. */ +const readCatalog = (profile: McpInputSchemaProfile): RecordedMcpCatalog => { + const bytes = readFileSync( + new URL( + profile === "full" ? "../mcp-catalog-full.json" : "../mcp-catalog.json", + import.meta.url, + ), + ); + if ( + createHash("sha256").update(bytes).digest("hex") !== + RECORDED_MCP_CATALOG_SHA256?.[profile] + ) + throw new Error(SERVER_START_FAILED); + // The digest binds these bytes to the same compiled producer and SDK converter. + const catalog: unknown = JSON.parse(bytes.toString("utf8")); + if ( + typeof catalog !== "object" || + catalog === null || + Reflect.get(catalog, "version") !== 1 + ) + throw new Error(SERVER_START_FAILED); + return catalog as RecordedMcpCatalog; +}; + +const initializeRuntime = async ( + server: DeferredEvidenceMcpServer, + dependencies: RuntimeDependencies, + context: McpRequestContext, + config: AppConfig, + startedAt: string, +): Promise => { + let shutdown: (() => Promise) | undefined; + let closeSession: (() => Promise) | undefined; + let sessionClosed = false; + server.ownRuntimeCleanup(async () => { + if (shutdown !== undefined) return shutdown(); + const results = await Promise.allSettled([ + server.closePartialRuntime(), + (async () => { + if (!sessionClosed && closeSession !== undefined) { + await closeSession(); + sessionClosed = true; + } + })(), + ]); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "MCP startup cleanup failed"); + }); + const [{ runEager }, { createServer }] = await Promise.all([ + import("./eagerRuntime.js"), + import("../server/createServer.js"), + ]); + const result = await runEager( + { + ...dependencies, + registerShutdown: () => () => undefined, + captureShutdown: (close) => { + shutdown = close; + }, + createServer: (source, options) => { + if (source.kind === "session") + closeSession = async () => { + const result = await source.session.close({ + retainProviderDocuments: true, + }); + if (!result.ok) throw result.error; + }; + return createServer(source, { + ...options, + mcpServer: server, + startedAt, + }); + }, + serve: (factory) => { + let close: (() => Promise) | undefined; + server.materialize(() => { + const configured = factory(context); + if (configured !== server) throw new Error(SERVER_START_FAILED); + close = configured.close.bind(configured); + }); + return { close: () => close?.() ?? Promise.resolve() }; + }, + }, + config, + ); + if (result !== 0 || shutdown === undefined) + throw new Error(SERVER_START_FAILED); +}; + +/** Own every SDK-created server, including discarded discovery instances. */ +export const runDeferred = async ( + dependencies: RuntimeDependencies, + config: AppConfig, +): Promise => { + const serverLogger = createLogger("mcp", config.logLevel).child({ + layer: "server", + }); + const profile = resolveMcpInputSchemaProfile(config.mcpInputSchemaProfile); + let firstCatalog: RecordedMcpCatalog | undefined = readCatalog(profile); + const servers = new OwnedMcpServers(); + let handle: McpServerHandle | undefined; + let attempt: Promise | undefined; + let unregister: () => void = () => undefined; + let closing = false; + let finishServing!: () => void; + const serving = new Promise((resolve) => { + finishServing = resolve; + }); + const close = (): Promise => { + closing = true; + attempt ??= Promise.resolve() + .then(async () => { + // A transport published after shutdown begins is still ours to close. + await serving; + const ownedHandle = handle; + const handleResults = Promise.allSettled([ + ...(ownedHandle === undefined + ? [] + : [ + Promise.resolve() + .then(() => ownedHandle.close()) + .then(() => { + handle = undefined; + }), + ]), + ]); + // Servers published before the serving handle settles join this owner; + // closeAll seals further publication before taking its snapshot. + const [handles, instances] = await Promise.all([ + handleResults, + servers.closeAll(), + ]); + const failures = [...handles, ...instances].flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "MCP cleanup failed"); + unregister(); + }) + .catch((cause) => { + attempt = undefined; + throw cause; + }); + return attempt; + }; + const requestClose = () => { + void close().catch((cause) => + reportMcpShutdownFailure(cause, dependencies, serverLogger), + ); + }; + try { + // Keep cleanup reachable even if serve creates a server and then throws. + unregister = dependencies.registerShutdown(requestClose); + dependencies.captureShutdown?.(close); + try { + handle = await dependencies.serve( + (context) => { + const server = servers.create(() => { + const catalog = firstCatalog ?? readCatalog(profile); + firstCatalog = undefined; + const startedAt = new Date().toISOString(); + const server = new DeferredEvidenceMcpServer( + catalog, + new ToolResultDelivery( + config.mcpMaxResponseBytes ?? STDIO_DEFAULT_MAX_BUFFER_SIZE, + ), + (target) => + initializeRuntime( + target, + dependencies, + context, + config, + startedAt, + ).catch((cause) => { + dependencies.setExitCode(1); + throw cause; + }), + ); + return server; + }); + const previous = server.server.onclose; + server.server.onclose = () => { + previous?.(); + requestClose(); + }; + return server; + }, + { + onerror: () => { + serverLogger.error(MCP_CONNECTION_LOST); + dependencies.writeStderr(`${MCP_CONNECTION_LOST}\n`); + }, + }, + ); + } finally { + finishServing(); + } + } catch (cause) { + finishServing(); + serverLogger.error( + { error: cause instanceof Error ? cause.message : String(cause) }, + SERVER_START_FAILED, + ); + try { + await close(); + } catch (cleanupCause) { + throw new AggregateError( + [cause, cleanupCause], + "MCP startup and cleanup failed", + ); + } + dependencies.writeStderr(`${SERVER_START_FAILED}\n`); + return 1; + } + if (closing) await close(); + return 0; +}; diff --git a/src/main/eagerRuntime.ts b/src/main/eagerRuntime.ts new file mode 100644 index 0000000000..874d8007d3 --- /dev/null +++ b/src/main/eagerRuntime.ts @@ -0,0 +1,70 @@ +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; + +import type { AppConfig } from "../config/types.js"; +import { ok } from "../domain/result.js"; +import { parseConfig } from "../config/parseConfig.js"; +import { createBinarySession } from "../composition/binary.js"; +import { createLogger } from "../logger.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { RuntimeDependencies } from "./types.js"; +import { openInitialTarget } from "./startup.js"; +import { startMcpTransport } from "./transport.js"; +import { + createOptionalObservationFactories, + loadOptionalObservationProviders, +} from "../composition/optionalObservationProviders.js"; +import { createToolResultDelivery } from "../server/toolResult.js"; +import { createShutdown } from "./shutdown.js"; + +/** + * Start the long-lived MCP adapter and install idempotent shutdown handlers. + * The adapter owns its BinarySession for the process lifetime; EOF and process + * signals close bridge resources without assuming ownership of the Hopper app. + */ +export const runEager = async ( + dependencies: RuntimeDependencies, + selectedConfig?: AppConfig, +): Promise => { + const environment = snapshotEnvironment(dependencies.env); + const config = + selectedConfig === undefined + ? parseConfig(environment) + : ok(selectedConfig); + if (!config.ok) { + dependencies.writeStderr(`${projectAnalysisError(config.error).message}\n`); + return 1; + } + const logger = createLogger("mcp", config.value.logLevel); + const serverLogger = logger.child({ layer: "server" }); + const session = createBinarySession(config.value, logger, environment); + const opened = await openInitialTarget( + session, + config.value, + serverLogger, + dependencies.writeStderr, + ); + if (!opened.ok) return opened.exitCode; + const transport = await startMcpTransport(dependencies, session, { + logger, + serverLogger, + environment, + delivery: createToolResultDelivery(config.value.mcpMaxResponseBytes), + loadOptionalProviders: + dependencies.loadOptionalProviders ?? + (() => + loadOptionalObservationProviders( + createOptionalObservationFactories(environment), + )), + }); + if (!transport.ok) return 1; + const shutdown = createShutdown({ + handle: transport.handle, + closeAndroid: transport.closeAndroid, + closeOptionalProviders: transport.closeOptionalProviders, + session, + dependencies, + serverLogger, + }); + dependencies.captureShutdown?.(shutdown.shutdown); + return 0; +}; diff --git a/src/main/shutdown.ts b/src/main/shutdown.ts index e74ce0c894..075faacde1 100644 --- a/src/main/shutdown.ts +++ b/src/main/shutdown.ts @@ -78,21 +78,30 @@ export const createShutdown = (input: { }; const requestShutdown = (): void => { shutdown().catch((cause: unknown) => { - serverLogger.debug( - { failure_cause: describeShutdownFailure(cause) }, - "MCP shutdown rejected", - ); - dependencies.setExitCode(1); - serverLogger.error(MCP_SHUTDOWN_FAILED); - dependencies.writeStderr(`${MCP_SHUTDOWN_FAILED}\n`); - if (cause instanceof AnalysisError) - dependencies.writeStderr(`${projectAnalysisError(cause).message}\n`); + reportMcpShutdownFailure(cause, dependencies, serverLogger); }); }; unregisterShutdown = dependencies.registerShutdown(requestShutdown); return { shutdown, request: requestShutdown }; }; +/** Preserve the same diagnostics for eager and deferred runtime owners. */ +export const reportMcpShutdownFailure = ( + cause: unknown, + dependencies: Pick, + serverLogger: Logger, +): void => { + serverLogger.debug( + { failure_cause: describeShutdownFailure(cause) }, + "MCP shutdown rejected", + ); + dependencies.setExitCode(1); + serverLogger.error(MCP_SHUTDOWN_FAILED); + dependencies.writeStderr(`${MCP_SHUTDOWN_FAILED}\n`); + if (cause instanceof AnalysisError) + dependencies.writeStderr(`${projectAnalysisError(cause).message}\n`); +}; + const describeShutdownFailure = ( cause: unknown, ): Readonly> => diff --git a/src/main/transport.ts b/src/main/transport.ts index 3a1e7e807f..871880f224 100644 --- a/src/main/transport.ts +++ b/src/main/transport.ts @@ -8,6 +8,7 @@ import { StreamableHttpListenError } from "../server/http/streamableHttpListenEr import type { ToolResultDelivery } from "../server/toolResult.js"; import type { McpServerHandle, RuntimeDependencies } from "./types.js"; import type { OptionalProviderLoadResult } from "../application/OptionalObservationProviders.js"; +import { OwnedMcpServers } from "./OwnedMcpServers.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { MCP_CONNECTION_LOST, @@ -62,11 +63,9 @@ export const startMcpTransport = async ( ); } const androidProviders: AndroidAnalysisPort[] = []; - const servers = new Set>(); + const servers = new OwnedMcpServers>(); const closeServers = async (): Promise => { - const results = await Promise.allSettled( - [...servers].map((server) => server.close()), - ); + const results = await servers.closeAll(); const failures = results.flatMap((result) => result.status === "rejected" ? [result.reason] : [], ); @@ -110,32 +109,26 @@ export const startMcpTransport = async ( let handle: McpServerHandle; try { handle = await dependencies.serve( - () => { - // The SDK can discard a discovery probe and construct a replacement server. - const android = createAndroidAnalysisProvider( - serverContext.environment, - ); - androidProviders.push(android); - const server = (dependencies.createServer ?? createServer)( - { kind: "session", session }, - { - logger: serverContext.logger, - environment: serverContext.environment, - delivery: serverContext.delivery, - ...optionalProviders, - optionalObservationProviderOwner: "caller", - androidAnalysis: android, - }, - ); - // The SDK can discard a server even when its provider cleanup failed. - servers.add(server); - const closeServer = server.close.bind(server); - server.close = async () => { - await closeServer(); - servers.delete(server); - }; - return server; - }, + () => + servers.create(() => { + // The SDK can discard a discovery probe and construct a replacement server. + const android = createAndroidAnalysisProvider( + serverContext.environment, + ); + androidProviders.push(android); + const server = (dependencies.createServer ?? createServer)( + { kind: "session", session }, + { + logger: serverContext.logger, + environment: serverContext.environment, + delivery: serverContext.delivery, + ...optionalProviders, + optionalObservationProviderOwner: "caller", + androidAnalysis: android, + }, + ); + return server; + }), { onerror: () => { serverLogger.error(MCP_CONNECTION_LOST); diff --git a/src/main/types.ts b/src/main/types.ts index 72430027a6..4cde88d1a8 100644 --- a/src/main/types.ts +++ b/src/main/types.ts @@ -23,6 +23,9 @@ export type McpServe = ( ) => McpServerHandle | Promise; export interface RuntimeDependencies { + /** HTTP keeps the eager runtime; deferred materialization is stdio-only. */ + readonly transport?: "stdio" | "http"; + readonly captureShutdown?: (shutdown: () => Promise) => void; readonly env: NodeJS.ProcessEnv; readonly serve: McpServe; readonly writeStderr: (text: string) => void; diff --git a/src/recordedMcpCatalog.ts b/src/recordedMcpCatalog.ts new file mode 100644 index 0000000000..23595540fe --- /dev/null +++ b/src/recordedMcpCatalog.ts @@ -0,0 +1,27 @@ +import type { + Implementation, + Prompt, + ServerCapabilities, + Tool, +} from "@modelcontextprotocol/server"; +import type { createCatalogIdentity } from "./catalogIdentity.js"; +import type { PromptArgumentContract } from "./contracts/promptContracts.js"; +import type { McpInputSchemaProfile } from "./config/mcpInputSchemaProfile.js"; + +/** Build-owned discovery data; canonical validators remain the runtime authority. */ +export interface RecordedMcpCatalog { + readonly version: 1; + readonly info: Implementation; + readonly capabilities: ServerCapabilities; + readonly instructions: string; + readonly identity: ReturnType; + readonly tools: readonly Tool[]; + readonly prompts: readonly (Prompt & { + readonly parameters: Readonly>; + })[]; +} + +/** Replaced after compilation by the recorder; source runs use eager composition. */ +export const RECORDED_MCP_CATALOG_SHA256: + | Readonly> + | undefined = undefined; diff --git a/src/server/DeferredEvidenceMcpServer.test.ts b/src/server/DeferredEvidenceMcpServer.test.ts new file mode 100644 index 0000000000..c7c75f23b1 --- /dev/null +++ b/src/server/DeferredEvidenceMcpServer.test.ts @@ -0,0 +1,343 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Tool, ServerContext } from "@modelcontextprotocol/server"; +import { afterEach, expect, it, vi } from "vitest"; +import { z } from "zod"; +import type { RecordedMcpCatalog } from "../recordedMcpCatalog.js"; +import { DeferredEvidenceMcpServer } from "./DeferredEvidenceMcpServer.js"; +import { ToolResultDelivery } from "./ToolResultDelivery.js"; +import { promptArgumentsSchema } from "./promptArguments.js"; + +const barrier = () => { + let release!: () => void; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { promise, release }; +}; +const input = z.strictObject({ n: z.number().int().nonnegative().default(3) }); +const output = z.strictObject({ n: z.number().int() }); +const parameters = { + document: { + description: "Document name", + required: true, + completion: "document" as const, + }, +}; +const catalog: RecordedMcpCatalog = { + version: 1, + info: { name: "deferred-fixture", version: "1" }, + instructions: "Fixture", + capabilities: { + tools: { listChanged: true }, + prompts: { listChanged: true }, + completions: {}, + }, + identity: { + counts: { cli_commands: 0, mcp_tools: 1, mcp_prompts: 1 }, + digests: { + tools_sha256: "fixture", + prompts_sha256: "fixture", + combined_sha256: "fixture", + }, + }, + tools: [ + { + name: "fixture", + inputSchema: z.toJSONSchema(input, { + io: "input", + }) as Tool["inputSchema"], + outputSchema: z.toJSONSchema(output), + }, + ], + prompts: [ + { + name: "explain", + title: "Explain", + description: "Explain a document", + arguments: [ + { name: "document", description: "Document name", required: true }, + ], + parameters, + }, + ], +}; +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + while (cleanups.length > 0) await cleanups.pop()?.(); +}); + +const fixture = async ( + options: { + readonly hold?: ReturnType; + readonly failClose?: () => boolean; + readonly failLoad?: boolean; + readonly invalidOutput?: boolean; + readonly failAfterTool?: boolean; + } = {}, +) => { + const started = barrier(); + const calls = vi.fn<(n: number, context: ServerContext) => void>(); + const closed = vi.fn(); + let loads = 0; + const server = new DeferredEvidenceMcpServer( + catalog, + new ToolResultDelivery(1024 * 1024), + async (target) => { + loads++; + target.ownRuntimeCleanup(async () => { + closed(); + if (options.failClose?.()) throw new Error("cleanup denied"); + await target.closePartialRuntime(); + }); + started.release(); + await options.hold?.promise; + if (options.failLoad) throw new Error("PRIVATE startup cause"); + target.materialize(() => { + target.registerTool( + "fixture", + { inputSchema: input, outputSchema: output }, + async (args, context) => { + calls(args.n, context); + return { + content: [], + structuredContent: { + n: options.invalidOutput ? "invalid" : args.n, + }, + }; + }, + ); + if (options.failAfterTool) + throw new Error("PRIVATE partial startup failure"); + const completion = target.bindPromptCompletion({ + complete: async () => ["Example"], + }); + target.registerCanonicalPrompt( + "explain", + { + title: "Explain", + description: "Explain a document", + argsSchema: promptArgumentsSchema( + { arguments: parameters }, + completion, + ), + }, + async (args) => ({ + messages: [ + { + role: "user", + content: { type: "text", text: String(args.document) }, + }, + ], + }), + ); + }); + }, + ); + const client = new Client({ name: "deferred-fixture-client", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + cleanups.push(async () => { + await client.close(); + await server.close(); + }); + return { + server, + client, + serverTransport, + started, + calls, + closed, + loads: () => loads, + }; +}; + +it("discovers without initialization, then shares one canonical runtime and preserves request metadata", async () => { + const hold = barrier(); + const f = await fixture({ hold }); + const before = await f.client.listTools(); + expect(before.tools).toEqual(catalog.tools); + expect((await f.client.listPrompts()).prompts).toEqual( + catalog.prompts.map(({ parameters: _parameters, ...prompt }) => prompt), + ); + expect(f.loads()).toBe(0); + const changed = vi.fn(); + f.client.setNotificationHandler("notifications/tools/list_changed", changed); + const a = f.client.callTool({ + name: "fixture", + arguments: {}, + _meta: { caller: "retained" }, + }); + await f.started.promise; + const b = f.client.callTool({ name: "fixture", arguments: { n: 7 } }); + hold.release(); + expect((await a).structuredContent).toEqual({ n: 3 }); + expect((await b).structuredContent).toEqual({ n: 7 }); + expect(f.loads()).toBe(1); + expect(f.calls.mock.calls[0]?.[1].mcpReq._meta?.caller).toBe("retained"); + expect(await f.client.listTools()).toEqual(before); + expect(changed).not.toHaveBeenCalled(); + const invalid = await f.client.callTool({ + name: "fixture", + arguments: { n: 1.5 }, + }); + expect(invalid.isError).toBe(true); + expect(f.calls).toHaveBeenCalledTimes(2); +}); + +it("preserves output validation after deferred registration", async () => { + const f = await fixture({ invalidOutput: true }); + const result = await f.client.callTool({ name: "fixture", arguments: {} }); + expect(result.isError).toBe(true); +}); + +it("initializes through prompt retrieval or completion without an internal client", async () => { + const f = await fixture(); + expect( + ( + await f.client.complete({ + ref: { type: "ref/prompt", name: "explain" }, + argument: { name: "document", value: "Ex" }, + }) + ).completion.values, + ).toEqual(["Example"]); + expect( + ( + await f.client.getPrompt({ + name: "explain", + arguments: { document: "Example" }, + }) + ).messages[0]?.content, + ).toEqual({ type: "text", text: "Example" }); + expect(f.loads()).toBe(1); +}); + +it("retrieves a prompt as the first operation", async () => { + const f = await fixture(); + const result = await f.client.getPrompt({ + name: "explain", + arguments: { document: "Example" }, + }); + expect(result.messages).toHaveLength(1); + expect(f.loads()).toBe(1); +}); + +it("does not execute a tool cancelled while canonical loading is pending", async () => { + const hold = barrier(); + const f = await fixture({ hold }); + const cancelled = barrier(); + const previous = f.serverTransport.onmessage; + f.serverTransport.onmessage = (message, extra) => { + previous?.(message, extra); + if ("method" in message && message.method === "notifications/cancelled") + cancelled.release(); + }; + const controller = new AbortController(); + const pending = f.client.callTool( + { name: "fixture", arguments: {} }, + { signal: controller.signal }, + ); + const rejected = expect(pending).rejects.toBeDefined(); + await f.started.promise; + controller.abort(); + await cancelled.promise; + hold.release(); + await rejected; + await f.server.ensureRuntime(); + expect(f.calls).not.toHaveBeenCalled(); + expect( + (await f.client.callTool({ name: "fixture", arguments: {} })).isError, + ).not.toBe(true); +}); + +it("keeps an independent caller alive when its peer cancels during shared loading", async () => { + const hold = barrier(); + const f = await fixture({ hold }); + const secondRequest = barrier(); + const cancelled = barrier(); + let toolCalls = 0; + const previous = f.serverTransport.onmessage; + f.serverTransport.onmessage = (message, extra) => { + previous?.(message, extra); + if ( + "method" in message && + message.method === "tools/call" && + ++toolCalls === 2 + ) + secondRequest.release(); + if ("method" in message && message.method === "notifications/cancelled") + cancelled.release(); + }; + const controller = new AbortController(); + const a = f.client + .callTool({ name: "fixture", arguments: {} }, { signal: controller.signal }) + .catch(() => undefined); + await f.started.promise; + const b = f.client.callTool({ name: "fixture", arguments: { n: 8 } }); + await secondRequest.promise; + controller.abort(); + await cancelled.promise; + hold.release(); + await a; + expect((await b).structuredContent).toEqual({ n: 8 }); + expect(f.calls).toHaveBeenCalledTimes(1); + expect(f.loads()).toBe(1); +}); + +it("joins late initialization during close and prevents target execution", async () => { + const hold = barrier(); + const f = await fixture({ hold }); + const pending = f.client + .callTool({ name: "fixture", arguments: {} }) + .catch(() => undefined); + await f.started.promise; + const closing = f.server.close(); + hold.release(); + await closing; + await pending; + expect(f.calls).not.toHaveBeenCalled(); + expect(f.closed).toHaveBeenCalledTimes(1); + await expect(f.server.ensureRuntime()).rejects.toThrow("closed"); +}); + +it("retains failed cleanup for retry without loading a replacement runtime", async () => { + let denied = true; + const f = await fixture({ failClose: () => denied }); + await f.client.callTool({ name: "fixture", arguments: {} }); + await expect(f.server.close()).rejects.toThrow("cleanup"); + await expect(f.server.ensureRuntime()).rejects.toThrow("closed"); + denied = false; + await f.server.close(); + expect(f.closed).toHaveBeenCalledTimes(2); + expect(f.loads()).toBe(1); +}); + +it("keeps startup failures redacted and owns partial cleanup", async () => { + const f = await fixture({ failLoad: true }); + const result = await f.client.callTool({ name: "fixture", arguments: {} }); + expect(result.isError).toBe(true); + expect(JSON.stringify(result)).not.toContain("PRIVATE"); + await f.server.close(); + expect(f.closed).toHaveBeenCalledTimes(1); +}); + +it("closes an unused instance without constructing the runtime", async () => { + const f = await fixture(); + await f.client.listTools(); + await f.server.close(); + expect(f.loads()).toBe(0); + expect(f.closed).not.toHaveBeenCalled(); +}); + +it("never executes a partially registered handler after initialization fails", async () => { + const f = await fixture({ failAfterTool: true }); + const first = await f.client.callTool({ name: "fixture", arguments: {} }); + const retry = await f.client.callTool({ name: "fixture", arguments: {} }); + expect(first.isError).toBe(true); + expect(retry.isError).toBe(true); + expect(f.calls).not.toHaveBeenCalled(); + expect(f.loads()).toBe(1); + await f.server.close(); + expect(f.closed).toHaveBeenCalledTimes(1); +}); diff --git a/src/server/DeferredEvidenceMcpServer.ts b/src/server/DeferredEvidenceMcpServer.ts new file mode 100644 index 0000000000..5095303340 --- /dev/null +++ b/src/server/DeferredEvidenceMcpServer.ts @@ -0,0 +1,321 @@ +import type { + RegisteredTool, + RegisteredPrompt, + StandardSchemaWithJSON, + ToolCallback, + PromptCallback, +} from "@modelcontextprotocol/server"; +import type { PromptCompletionSource } from "./promptCompletion.js"; +import type { RecordedMcpCatalog } from "../recordedMcpCatalog.js"; +import type { ToolResultDelivery } from "./toolResult.js"; +import { + EvidenceMcpServerCore, + currentMcpRequestSignal, + type StandardToolRegistration, +} from "./EvidenceMcpServerCore.js"; +import { promptArgumentsSchema } from "./promptArguments.js"; +import { SERVER_START_FAILED } from "../main/messages.js"; + +type ToolConfig = StandardToolRegistration; +type InputCallback = ToolCallback; +interface DeferredTool { + readonly registered: RegisteredTool; + readonly input: Record; + readonly output: Record | undefined; + callback?: InputCallback; +} +interface DeferredPrompt { + readonly registered: RegisteredPrompt; + callback?: PromptCallback; +} + +/** Borrow the build-owned wire projection while retaining the canonical parser. */ +const recordedProjection = ( + schema: StandardSchemaWithJSON, + projection: Record, +): StandardSchemaWithJSON => ({ + "~standard": { + ...schema["~standard"], + jsonSchema: { input: () => projection, output: () => projection }, + }, +}); + +/** One SDK server, with async canonical validation installed on first use. */ +export class DeferredEvidenceMcpServer extends EvidenceMcpServerCore { + private readonly tools = new Map(); + private readonly prompts = new Map(); + private loading: Promise | undefined; + private closing = false; + private closeAttempt: Promise | undefined; + private protocolClosed = false; + private runtimeClosed = false; + private runtimeCleanup: (() => Promise) | undefined; + private partialClose: (() => Promise) | undefined; + private materializing = false; + private readonly protocolClose = () => super.close(); + + constructor( + catalog: RecordedMcpCatalog, + delivery: ToolResultDelivery, + private readonly initialize: ( + server: DeferredEvidenceMcpServer, + ) => Promise, + ) { + super( + catalog.info, + { + capabilities: catalog.capabilities, + instructions: catalog.instructions, + }, + undefined, + delivery, + undefined, + false, + catalog.identity, + ); + for (const tool of catalog.tools) { + const { name, inputSchema, outputSchema, ...metadata } = tool; + let binding: DeferredTool; + const input: StandardSchemaWithJSON = { + "~standard": { + version: 1, + vendor: "rea", + jsonSchema: { input: () => inputSchema, output: () => inputSchema }, + validate: async (value) => { + await this.ensureRuntime(); + const canonical = binding.registered.inputSchema; + if (canonical === undefined || canonical === input) + throw new Error(SERVER_START_FAILED); + return canonical["~standard"].validate(value); + }, + }, + }; + const output: StandardSchemaWithJSON | undefined = + outputSchema === undefined + ? undefined + : { + "~standard": { + version: 1, + vendor: "rea", + jsonSchema: { + input: () => outputSchema, + output: () => outputSchema, + }, + validate: async (value) => { + await this.ensureRuntime(); + const canonical = binding.registered.outputSchema; + if (canonical === undefined || canonical === output) + throw new Error(SERVER_START_FAILED); + return canonical["~standard"].validate(value); + }, + }, + }; + const registered = super.bindTool( + name, + { + ...metadata, + inputSchema: input, + ...(output === undefined ? {} : { outputSchema: output }), + } as StandardToolRegistration, + (async (args, ctx) => { + await this.ensureRuntime(); + ctx.mcpReq.signal.throwIfAborted(); + if (binding.callback === undefined) + throw new Error(SERVER_START_FAILED); + return binding.callback(args, ctx); + }) as InputCallback, + ); + binding = { registered, input: inputSchema, output: outputSchema }; + this.tools.set(name, binding); + } + const completion = { + complete: async ( + ...args: Parameters< + NonNullable["complete"] + > + ) => { + await this.ensureRuntime(); + currentMcpRequestSignal()?.throwIfAborted(); + if (this.promptCompletion === undefined) + throw new Error(SERVER_START_FAILED); + return this.promptCompletion.complete(...args); + }, + }; + for (const prompt of catalog.prompts) { + const { name, parameters, title, description } = prompt; + let binding: DeferredPrompt; + const argsSchema = promptArgumentsSchema( + { arguments: parameters }, + completion, + ); + const registered = super.registerCanonicalPrompt( + name, + { + ...(title === undefined ? {} : { title }), + ...(description === undefined ? {} : { description }), + argsSchema, + }, + async (args, ctx) => { + await this.ensureRuntime(); + ctx.mcpReq.signal.throwIfAborted(); + if (binding.callback === undefined) + throw new Error(SERVER_START_FAILED); + return binding.callback(args, ctx); + }, + ); + binding = { registered }; + this.prompts.set(name, binding); + } + } + + ensureRuntime(): Promise { + if (this.closing) + return Promise.reject(new Error("MCP connection is closed")); + this.loading ??= Promise.resolve() + .then(() => this.initialize(this)) + .catch(() => { + throw new Error(SERVER_START_FAILED); + }); + return this.loading; + } + + /** Register an awaitable cleanup capability before runtime startup can fail. */ + ownRuntimeCleanup(close: () => Promise): void { + this.runtimeCleanup = close; + } + + /** Canonical composition may wrap close; retain that exact cleanup capability. */ + materialize(build: () => void): void { + const ownedClose = this.close.bind(this); + this.materializing = true; + this.close = this.protocolClose; + try { + build(); + if (this.tools.size !== 0 || this.prompts.size !== 0) + throw new Error("MCP catalog does not match runtime registrations"); + } finally { + this.partialClose = this.close.bind(this); + this.close = ownedClose; + this.materializing = false; + } + } + + closePartialRuntime(): Promise { + return this.partialClose?.() ?? Promise.resolve(); + } + + protected override bindTool( + name: string, + config: ToolConfig, + callback: ToolCallback, + ): RegisteredTool { + const binding = this.tools.get(name); + if ( + !this.materializing || + binding === undefined || + config.inputSchema === undefined || + (config.outputSchema === undefined) !== (binding.output === undefined) + ) + throw new Error("Unexpected MCP runtime registration"); + const canonical = callback as InputCallback; + const guarded: InputCallback = async (args, ctx) => { + ctx.mcpReq.signal.throwIfAborted(); + await this.ensureRuntime(); + ctx.mcpReq.signal.throwIfAborted(); + return canonical(args, ctx); + }; + binding.callback = guarded; + const { inputSchema: _input, outputSchema: _output, ...metadata } = config; + binding.registered.update({ + ...metadata, + paramsSchema: recordedProjection(config.inputSchema, binding.input), + ...(config.outputSchema === undefined || binding.output === undefined + ? {} + : { + outputSchema: recordedProjection( + config.outputSchema, + binding.output, + ), + }), + callback: guarded, + }); + this.tools.delete(name); + return binding.registered; + } + + override registerCanonicalPrompt( + name: string, + config: { title?: string; description?: string; argsSchema: Args }, + callback: PromptCallback, + ): RegisteredPrompt { + const binding = this.prompts.get(name); + if (!this.materializing || binding === undefined) + throw new Error("Unexpected MCP runtime prompt"); + const canonical = callback as PromptCallback; + binding.callback = async (args, ctx) => { + ctx.mcpReq.signal.throwIfAborted(); + await this.ensureRuntime(); + ctx.mcpReq.signal.throwIfAborted(); + return canonical(args, ctx); + }; + binding.registered.update({ + ...config, + callback: binding.callback, + }); + this.prompts.delete(name); + return binding.registered; + } + + override bindPromptCompletion( + completion: PromptCompletionSource, + ): PromptCompletionSource { + return super.bindPromptCompletion({ + complete: async (...args) => { + await this.ensureRuntime(); + currentMcpRequestSignal()?.throwIfAborted(); + return completion.complete(...args); + }, + }); + } + + override sendToolListChanged(): void { + if (!this.materializing) super.sendToolListChanged(); + } + override sendPromptListChanged(): void { + if (!this.materializing) super.sendPromptListChanged(); + } + + override close(): Promise { + this.closing = true; + this.closeAttempt ??= Promise.resolve() + .then(() => this.finishClose()) + .catch((cause) => { + this.closeAttempt = undefined; + throw cause; + }); + return this.closeAttempt; + } + + private async finishClose(): Promise { + const errors: unknown[] = []; + if (!this.protocolClosed) { + try { + await this.protocolClose(); + this.protocolClosed = true; + } catch (cause) { + errors.push(cause); + } + } + await this.loading?.catch(() => undefined); + if (!this.runtimeClosed && this.runtimeCleanup !== undefined) { + try { + await this.runtimeCleanup(); + this.runtimeClosed = true; + } catch (cause) { + errors.push(cause); + } + } + if (errors.length > 0) + throw new AggregateError(errors, "MCP runtime cleanup failed"); + } +} diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts index 59e980c4b9..5ad934a5e1 100644 --- a/src/server/EvidenceMcpServer.ts +++ b/src/server/EvidenceMcpServer.ts @@ -1,643 +1,48 @@ -import { AsyncLocalStorage } from "node:async_hooks"; - -import { - isCallToolResult, - isJSONRPCErrorResponse, - isJSONRPCRequest, - isJSONRPCNotification, - isJSONRPCResultResponse, - type CallToolResult, - type Implementation, - type Icon, - type JSONRPCMessage, - type McpServerOptions, - type RegisteredTool, - type RequestId, - type ScopeChallengeHandler, - type StandardSchemaWithJSON, - type ToolAnnotations, - type ToolCallback, - type Transport, - type TransportSendOptions, +import type { + Implementation, + McpServerOptions, } from "@modelcontextprotocol/server"; - -import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; -import { - presentAdvertisedInputSchema, - type CompactInputSchemaPresentation, -} from "../contracts/compactInputSchema.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { FULL_CATALOG_IDENTITY, createCatalogIdentity, } from "../catalogIdentity.js"; -import { transformAdvertisedInputJsonSchema } from "../contracts/toolSchemaMetadata.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import type { ToolResultDelivery } from "./toolResult.js"; import { - AnalysisInputError, - AnalysisResourceConstraintError, -} from "../domain/analysisErrorCore.js"; -import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; -import { createEvidence } from "../domain/evidence.js"; -import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; -import { - mcpResponseOverflowProjection, - type ToolResultDelivery, -} from "./toolResult.js"; -import { toolReturnsTargetContent } from "../contracts/toolEffects.js"; -import { - carriesTargetContentMarker, - encodeToolResult, - measureMcpResultWithinBudget, - TARGET_CONTENT_RESULT_META, -} from "./toolResultEncoding.js"; -import { RequestTrackingMcpServer } from "./RequestTrackingMcpServer.js"; + EvidenceMcpServerCore, + type CompactInputSchemaPresentation, +} from "./EvidenceMcpServerCore.js"; -export type { CompactInputSchemaPresentation } from "../contracts/compactInputSchema.js"; - -const currentRequestSignal = new AsyncLocalStorage(); - -/** Signal for the completion request currently being dispatched by the SDK. */ -export const currentMcpRequestSignal = (): AbortSignal | undefined => - currentRequestSignal.getStore(); - -/** A Zod schema whose standard interface exposes a JSON Schema projection. */ -const hasAdvertisedInputJsonSchema = (value: unknown): boolean => { - if (typeof value !== "object" || value === null) return false; - const standard = Reflect.get(value, "~standard"); - if (typeof standard !== "object" || standard === null) return false; - const jsonSchema = Reflect.get(standard, "jsonSchema"); - return ( - typeof jsonSchema === "object" && - jsonSchema !== null && - Reflect.get(jsonSchema, "input") !== undefined - ); -}; - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -/** Recognize the SDK's argument diagnostic after its wire size is checked. */ -const sdkInputValidationDiagnostic = ( - text: string | undefined, - tool: string | undefined, -): string | undefined => { - if (text === undefined || tool === undefined) return undefined; - const prefix = `Input validation error: Invalid arguments for tool ${tool}`; - if (!text.startsWith(prefix) || text[prefix.length] !== ":") return undefined; - const diagnostic = text.slice(prefix.length + 1).trimStart(); - return diagnostic.length === 0 ? undefined : diagnostic; -}; - -/** Replace the tool payload after the SDK codec while retaining its envelope. */ -const replaceToolResultPayload = ( - original: CallToolResult, - replacement: CallToolResult, -): CallToolResult => { - const { - content: _content, - structuredContent: _structuredContent, - isError: _isError, - _meta, - ...envelope - } = original; - return { - ...envelope, - ...replacement, - ...(_meta === undefined && replacement._meta === undefined - ? {} - : { _meta: { ..._meta, ...replacement._meta } }), - }; -}; - -/** Remove only client-facing output schemas; SDK registrations stay canonical. */ -const omitOutputSchemasFromToolsList = ( - message: JSONRPCMessage, - requestId: RequestId, -): JSONRPCMessage => { - if ( - !isJSONRPCResultResponse(message) || - message.id !== requestId || - !isRecord(message.result) || - !Array.isArray(message.result.tools) - ) - return message; - return { - ...message, - result: { - ...message.result, - tools: message.result.tools.map((tool: unknown) => { - if (!isRecord(tool)) return tool; - const { outputSchema: _outputSchema, ...advertisedTool } = tool; - return advertisedTool; - }), - }, - } as JSONRPCMessage; -}; - -/** Release every transport-owned reference for a cancelled/completed request. */ -export const forgetPendingMcpRequest = ( - pendingTools: Map, - pendingToolLists: Set, - pendingCompletions: Map, - id: RequestId, -): void => { - pendingTools.delete(id); - pendingToolLists.delete(id); - pendingCompletions.get(id)?.abort(); - pendingCompletions.delete(id); -}; - -/** Bind oversized MCP error recovery to this server's Evidence ledger. */ -export class EvidenceMcpServer extends RequestTrackingMcpServer { - readonly catalogIdentity: ReturnType; - private readonly compactPresentation: - | CompactInputSchemaPresentation - | undefined; +export { + forgetPendingMcpRequest, + type CompactInputSchemaPresentation, +} from "./EvidenceMcpServerCore.js"; +/** Canonical composition retains the constructor's existing identity defaults. */ +export class EvidenceMcpServer extends EvidenceMcpServerCore { constructor( info: Implementation, options: McpServerOptions, - private readonly recordEvidence: - | EvidenceWriter["recordEvidence"] - | undefined, - readonly delivery: ToolResultDelivery, + recordEvidence: EvidenceWriter["recordEvidence"] | undefined, + delivery: ToolResultDelivery, compactPresentation: CompactInputSchemaPresentation | undefined = undefined, - private readonly omitOutputSchemas = false, + omitOutputSchemas = false, + identity?: ReturnType, ) { - super(info, options); - this.compactPresentation = - compactPresentation === undefined - ? undefined - : { budgetBytes: compactPresentation.budgetBytes }; - this.catalogIdentity = - this.compactPresentation === undefined && !this.omitOutputSchemas - ? FULL_CATALOG_IDENTITY - : createCatalogIdentity( - this.compactPresentation, - this.omitOutputSchemas ? "off" : "full", - ); - } - - /** - * Compact advertised input schemas and optionally omit output-schema - * advertisements. Canonical parsers and contracts remain registered; the - * output-schema-off path validates successful structured results locally. - * The signatures mirror the SDK's registrations so call sites keep their - * inferred handler argument types. - */ - override registerTool< - OutputArgs extends StandardSchemaWithJSON, - InputArgs extends StandardSchemaWithJSON | undefined = undefined, - >( - name: string, - config: { - title?: string; - description?: string; - inputSchema?: InputArgs; - outputSchema?: OutputArgs; - annotations?: ToolAnnotations; - icons?: Icon[]; - scopeChallenge?: ScopeChallengeHandler; - _meta?: Record; - }, - callback: ToolCallback, - ): RegisteredTool { - const budget = this.compactPresentation?.budgetBytes; - const inputSchema = - budget !== undefined && - config !== undefined && - hasAdvertisedInputJsonSchema(config.inputSchema) - ? transformAdvertisedInputJsonSchema( - config.inputSchema as Parameters< - typeof transformAdvertisedInputJsonSchema - >[0], - (projected) => - presentAdvertisedInputSchema(projected, { budgetBytes: budget }), - ) - : undefined; - const registeredConfig = - inputSchema === undefined ? config : { ...config, inputSchema }; - return super.registerTool( - name, - registeredConfig as never, - callback as never, - ); - } - - /** Preserve the SDK transport lifecycle while retaining oversized errors. */ - override connect(transport: Transport): Promise { - return super.connect( - new ErrorEvidenceTransport( - transport, - this.recordEvidence, - this.delivery, - this.omitOutputSchemas, - ), + super( + info, + options, + recordEvidence, + delivery, + compactPresentation, + omitOutputSchemas, + identity ?? + (compactPresentation === undefined && !omitOutputSchemas + ? FULL_CATALOG_IDENTITY + : createCatalogIdentity( + compactPresentation, + omitOutputSchemas ? "off" : "full", + )), ); } } - -class ErrorEvidenceTransport implements Transport { - onclose: Transport["onclose"]; - onerror: Transport["onerror"]; - onmessage: Transport["onmessage"]; - readonly hasPerRequestStream?: boolean; - private readonly pendingTools = new Map(); - private readonly pendingToolLists = new Set(); - private readonly pendingCompletions = new Map(); - - constructor( - private readonly transport: Transport, - private readonly recordEvidence: - | EvidenceWriter["recordEvidence"] - | undefined, - private readonly delivery: ToolResultDelivery, - private readonly omitOutputSchemas: boolean, - ) { - if (transport.hasPerRequestStream !== undefined) - this.hasPerRequestStream = transport.hasPerRequestStream; - transport.onmessage = (message, extra) => { - if ( - isJSONRPCRequest(message) && - message.method === "tools/call" && - typeof message.params?.["name"] === "string" - ) - this.pendingTools.set(message.id, message.params["name"]); - if (isJSONRPCRequest(message) && message.method === "tools/list") - this.pendingToolLists.add(message.id); - if (isJSONRPCRequest(message) && message.method === "completion/complete") - this.pendingCompletions.set(message.id, new AbortController()); - // The SDK suppresses replies after cancellation, so send() cannot release - // this bookkeeping. Keep transport authentication outside retained data. - if ( - isJSONRPCNotification(message) && - message.method === "notifications/cancelled" - ) { - const id = message.params?.["requestId"]; - if (typeof id === "string" || typeof id === "number") - forgetPendingMcpRequest( - this.pendingTools, - this.pendingToolLists, - this.pendingCompletions, - id, - ); - } - const completion = - isJSONRPCRequest(message) && message.method === "completion/complete" - ? this.pendingCompletions.get(message.id) - : undefined; - if (completion === undefined) this.onmessage?.(message, extra); - else - currentRequestSignal.run(completion.signal, () => - this.onmessage?.(message, extra), - ); - }; - transport.onerror = (error) => this.onerror?.(error); - transport.onclose = () => { - this.pendingTools.clear(); - this.pendingToolLists.clear(); - for (const completion of this.pendingCompletions.values()) - completion.abort(); - this.pendingCompletions.clear(); - this.onclose?.(); - }; - } - - get sessionId(): string | undefined { - return this.transport.sessionId; - } - set sessionId(value: string | undefined) { - this.transport.sessionId = value; - } - - start(): Promise { - return this.transport.start(); - } - - async close(): Promise { - try { - await this.transport.close(); - } finally { - this.pendingTools.clear(); - this.pendingToolLists.clear(); - for (const completion of this.pendingCompletions.values()) - completion.abort(); - this.pendingCompletions.clear(); - } - } - - setProtocolVersion(version: string): void { - this.transport.setProtocolVersion?.(version); - } - - setSupportedProtocolVersions(versions: string[]): void { - this.transport.setSupportedProtocolVersions?.(versions); - } - - async send( - message: JSONRPCMessage, - options?: TransportSendOptions, - ): Promise { - const id = - "id" in message && - (typeof message.id === "string" || typeof message.id === "number") - ? message.id - : undefined; - const tool = id === undefined ? undefined : this.pendingTools.get(id); - const omitToolSchemas = - this.omitOutputSchemas && - id !== undefined && - this.pendingToolLists.has(id); - try { - const recovered = this.recoverProtocolError(message, tool); - const advertised = omitToolSchemas - ? omitOutputSchemasFromToolsList(recovered, id) - : recovered; - if ( - isJSONRPCResultResponse(advertised) && - isCallToolResult(advertised.result) && - advertised.result.isError === true - ) { - // Keep the complete typed JSON in content. Error projections are not - // success outputSchema data; oversized recovery must consume their - // private structured carrier before it is removed from the wire. - const { structuredContent: _diagnostic, ...result } = advertised.result; - // Errors the SDK builds itself (input validation, a thrown handler) - // bypass ToolResultDelivery; mark them for target-content tools too. - const marked = - tool !== undefined && - toolReturnsTargetContent(tool) && - !carriesTargetContentMarker(result) - ? { - ...result, - _meta: { ...result._meta, ...TARGET_CONTENT_RESULT_META }, - } - : result; - await this.transport.send({ ...advertised, result: marked }, options); - } else await this.transport.send(advertised, options); - } finally { - if (!isJSONRPCRequest(message) && id !== undefined) { - forgetPendingMcpRequest( - this.pendingTools, - this.pendingToolLists, - this.pendingCompletions, - id, - ); - } - } - } - - private recoverError( - message: JSONRPCMessage, - tool: string | undefined, - ): JSONRPCMessage { - if ( - !isJSONRPCResultResponse(message) || - !isCallToolResult(message.result) || - message.result.isError !== true - ) - return message; - const budget = this.delivery.resultBudgetBytes; - // Count the provenance marker exactly as the projected error did, and keep - // it on the replacement: the retained original may hold target content. - const targetContent = - carriesTargetContentMarker(message.result) || - (tool !== undefined && toolReturnsTargetContent(tool)); - // SDK validation failures have no structured carrier. Size their existing - // text-only result without serializing the complete JSON-RPC message. - const isStructured = message.result.structuredContent !== undefined; - const structured = isStructured - ? jsonObjectSchema.parse(message.result.structuredContent) - : undefined; - const sdkResultWithMarker = - !isStructured && - targetContent && - !carriesTargetContentMarker(message.result) - ? { - ...message.result, - _meta: { - ...message.result._meta, - ...TARGET_CONTENT_RESULT_META, - }, - } - : message.result; - const sdkDiagnostic = !isStructured - ? message.result.content.find((item) => item.type === "text")?.text - : undefined; - let encoded = - structured !== undefined - ? encodeToolResult(structured, budget, "error-text", targetContent) - : measureMcpResultWithinBudget(sdkResultWithMarker, budget); - const normalized = encoded.ok - ? this.normalizeSdkValidationError( - message, - tool, - targetContent, - sdkInputValidationDiagnostic(sdkDiagnostic, tool), - ) - : undefined; - if ( - normalized !== undefined && - isJSONRPCResultResponse(normalized) && - isCallToolResult(normalized.result) - ) { - // The structured diagnostic is a private carrier removed before sending. - // Measure the actual wire result, including its preserved SDK metadata. - const { structuredContent, ...wireResult } = normalized.result; - encoded = - normalized.result.content.length === 0 - ? encodeToolResult( - jsonObjectSchema.parse(structuredContent), - budget, - "error-text", - targetContent, - ) - : measureMcpResultWithinBudget(wireResult, budget); - if (encoded.ok) return normalized; - // JSON escaping and recovery fields can exceed the budget even when - // the original SDK result fits. Retain that original before replacing it. - } - if (encoded.ok) return message; - const operation = tool ?? "mcp_tool_error"; - const originalResult = jsonObjectSchema.parse(message.result); - const retainedResult = structured ?? originalResult; - const originalError = isStructured - ? analysisErrorProjectionSchema.safeParse(structured?.["error"]) - : undefined; - // Retain the complete original response, including SDK validation text. - // The SDK error contains its rendered issue, not a separate argument bag; - // transport authentication and caller arguments are never added here. - const evidence = createEvidence( - undefined, - { id: "rea-mcp", name: "REA MCP delivery", version: "1" }, - { - operation: "mcp_tool_error", - parameters: { tool_name: operation }, - result: - sdkDiagnostic === undefined - ? retainedResult - : { - boundary: "sdk-input-validation", - diagnostic_preview: sdkDiagnostic.slice(0, 256), - diagnostic_truncated: sdkDiagnostic.length > 256, - }, - rawResult: originalResult, - }, - ); - const recorded = this.recordEvidence?.(evidence); - const retained = recorded !== undefined && recorded.ok; - const projected = mcpResponseOverflowProjection({ - completed: false, - constraint: encoded.constraint, - effectiveResponseBytes: this.delivery.maximumResponseBytes, - nextStep: retained - ? "Export the retained original error through export_evidence_bundle to a caller-selected path. The connection remains usable." - : encoded.constraint === "string-length" - ? "The original error could not be retained by this server. Use complete CLI JSON output when the analysis completed. Repeating the unchanged MCP call fails the same way." - : "The original error could not be retained by this server. Restart REA with a larger REA_MCP_MAX_RESPONSE_BYTES and a matching client receive buffer, then retry.", - }); - const replacement = this.transportConstraintResult( - operation, - projected.reason, - projected.remediationAction, - { - boundary: "mcp-response", - ...(originalError?.success - ? { original_error_code: originalError.data.code } - : {}), - result_budget_bytes: budget, - response_bytes_at_least: encoded.bytesAtLeast, - constraint: encoded.constraint, - ...(sdkDiagnostic === undefined - ? {} - : { - diagnostic_preview: sdkDiagnostic.slice(0, 256), - diagnostic_truncated: sdkDiagnostic.length > 256, - }), - ...(retained - ? { - evidence_reference: { - kind: "retained-evidence", - evidence_id: evidence.evidence_id, - }, - } - : { retention: recorded === undefined ? "unavailable" : "failed" }), - ...(recorded !== undefined && !recorded.ok - ? { retention_error: projectAnalysisError(recorded.error) } - : {}), - }, - targetContent, - ); - return { - ...message, - result: replaceToolResultPayload(message.result, replacement), - }; - } - - /** - * Encode a transport-constraint error inside the MCP text budget. - * The complete SDK diagnostic stays in retained Evidence. Its character - * preview can still expand past the budget when quotes and backslashes are - * escaped into that text, so shorten only the preview and keep the reference. - */ - private transportConstraintResult( - operation: string, - reason: string, - remediationAction: string, - limits: Readonly>, - targetContent: boolean, - ): CallToolResult { - const deliver = ( - reportedLimits: Readonly>, - ): CallToolResult => - this.delivery.toErrorToolResult( - new AnalysisResourceConstraintError( - operation, - "transport", - reason, - reportedLimits, - { remediationAction }, - ), - targetContent, - ); - const delivered = deliver(limits); - const preview = limits.diagnostic_preview; - if (delivered.content.length > 0 || typeof preview !== "string") - return delivered; - let low = 0; - let high = preview.length - 1; - let fitted: CallToolResult | undefined; - let fittedLength = -1; - while (low <= high) { - const length = Math.floor((low + high) / 2); - const candidate = deliver({ - ...limits, - diagnostic_preview: preview.slice(0, length), - diagnostic_truncated: true, - }); - if (candidate.content.length > 0) { - fitted = candidate; - fittedLength = length; - low = length + 1; - } else high = length - 1; - } - if (fitted !== undefined && fittedLength > 0) return fitted; - const { diagnostic_preview: _preview, ...withoutPreview } = limits; - const omitted = deliver({ - ...withoutPreview, - diagnostic_truncated: preview.length > 0, - }); - return omitted.content.length > 0 || fitted === undefined - ? omitted - : fitted; - } - - private normalizeSdkValidationError( - message: JSONRPCMessage, - tool: string | undefined, - targetContent: boolean, - diagnostic: string | undefined, - ): JSONRPCMessage | undefined { - if ( - tool === undefined || - diagnostic === undefined || - !isJSONRPCResultResponse(message) || - !isCallToolResult(message.result) - ) - return undefined; - const error = new AnalysisInputError( - tool, - { - userMessage: `The arguments for MCP tool ${tool} do not match its advertised input schema.`, - }, - [{ path: [], reason: "invalid_value", message: diagnostic }], - ); - const result = this.delivery.toErrorToolResult(error, targetContent); - return { - ...message, - result: replaceToolResultPayload(message.result, result), - }; - } - - private recoverProtocolError( - message: JSONRPCMessage, - tool: string | undefined, - ): JSONRPCMessage { - const recovered = this.recoverError(message, tool); - if ( - !isJSONRPCErrorResponse(recovered) || - tool === undefined || - recovered.error.code !== -32602 || - recovered.error.message !== `Tool ${tool} not found` - ) - return recovered; - return { - ...recovered, - error: { - ...recovered.error, - message: - "The requested MCP tool is not advertised by this REA server. Call tools/list, select an advertised tool, and retry; do not repeat the unknown tool name.", - }, - }; - } -} diff --git a/src/server/EvidenceMcpServerCore.ts b/src/server/EvidenceMcpServerCore.ts new file mode 100644 index 0000000000..77a03fa8de --- /dev/null +++ b/src/server/EvidenceMcpServerCore.ts @@ -0,0 +1,699 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +import { + isCallToolResult, + isJSONRPCErrorResponse, + isJSONRPCRequest, + isJSONRPCNotification, + isJSONRPCResultResponse, + type CallToolResult, + type Implementation, + type Icon, + type JSONRPCMessage, + type McpServerOptions, + type RegisteredTool, + type RegisteredPrompt, + type PromptCallback, + type RequestId, + type ScopeChallengeHandler, + type StandardSchemaWithJSON, + type ToolAnnotations, + type ToolCallback, + type Transport, + type TransportSendOptions, +} from "@modelcontextprotocol/server"; + +import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; +import { + presentAdvertisedInputSchema, + type CompactInputSchemaPresentation, +} from "../contracts/compactInputSchema.js"; +import type { createCatalogIdentity } from "../catalogIdentity.js"; +import { transformAdvertisedInputJsonSchema } from "../contracts/toolSchemaMetadata.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { createEvidence } from "../domain/evidence.js"; +import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; +import { + mcpResponseOverflowProjection, + type ToolResultDelivery, +} from "./ToolResultDelivery.js"; +import { toolReturnsTargetContent } from "../contracts/toolEffects.js"; +import { + carriesTargetContentMarker, + encodeToolResult, + measureMcpResultWithinBudget, + TARGET_CONTENT_RESULT_META, +} from "./toolResultEncoding.js"; +import { RequestTrackingMcpServer } from "./RequestTrackingMcpServer.js"; + +export type { CompactInputSchemaPresentation } from "../contracts/compactInputSchema.js"; + +const currentRequestSignal = new AsyncLocalStorage(); + +/** Signal for the completion request currently being dispatched by the SDK. */ +export const currentMcpRequestSignal = (): AbortSignal | undefined => + currentRequestSignal.getStore(); + +/** A Zod schema whose standard interface exposes a JSON Schema projection. */ +const hasAdvertisedInputJsonSchema = (value: unknown): boolean => { + if (typeof value !== "object" || value === null) return false; + const standard = Reflect.get(value, "~standard"); + if (typeof standard !== "object" || standard === null) return false; + const jsonSchema = Reflect.get(standard, "jsonSchema"); + return ( + typeof jsonSchema === "object" && + jsonSchema !== null && + Reflect.get(jsonSchema, "input") !== undefined + ); +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Recognize the SDK's argument diagnostic after its wire size is checked. */ +const sdkInputValidationDiagnostic = ( + text: string | undefined, + tool: string | undefined, +): string | undefined => { + if (text === undefined || tool === undefined) return undefined; + const prefix = `Input validation error: Invalid arguments for tool ${tool}`; + if (!text.startsWith(prefix) || text[prefix.length] !== ":") return undefined; + const diagnostic = text.slice(prefix.length + 1).trimStart(); + return diagnostic.length === 0 ? undefined : diagnostic; +}; + +/** Replace the tool payload after the SDK codec while retaining its envelope. */ +const replaceToolResultPayload = ( + original: CallToolResult, + replacement: CallToolResult, +): CallToolResult => { + const { + content: _content, + structuredContent: _structuredContent, + isError: _isError, + _meta, + ...envelope + } = original; + return { + ...envelope, + ...replacement, + ...(_meta === undefined && replacement._meta === undefined + ? {} + : { _meta: { ..._meta, ...replacement._meta } }), + }; +}; + +/** Remove only client-facing output schemas; SDK registrations stay canonical. */ +const omitOutputSchemasFromToolsList = ( + message: JSONRPCMessage, + requestId: RequestId, +): JSONRPCMessage => { + if ( + !isJSONRPCResultResponse(message) || + message.id !== requestId || + !isRecord(message.result) || + !Array.isArray(message.result.tools) + ) + return message; + return { + ...message, + result: { + ...message.result, + tools: message.result.tools.map((tool: unknown) => { + if (!isRecord(tool)) return tool; + const { outputSchema: _outputSchema, ...advertisedTool } = tool; + return advertisedTool; + }), + }, + } as JSONRPCMessage; +}; + +/** Release every transport-owned reference for a cancelled/completed request. */ +export const forgetPendingMcpRequest = ( + pendingTools: Map, + pendingToolLists: Set, + pendingCompletions: Map, + id: RequestId, +): void => { + pendingTools.delete(id); + pendingToolLists.delete(id); + pendingCompletions.get(id)?.abort(); + pendingCompletions.delete(id); +}; +import type { PromptCompletionSource } from "./promptCompletion.js"; + +/** Standard-schema registration boundary shared with deferred SDK bindings. */ +export interface StandardToolRegistration { + readonly title?: string; + readonly description?: string; + readonly inputSchema?: StandardSchemaWithJSON; + readonly outputSchema?: StandardSchemaWithJSON; + readonly annotations?: ToolAnnotations; + readonly icons?: Icon[]; + readonly scopeChallenge?: ScopeChallengeHandler; + readonly _meta?: Record; +} + +/** Bind oversized MCP error recovery to this server's Evidence ledger. */ +export class EvidenceMcpServerCore extends RequestTrackingMcpServer { + private identity: ReturnType | undefined; + + /** Composition supplies identity without importing the canonical schema graph. */ + get catalogIdentity(): ReturnType { + if (this.identity === undefined) + throw new Error("MCP catalog identity has not been configured"); + return this.identity; + } + + /** Attach the process-owned session only after lazy runtime construction. */ + attachRuntime( + recordEvidence: EvidenceWriter["recordEvidence"] | undefined, + delivery: ToolResultDelivery, + identity: ReturnType, + ): void { + if ( + this.identity !== undefined && + this.identity.digests.combined_sha256 !== identity.digests.combined_sha256 + ) + throw new Error("MCP catalog identity does not match runtime"); + this.recordEvidence = recordEvidence; + this.delivery = delivery; + this.identity = identity; + } + + protected promptCompletion: PromptCompletionSource | undefined; + bindPromptCompletion( + completion: PromptCompletionSource, + ): PromptCompletionSource { + this.promptCompletion = completion; + return completion; + } + + registerCanonicalPrompt( + name: string, + config: { title?: string; description?: string; argsSchema: Args }, + callback: PromptCallback, + ): RegisteredPrompt { + return super.registerPrompt(name, config, callback); + } + private readonly compactPresentation: + | CompactInputSchemaPresentation + | undefined; + + constructor( + info: Implementation, + options: McpServerOptions, + private recordEvidence: EvidenceWriter["recordEvidence"] | undefined, + public delivery: ToolResultDelivery, + compactPresentation: CompactInputSchemaPresentation | undefined = undefined, + private readonly omitOutputSchemas = false, + identity?: ReturnType, + ) { + super(info, options); + this.compactPresentation = + compactPresentation === undefined + ? undefined + : { budgetBytes: compactPresentation.budgetBytes }; + this.identity = identity; + } + + /** + * Compact advertised input schemas and optionally omit output-schema + * advertisements. Canonical parsers and contracts remain registered; the + * output-schema-off path validates successful structured results locally. + * The signatures mirror the SDK's registrations so call sites keep their + * inferred handler argument types. + */ + override registerTool< + OutputArgs extends StandardSchemaWithJSON, + InputArgs extends StandardSchemaWithJSON | undefined = undefined, + >( + name: string, + config: { + title?: string; + description?: string; + inputSchema?: InputArgs; + outputSchema?: OutputArgs; + annotations?: ToolAnnotations; + icons?: Icon[]; + scopeChallenge?: ScopeChallengeHandler; + _meta?: Record; + }, + callback: ToolCallback, + ): RegisteredTool { + const budget = this.compactPresentation?.budgetBytes; + const inputSchema = + budget !== undefined && + config !== undefined && + hasAdvertisedInputJsonSchema(config.inputSchema) + ? transformAdvertisedInputJsonSchema( + config.inputSchema as Parameters< + typeof transformAdvertisedInputJsonSchema + >[0], + (projected) => + presentAdvertisedInputSchema(projected, { budgetBytes: budget }), + ) + : undefined; + const registeredConfig = + inputSchema === undefined ? config : { ...config, inputSchema }; + return this.bindTool(name, registeredConfig as never, callback as never); + } + + protected bindTool( + name: string, + config: StandardToolRegistration, + callback: ToolCallback, + ): RegisteredTool { + return super.registerTool(name, config as never, callback as never); + } + + /** Preserve the SDK transport lifecycle while retaining oversized errors. */ + override connect(transport: Transport): Promise { + return super.connect( + new ErrorEvidenceTransport( + transport, + () => this.recordEvidence, + () => this.delivery, + this.omitOutputSchemas, + ), + ); + } +} + +class ErrorEvidenceTransport implements Transport { + onclose: Transport["onclose"]; + onerror: Transport["onerror"]; + onmessage: Transport["onmessage"]; + readonly hasPerRequestStream?: boolean; + private readonly pendingTools = new Map(); + private readonly pendingToolLists = new Set(); + private readonly pendingCompletions = new Map(); + + private get recordEvidence(): EvidenceWriter["recordEvidence"] | undefined { + return this.currentEvidenceWriter(); + } + + private get delivery(): ToolResultDelivery { + return this.currentDelivery(); + } + + constructor( + private readonly transport: Transport, + private readonly currentEvidenceWriter: () => + | EvidenceWriter["recordEvidence"] + | undefined, + private readonly currentDelivery: () => ToolResultDelivery, + private readonly omitOutputSchemas: boolean, + ) { + if (transport.hasPerRequestStream !== undefined) + this.hasPerRequestStream = transport.hasPerRequestStream; + transport.onmessage = (message, extra) => { + if ( + isJSONRPCRequest(message) && + message.method === "tools/call" && + typeof message.params?.["name"] === "string" + ) + this.pendingTools.set(message.id, message.params["name"]); + if (isJSONRPCRequest(message) && message.method === "tools/list") + this.pendingToolLists.add(message.id); + if (isJSONRPCRequest(message) && message.method === "completion/complete") + this.pendingCompletions.set(message.id, new AbortController()); + // The SDK suppresses replies after cancellation, so send() cannot release + // this bookkeeping. Keep transport authentication outside retained data. + if ( + isJSONRPCNotification(message) && + message.method === "notifications/cancelled" + ) { + const id = message.params?.["requestId"]; + if (typeof id === "string" || typeof id === "number") + forgetPendingMcpRequest( + this.pendingTools, + this.pendingToolLists, + this.pendingCompletions, + id, + ); + } + const completion = + isJSONRPCRequest(message) && message.method === "completion/complete" + ? this.pendingCompletions.get(message.id) + : undefined; + if (completion === undefined) this.onmessage?.(message, extra); + else + currentRequestSignal.run(completion.signal, () => + this.onmessage?.(message, extra), + ); + }; + transport.onerror = (error) => this.onerror?.(error); + transport.onclose = () => { + this.pendingTools.clear(); + this.pendingToolLists.clear(); + for (const completion of this.pendingCompletions.values()) + completion.abort(); + this.pendingCompletions.clear(); + this.onclose?.(); + }; + } + + get sessionId(): string | undefined { + return this.transport.sessionId; + } + set sessionId(value: string | undefined) { + this.transport.sessionId = value; + } + + start(): Promise { + return this.transport.start(); + } + + async close(): Promise { + try { + await this.transport.close(); + } finally { + this.pendingTools.clear(); + this.pendingToolLists.clear(); + for (const completion of this.pendingCompletions.values()) + completion.abort(); + this.pendingCompletions.clear(); + } + } + + setProtocolVersion(version: string): void { + this.transport.setProtocolVersion?.(version); + } + + setSupportedProtocolVersions(versions: string[]): void { + this.transport.setSupportedProtocolVersions?.(versions); + } + + async send( + message: JSONRPCMessage, + options?: TransportSendOptions, + ): Promise { + const id = + "id" in message && + (typeof message.id === "string" || typeof message.id === "number") + ? message.id + : undefined; + const tool = id === undefined ? undefined : this.pendingTools.get(id); + const omitToolSchemas = + this.omitOutputSchemas && + id !== undefined && + this.pendingToolLists.has(id); + try { + const recovered = this.recoverProtocolError(message, tool); + const advertised = omitToolSchemas + ? omitOutputSchemasFromToolsList(recovered, id) + : recovered; + if ( + isJSONRPCResultResponse(advertised) && + isCallToolResult(advertised.result) && + advertised.result.isError === true + ) { + // Keep the complete typed JSON in content. Error projections are not + // success outputSchema data; oversized recovery must consume their + // private structured carrier before it is removed from the wire. + const { structuredContent: _diagnostic, ...result } = advertised.result; + // Errors the SDK builds itself (input validation, a thrown handler) + // bypass ToolResultDelivery; mark them for target-content tools too. + const marked = + tool !== undefined && + toolReturnsTargetContent(tool) && + !carriesTargetContentMarker(result) + ? { + ...result, + _meta: { ...result._meta, ...TARGET_CONTENT_RESULT_META }, + } + : result; + await this.transport.send({ ...advertised, result: marked }, options); + } else await this.transport.send(advertised, options); + } finally { + if (!isJSONRPCRequest(message) && id !== undefined) { + forgetPendingMcpRequest( + this.pendingTools, + this.pendingToolLists, + this.pendingCompletions, + id, + ); + } + } + } + + private recoverError( + message: JSONRPCMessage, + tool: string | undefined, + ): JSONRPCMessage { + if ( + !isJSONRPCResultResponse(message) || + !isCallToolResult(message.result) || + message.result.isError !== true + ) + return message; + const budget = this.delivery.resultBudgetBytes; + // Count the provenance marker exactly as the projected error did, and keep + // it on the replacement: the retained original may hold target content. + const targetContent = + carriesTargetContentMarker(message.result) || + (tool !== undefined && toolReturnsTargetContent(tool)); + // SDK validation failures have no structured carrier. Size their existing + // text-only result without serializing the complete JSON-RPC message. + const isStructured = message.result.structuredContent !== undefined; + const structured = isStructured + ? jsonObjectSchema.parse(message.result.structuredContent) + : undefined; + const sdkResultWithMarker = + !isStructured && + targetContent && + !carriesTargetContentMarker(message.result) + ? { + ...message.result, + _meta: { + ...message.result._meta, + ...TARGET_CONTENT_RESULT_META, + }, + } + : message.result; + const sdkDiagnostic = !isStructured + ? message.result.content.find((item) => item.type === "text")?.text + : undefined; + let encoded = + structured !== undefined + ? encodeToolResult(structured, budget, "error-text", targetContent) + : measureMcpResultWithinBudget(sdkResultWithMarker, budget); + const normalized = encoded.ok + ? this.normalizeSdkValidationError( + message, + tool, + targetContent, + sdkInputValidationDiagnostic(sdkDiagnostic, tool), + ) + : undefined; + if ( + normalized !== undefined && + isJSONRPCResultResponse(normalized) && + isCallToolResult(normalized.result) + ) { + // The structured diagnostic is a private carrier removed before sending. + // Measure the actual wire result, including its preserved SDK metadata. + const { structuredContent, ...wireResult } = normalized.result; + encoded = + normalized.result.content.length === 0 + ? encodeToolResult( + jsonObjectSchema.parse(structuredContent), + budget, + "error-text", + targetContent, + ) + : measureMcpResultWithinBudget(wireResult, budget); + if (encoded.ok) return normalized; + // JSON escaping and recovery fields can exceed the budget even when + // the original SDK result fits. Retain that original before replacing it. + } + if (encoded.ok) return message; + const operation = tool ?? "mcp_tool_error"; + const originalResult = jsonObjectSchema.parse(message.result); + const retainedResult = structured ?? originalResult; + const originalError = isStructured + ? analysisErrorProjectionSchema.safeParse(structured?.["error"]) + : undefined; + // Retain the complete original response, including SDK validation text. + // The SDK error contains its rendered issue, not a separate argument bag; + // transport authentication and caller arguments are never added here. + const evidence = createEvidence( + undefined, + { id: "rea-mcp", name: "REA MCP delivery", version: "1" }, + { + operation: "mcp_tool_error", + parameters: { tool_name: operation }, + result: + sdkDiagnostic === undefined + ? retainedResult + : { + boundary: "sdk-input-validation", + diagnostic_preview: sdkDiagnostic.slice(0, 256), + diagnostic_truncated: sdkDiagnostic.length > 256, + }, + rawResult: originalResult, + }, + ); + const recorded = this.recordEvidence?.(evidence); + const retained = recorded !== undefined && recorded.ok; + const projected = mcpResponseOverflowProjection({ + completed: false, + constraint: encoded.constraint, + effectiveResponseBytes: this.delivery.maximumResponseBytes, + nextStep: retained + ? "Export the retained original error through export_evidence_bundle to a caller-selected path. The connection remains usable." + : encoded.constraint === "string-length" + ? "The original error could not be retained by this server. Use complete CLI JSON output when the analysis completed. Repeating the unchanged MCP call fails the same way." + : "The original error could not be retained by this server. Restart REA with a larger REA_MCP_MAX_RESPONSE_BYTES and a matching client receive buffer, then retry.", + }); + const replacement = this.transportConstraintResult( + operation, + projected.reason, + projected.remediationAction, + { + boundary: "mcp-response", + ...(originalError?.success + ? { original_error_code: originalError.data.code } + : {}), + result_budget_bytes: budget, + response_bytes_at_least: encoded.bytesAtLeast, + constraint: encoded.constraint, + ...(sdkDiagnostic === undefined + ? {} + : { + diagnostic_preview: sdkDiagnostic.slice(0, 256), + diagnostic_truncated: sdkDiagnostic.length > 256, + }), + ...(retained + ? { + evidence_reference: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + } + : { retention: recorded === undefined ? "unavailable" : "failed" }), + ...(recorded !== undefined && !recorded.ok + ? { retention_error: projectAnalysisError(recorded.error) } + : {}), + }, + targetContent, + ); + return { + ...message, + result: replaceToolResultPayload(message.result, replacement), + }; + } + + /** + * Encode a transport-constraint error inside the MCP text budget. + * The complete SDK diagnostic stays in retained Evidence. Its character + * preview can still expand past the budget when quotes and backslashes are + * escaped into that text, so shorten only the preview and keep the reference. + */ + private transportConstraintResult( + operation: string, + reason: string, + remediationAction: string, + limits: Readonly>, + targetContent: boolean, + ): CallToolResult { + const deliver = ( + reportedLimits: Readonly>, + ): CallToolResult => + this.delivery.toErrorToolResult( + new AnalysisResourceConstraintError( + operation, + "transport", + reason, + reportedLimits, + { remediationAction }, + ), + targetContent, + ); + const delivered = deliver(limits); + const preview = limits.diagnostic_preview; + if (delivered.content.length > 0 || typeof preview !== "string") + return delivered; + let low = 0; + let high = preview.length - 1; + let fitted: CallToolResult | undefined; + let fittedLength = -1; + while (low <= high) { + const length = Math.floor((low + high) / 2); + const candidate = deliver({ + ...limits, + diagnostic_preview: preview.slice(0, length), + diagnostic_truncated: true, + }); + if (candidate.content.length > 0) { + fitted = candidate; + fittedLength = length; + low = length + 1; + } else high = length - 1; + } + if (fitted !== undefined && fittedLength > 0) return fitted; + const { diagnostic_preview: _preview, ...withoutPreview } = limits; + const omitted = deliver({ + ...withoutPreview, + diagnostic_truncated: preview.length > 0, + }); + return omitted.content.length > 0 || fitted === undefined + ? omitted + : fitted; + } + + private normalizeSdkValidationError( + message: JSONRPCMessage, + tool: string | undefined, + targetContent: boolean, + diagnostic: string | undefined, + ): JSONRPCMessage | undefined { + if ( + tool === undefined || + diagnostic === undefined || + !isJSONRPCResultResponse(message) || + !isCallToolResult(message.result) + ) + return undefined; + const error = new AnalysisInputError( + tool, + { + userMessage: `The arguments for MCP tool ${tool} do not match its advertised input schema.`, + }, + [{ path: [], reason: "invalid_value", message: diagnostic }], + ); + const result = this.delivery.toErrorToolResult(error, targetContent); + return { + ...message, + result: replaceToolResultPayload(message.result, result), + }; + } + + private recoverProtocolError( + message: JSONRPCMessage, + tool: string | undefined, + ): JSONRPCMessage { + const recovered = this.recoverError(message, tool); + if ( + !isJSONRPCErrorResponse(recovered) || + tool === undefined || + recovered.error.code !== -32602 || + recovered.error.message !== `Tool ${tool} not found` + ) + return recovered; + return { + ...recovered, + error: { + ...recovered.error, + message: + "The requested MCP tool is not advertised by this REA server. Call tools/list, select an advertised tool, and retry; do not repeat the unknown tool name.", + }, + }; + } +} diff --git a/src/server/ToolResultDelivery.ts b/src/server/ToolResultDelivery.ts new file mode 100644 index 0000000000..ecdcb87b98 --- /dev/null +++ b/src/server/ToolResultDelivery.ts @@ -0,0 +1,250 @@ +import type { CallToolResult } from "@modelcontextprotocol/server"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; + +import type { ToolContract } from "../contracts/toolContractTypes.js"; +import type { Evidence } from "../domain/evidence.js"; +import { + projectAnalysisError, + type AnalysisErrorProjection, +} from "../domain/analysisErrorProjection.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import type { Result } from "../domain/result.js"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { + MCP_TRANSPORT_EVIDENCE_DISCOVERY, + MCP_TRANSPORT_EVIDENCE_EXPORT, +} from "../domain/analysisErrorPresentation.js"; +import { + nodeStringCeilingRecovery, + stdioResponseBudgetRecovery, +} from "../config/mcpResponseBudget.js"; +import { + encodeToolResult, + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + TARGET_CONTENT_RESULT_META, +} from "./toolResultEncoding.js"; + +/** Project a response-budget refusal without labeling REA's reserved string limit as Node's ceiling. */ +export const mcpResponseOverflowProjection = (input: { + readonly completed: boolean; + readonly constraint: "receive-buffer" | "string-length"; + readonly effectiveResponseBytes: number; + readonly nextStep: string; +}): { readonly reason: string; readonly remediationAction: string } => { + const facts = + input.constraint === "string-length" + ? nodeStringCeilingRecovery( + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ) + : stdioResponseBudgetRecovery( + input.effectiveResponseBytes, + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ); + const summary = + input.constraint === "string-length" + ? input.completed + ? "The operation completed, but its complete MCP response exceeds REA's reserved single-string limit." + : "The tool failed, but its complete error response exceeds REA's reserved single-string limit." + : input.completed + ? "The operation completed, but its complete MCP response exceeds the stdio response budget. The analysis result was not truncated." + : "The tool failed, but its complete error response exceeds the MCP response budget."; + return { + reason: `${summary} ${facts} ${input.nextStep}`, + remediationAction: `${input.nextStep} ${facts}`, + }; +}; + +/** + * A summary projection has already used `detail: "summary"`. + * Export still writes the retained bundle outside one MCP frame. + */ +const evidenceBundleSummary = (value: JsonValue): boolean => { + const payload = + typeof value === "object" && + value !== null && + !Array.isArray(value) && + "result" in value + ? value.result + : value; + return ( + typeof payload === "object" && + payload !== null && + !Array.isArray(payload) && + payload.kind === "evidence-bundle-summary" + ); +}; + +/** Immutable encoding and recovery policy shared by one MCP server. */ +export class ToolResultDelivery { + readonly maximumResponseBytes: number; + readonly resultBudgetBytes: number; + constructor( + maximumResponseBytes: number, + private readonly nativeContinuation: ( + evidence: Evidence, + ) => Readonly> = () => ({}), + ) { + this.maximumResponseBytes = maximumResponseBytes; + this.resultBudgetBytes = maximumResponseBytes - 1024; + Object.freeze(this); + } + + /** Serialize a plain application result or its actionable error. */ + toCallToolResult( + result: Result, + contract: ToolContract, + ): CallToolResult { + return result.ok + ? this.successResult( + contract.kind === "session" ? { result: result.value } : result.value, + contract, + ) + : this.toErrorToolResult(result.error, contract.returnsTargetContent); + } + + /** Deliver Evidence using the producer's explicit recording acknowledgment. */ + toEvidenceToolResult( + evidence: Evidence, + contract: ToolContract, + recorded: Result | undefined, + ): CallToolResult { + return recorded !== undefined && !recorded.ok + ? this.toErrorToolResult(recorded.error, contract.returnsTargetContent) + : this.successResult(evidence, contract, { + evidence_id: evidence.evidence_id, + ...(recorded === undefined + ? {} + : { + evidence_reference: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + ...this.nativeContinuation(evidence), + }), + }); + } + + /** + * Project an error without allocating oversized repeated MCP text. + * `targetContent` is the failing tool's `returnsTargetContent`: a failure + * can carry target-derived text (a message, captured output, or a partial + * observation), so it is marked exactly like that tool's successful results. + */ + toErrorToolResult( + error: AnalysisError, + targetContent: boolean, + ): CallToolResult { + return this.projectedErrorResult( + projectAnalysisError(error), + targetContent, + ); + } + + /** Deliver the original failure with the producer-acknowledged partial Evidence reference. */ + toRecordedPartialErrorToolResult( + error: AnalysisError, + evidenceId: string, + recorded: Result, + targetContent: boolean, + ): CallToolResult { + if (!recorded.ok) return this.toErrorToolResult(error, targetContent); + const projected = projectAnalysisError(error); + return this.projectedErrorResult( + { + ...projected, + details: { + ...projected.details, + partial_observation: { + kind: "retained-evidence", + evidence_id: evidenceId, + }, + }, + }, + targetContent, + ); + } + + private projectedErrorResult( + projected: AnalysisErrorProjection, + targetContent: boolean, + ): CallToolResult { + const structuredContent = { error: projected }; + // Errors reach the wire as text only; the structured copy is a private + // carrier the transport removes after its own budget check. + const encoded = encodeToolResult( + structuredContent, + this.resultBudgetBytes, + "error-text", + targetContent, + ); + // The transport retains an oversized projected error before replacing it + // with a recoverable delivery constraint, using this same budget. + return { + content: encoded.ok ? [{ type: "text", text: encoded.text }] : [], + structuredContent, + isError: true, + ...(targetContent ? { _meta: TARGET_CONTENT_RESULT_META } : {}), + }; + } + + private successResult( + value: JsonValue, + contract: ToolContract, + recovery: Readonly> = {}, + ): CallToolResult { + const encoded = encodeToolResult( + value, + this.resultBudgetBytes, + "structured-and-text", + contract.returnsTargetContent, + ); + if (encoded.ok) + return { + content: [{ type: "text", text: encoded.text }], + structuredContent: value, + ...(contract.returnsTargetContent + ? { _meta: TARGET_CONTENT_RESULT_META } + : {}), + }; + const retainedEvidence = recovery.evidence_reference !== undefined; + const nextStep = + recovery.continuation !== undefined + ? "Call inspect_analysis_view with the supplied continuation arguments to read retained analysis. Follow coverage.next_offset for subsequent pages; reduce the page limit if needed, or export the complete retained Evidence." + : retainedEvidence + ? "Export retained evidence to consume the complete analysis, or use complete CLI JSON output." + : contract.name === "get_evidence_bundle" + ? evidenceBundleSummary(value) + ? MCP_TRANSPORT_EVIDENCE_EXPORT + : MCP_TRANSPORT_EVIDENCE_DISCOVERY + : "Use complete CLI JSON output to consume the analysis."; + const projected = mcpResponseOverflowProjection({ + completed: true, + constraint: encoded.constraint, + effectiveResponseBytes: this.maximumResponseBytes, + nextStep, + }); + return this.toErrorToolResult( + new AnalysisResourceConstraintError( + contract.name, + "transport", + projected.reason, + { + boundary: "mcp-response", + default_receive_buffer_bytes: STDIO_DEFAULT_MAX_BUFFER_SIZE, + result_budget_bytes: this.resultBudgetBytes, + max_string_code_units: MCP_RESULT_STRING_LIMIT, + response_bytes_at_least: encoded.bytesAtLeast, + response_code_units_at_least: encoded.codeUnitsAtLeast, + constraint: encoded.constraint, + ...recovery, + }, + { remediationAction: projected.remediationAction }, + ), + contract.returnsTargetContent, + ); + } +} diff --git a/src/server/createServer.ts b/src/server/createServer.ts index fa98728744..51d85aeceb 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -48,6 +48,7 @@ import { FridaInstrumentationManager } from "../frida/FridaInstrumentationManage import { registerFridaTools } from "./registerFridaTools.js"; import { createBinaryLayoutService } from "../composition/binaryDiagnostics.js"; import type { BinaryLayoutService } from "../application/binaryDiagnostics/BinaryLayoutService.js"; +import { createCatalogIdentity } from "../catalogIdentity.js"; import { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import { createToolResultDelivery, @@ -161,6 +162,10 @@ const RECOVERY_INSTRUCTIONS = "Recovery: For invalid arguments, read the connected tool's input schema (refresh tools/list if stale), fix the reported fields and do not repeat the unchanged call. A returned REA provider_timeout is a server/provider deadline: follow its remediation and reuse accessible partial Evidence. A host error such as 'timed out awaiting tools/call' without a REA result is a client deadline, not proof that work or cleanup stopped. Do not repeat the unchanged analysis. Increase the host's tool deadline with execution/cleanup/delivery headroom; when configuration is not editable, report the required setting (Codex: mcp_servers..tool_timeout_sec, seconds). Server analysis_timeout_ms, when advertised, is a separate millisecond budget; raising it cannot fix a shorter client deadline. Session state does not establish completion of every target-free operation."; export interface CreateServerOptions { + /** Preserve the connection's creation time when materializing its runtime. */ + readonly startedAt?: string; + /** Existing SDK server whose deferred registrations are being materialized. */ + readonly mcpServer?: EvidenceMcpServer; readonly environment?: Readonly; readonly delivery?: ToolResultDelivery; readonly packetCapture?: PacketCaptureService; @@ -298,6 +303,10 @@ const createMcpServer = ( delivery, compactPresentation, omitOutputSchemas, + createCatalogIdentity( + compactPresentation, + omitOutputSchemas ? "off" : "full", + ), ); /** @@ -361,14 +370,28 @@ export const createServer = ( webModuleTrace, javascriptRecovery, }; - const startedAt = new Date().toISOString(); + const startedAt = options.startedAt ?? new Date().toISOString(); const logger = options.logger ?? silentLogger; - const server = createMcpServer( - session, - delivery, - selectCompactInputSchemaPresentation(environment), - omitOutputSchemas, - ); + const server = + options.mcpServer ?? + createMcpServer( + session, + delivery, + selectCompactInputSchemaPresentation(environment), + omitOutputSchemas, + ); + if (options.mcpServer !== undefined) { + server.attachRuntime( + session === undefined + ? undefined + : (evidence) => session.recordEvidence(evidence), + delivery, + createCatalogIdentity( + selectCompactInputSchemaPresentation(environment), + omitOutputSchemas ? "off" : "full", + ), + ); + } const gdbSessions = new GdbSessionManager({ environment: providerEnvironment, }); diff --git a/src/server/promptArguments.ts b/src/server/promptArguments.ts new file mode 100644 index 0000000000..f2b132f503 --- /dev/null +++ b/src/server/promptArguments.ts @@ -0,0 +1,36 @@ +import { completable } from "@modelcontextprotocol/server"; +import { z } from "zod"; +import type { + PromptArgumentContract, + PromptContract, +} from "../contracts/promptContracts.js"; +import type { PromptCompletionSource } from "./promptCompletion.js"; + +export const promptArgumentsSchema = ( + contract: Pick, + completion: PromptCompletionSource, +) => { + const shape: Record = {}; + for (const [name, argument] of Object.entries(contract.arguments)) + shape[name] = argumentSchema(argument, completion); + return z.object(shape); +}; + +const argumentSchema = ( + argument: PromptArgumentContract, + completion: PromptCompletionSource, +): z.ZodType => { + const value = z + .string() + .min(1) + .refine((value) => value.trim().length > 0) + .describe(argument.description); + const kind = argument.completion; + const completed = + kind === undefined + ? value + : completable(value, async (partial, context) => [ + ...(await completion.complete(kind, partial, context)), + ]); + return argument.required ? completed : completed.optional(); +}; diff --git a/src/server/promptCompletion.ts b/src/server/promptCompletion.ts index 59fa48e6ce..b10bae84c1 100644 --- a/src/server/promptCompletion.ts +++ b/src/server/promptCompletion.ts @@ -8,7 +8,7 @@ import { parseProcessCaptureEvidence } from "../application/process/ProcessEvide import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; import type { PromptCompletionKind } from "../contracts/promptContracts.js"; import { addressedValueSchema } from "../domain/native/nativeInventory.js"; -import { currentMcpRequestSignal } from "./EvidenceMcpServer.js"; +import { currentMcpRequestSignal } from "./EvidenceMcpServerCore.js"; import { parseArtifactInventoryEvidence } from "../domain/artifactInventoryEvidence.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; diff --git a/src/server/registerPrompts.ts b/src/server/registerPrompts.ts index a20713452b..bac78c0c3f 100644 --- a/src/server/registerPrompts.ts +++ b/src/server/registerPrompts.ts @@ -1,11 +1,7 @@ -import { - currentMcpRequestSignal, - type EvidenceMcpServer, -} from "./EvidenceMcpServer.js"; -import { - completable, - type RegisteredPrompt, -} from "@modelcontextprotocol/server"; +import { promptArgumentsSchema } from "./promptArguments.js"; +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import { currentMcpRequestSignal } from "./EvidenceMcpServerCore.js"; +import type { RegisteredPrompt } from "@modelcontextprotocol/server"; import { z } from "zod"; import type { AnalysisOperationPort } from "../application/AnalysisProvider.js"; @@ -13,7 +9,6 @@ import type { BinarySessionPort } from "../application/binary/BinarySessionPort. import { PROMPT_CONTRACTS, renderGuidedPrompt, - type PromptArgumentContract, type PromptContract, } from "../contracts/promptContracts.js"; import { @@ -35,7 +30,7 @@ export const registerGuidedPrompts = ( session?: BinarySessionPort, ): GuidedPromptRegistry => { const source = createPromptCompletionSource(analysis, session); - const completion: PromptCompletionSource = { + const completion = server.bindPromptCompletion({ async complete(kind, value, context) { // Completions can query the same providers as tools. Cancellation alone // does not mean their provider cleanup has finished. @@ -43,7 +38,7 @@ export const registerGuidedPrompts = ( source.complete(kind, value, context), ); }, - }; + }); const prompts = new Map(); for (const contract of PROMPT_CONTRACTS) prompts.set(contract.name, registerPrompt(server, contract, completion)); @@ -69,7 +64,7 @@ const registerPrompt = ( completion: PromptCompletionSource, ): RegisteredPrompt => { const argsSchema = promptArgumentsSchema(contract, completion); - return server.registerPrompt( + return server.registerCanonicalPrompt( contract.name, { title: contract.title, @@ -80,35 +75,6 @@ const registerPrompt = ( ); }; -const promptArgumentsSchema = ( - contract: PromptContract, - completion: PromptCompletionSource, -) => { - const shape: Record = {}; - for (const [name, argument] of Object.entries(contract.arguments)) - shape[name] = argumentSchema(argument, completion); - return z.object(shape); -}; - -const argumentSchema = ( - argument: PromptArgumentContract, - completion: PromptCompletionSource, -): z.ZodType => { - const value = z - .string() - .min(1) - .refine((value) => value.trim().length > 0) - .describe(argument.description); - const kind = argument.completion; - const completed = - kind === undefined - ? value - : completable(value, async (partial, context) => [ - ...(await completion.complete(kind, partial, context)), - ]); - return argument.required ? completed : completed.optional(); -}; - const promptResult = (contract: PromptContract, arguments_: unknown) => { const values = promptValuesSchema.parse(arguments_); for (const [name, argument] of Object.entries(contract.arguments)) diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 64324abeae..3152d5b454 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -1,65 +1,17 @@ import { isNativeInventoryOperation } from "../domain/native/nativeInventory.js"; -import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../domain/analysisView/analysisViewErrors.js"; -import type { CallToolResult } from "@modelcontextprotocol/server"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; - -import type { ToolContract } from "../contracts/toolContractTypes.js"; -import type { Evidence } from "../domain/evidence.js"; +import { ToolResultDelivery as Delivery } from "./ToolResultDelivery.js"; +import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../domain/analysisView/analysisViewErrors.js"; import { inspectAnalysisViewInputSchema } from "../domain/analysisView/analysisView.js"; -import { - projectAnalysisError, - type AnalysisErrorProjection, -} from "../domain/analysisErrorProjection.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import type { Evidence } from "../domain/evidence.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; -import type { Result } from "../domain/result.js"; -import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; -import { - MCP_TRANSPORT_EVIDENCE_DISCOVERY, - MCP_TRANSPORT_EVIDENCE_EXPORT, -} from "../domain/analysisErrorPresentation.js"; -import { - nodeStringCeilingRecovery, - stdioResponseBudgetRecovery, -} from "../config/mcpResponseBudget.js"; -import { - encodeToolResult, - MCP_RESULT_STRING_LIMIT, - NODE_MAX_STRING_LENGTH, - TARGET_CONTENT_RESULT_META, -} from "./toolResultEncoding.js"; -/** Project a response-budget refusal without labeling REA's reserved string limit as Node's ceiling. */ -export const mcpResponseOverflowProjection = (input: { - readonly completed: boolean; - readonly constraint: "receive-buffer" | "string-length"; - readonly effectiveResponseBytes: number; - readonly nextStep: string; -}): { readonly reason: string; readonly remediationAction: string } => { - const facts = - input.constraint === "string-length" - ? nodeStringCeilingRecovery( - MCP_RESULT_STRING_LIMIT, - NODE_MAX_STRING_LENGTH, - ) - : stdioResponseBudgetRecovery( - input.effectiveResponseBytes, - MCP_RESULT_STRING_LIMIT, - NODE_MAX_STRING_LENGTH, - ); - const summary = - input.constraint === "string-length" - ? input.completed - ? "The operation completed, but its complete MCP response exceeds REA's reserved single-string limit." - : "The tool failed, but its complete error response exceeds REA's reserved single-string limit." - : input.completed - ? "The operation completed, but its complete MCP response exceeds the stdio response budget. The analysis result was not truncated." - : "The tool failed, but its complete error response exceeds the MCP response budget."; - return { - reason: `${summary} ${facts} ${input.nextStep}`, - remediationAction: `${input.nextStep} ${facts}`, - }; -}; +/** Full delivery policy, including canonical native Evidence continuation. */ +export class ToolResultDelivery extends Delivery { + constructor(maximumResponseBytes: number) { + super(maximumResponseBytes, retainedEvidenceContinuation); + } +} /** Create one delivery policy from the already selected, parsed response budget. */ export const createToolResultDelivery = ( @@ -91,188 +43,4 @@ const retainedEvidenceContinuation = ( }; }; -/** - * A summary projection has already used `detail: "summary"`. - * Export still writes the retained bundle outside one MCP frame. - */ -const evidenceBundleSummary = (value: JsonValue): boolean => { - const payload = - typeof value === "object" && - value !== null && - !Array.isArray(value) && - "result" in value - ? value.result - : value; - return ( - typeof payload === "object" && - payload !== null && - !Array.isArray(payload) && - payload.kind === "evidence-bundle-summary" - ); -}; - -/** Immutable encoding and recovery policy shared by one MCP server. */ -export class ToolResultDelivery { - readonly maximumResponseBytes: number; - readonly resultBudgetBytes: number; - constructor(maximumResponseBytes: number) { - this.maximumResponseBytes = maximumResponseBytes; - this.resultBudgetBytes = maximumResponseBytes - 1024; - Object.freeze(this); - } - - /** Serialize a plain application result or its actionable error. */ - toCallToolResult( - result: Result, - contract: ToolContract, - ): CallToolResult { - return result.ok - ? this.successResult( - contract.kind === "session" ? { result: result.value } : result.value, - contract, - ) - : this.toErrorToolResult(result.error, contract.returnsTargetContent); - } - - /** Deliver Evidence using the producer's explicit recording acknowledgment. */ - toEvidenceToolResult( - evidence: Evidence, - contract: ToolContract, - recorded: Result | undefined, - ): CallToolResult { - return recorded !== undefined && !recorded.ok - ? this.toErrorToolResult(recorded.error, contract.returnsTargetContent) - : this.successResult(evidence, contract, { - evidence_id: evidence.evidence_id, - ...(recorded === undefined - ? {} - : { - evidence_reference: { - kind: "retained-evidence", - evidence_id: evidence.evidence_id, - }, - ...retainedEvidenceContinuation(evidence), - }), - }); - } - - /** - * Project an error without allocating oversized repeated MCP text. - * `targetContent` is the failing tool's `returnsTargetContent`: a failure - * can carry target-derived text (a message, captured output, or a partial - * observation), so it is marked exactly like that tool's successful results. - */ - toErrorToolResult( - error: AnalysisError, - targetContent: boolean, - ): CallToolResult { - return this.projectedErrorResult( - projectAnalysisError(error), - targetContent, - ); - } - - /** Deliver the original failure with the producer-acknowledged partial Evidence reference. */ - toRecordedPartialErrorToolResult( - error: AnalysisError, - evidenceId: string, - recorded: Result, - targetContent: boolean, - ): CallToolResult { - if (!recorded.ok) return this.toErrorToolResult(error, targetContent); - const projected = projectAnalysisError(error); - return this.projectedErrorResult( - { - ...projected, - details: { - ...projected.details, - partial_observation: { - kind: "retained-evidence", - evidence_id: evidenceId, - }, - }, - }, - targetContent, - ); - } - - private projectedErrorResult( - projected: AnalysisErrorProjection, - targetContent: boolean, - ): CallToolResult { - const structuredContent = { error: projected }; - // Errors reach the wire as text only; the structured copy is a private - // carrier the transport removes after its own budget check. - const encoded = encodeToolResult( - structuredContent, - this.resultBudgetBytes, - "error-text", - targetContent, - ); - // The transport retains an oversized projected error before replacing it - // with a recoverable delivery constraint, using this same budget. - return { - content: encoded.ok ? [{ type: "text", text: encoded.text }] : [], - structuredContent, - isError: true, - ...(targetContent ? { _meta: TARGET_CONTENT_RESULT_META } : {}), - }; - } - - private successResult( - value: JsonValue, - contract: ToolContract, - recovery: Readonly> = {}, - ): CallToolResult { - const encoded = encodeToolResult( - value, - this.resultBudgetBytes, - "structured-and-text", - contract.returnsTargetContent, - ); - if (encoded.ok) - return { - content: [{ type: "text", text: encoded.text }], - structuredContent: value, - ...(contract.returnsTargetContent - ? { _meta: TARGET_CONTENT_RESULT_META } - : {}), - }; - const retainedEvidence = recovery.evidence_reference !== undefined; - const nextStep = - recovery.continuation !== undefined - ? "Call inspect_analysis_view with the supplied continuation arguments to read retained analysis. Follow coverage.next_offset for subsequent pages; reduce the page limit if needed, or export the complete retained Evidence." - : retainedEvidence - ? "Export retained evidence to consume the complete analysis, or use complete CLI JSON output." - : contract.name === "get_evidence_bundle" - ? evidenceBundleSummary(value) - ? MCP_TRANSPORT_EVIDENCE_EXPORT - : MCP_TRANSPORT_EVIDENCE_DISCOVERY - : "Use complete CLI JSON output to consume the analysis."; - const projected = mcpResponseOverflowProjection({ - completed: true, - constraint: encoded.constraint, - effectiveResponseBytes: this.maximumResponseBytes, - nextStep, - }); - return this.toErrorToolResult( - new AnalysisResourceConstraintError( - contract.name, - "transport", - projected.reason, - { - boundary: "mcp-response", - default_receive_buffer_bytes: STDIO_DEFAULT_MAX_BUFFER_SIZE, - result_budget_bytes: this.resultBudgetBytes, - max_string_code_units: MCP_RESULT_STRING_LIMIT, - response_bytes_at_least: encoded.bytesAtLeast, - response_code_units_at_least: encoded.codeUnitsAtLeast, - constraint: encoded.constraint, - ...recovery, - }, - { remediationAction: projected.remediationAction }, - ), - contract.returnsTargetContent, - ); - } -} +export { mcpResponseOverflowProjection } from "./ToolResultDelivery.js"; diff --git a/tests/boundary/mcp/deferredRuntime.test.ts b/tests/boundary/mcp/deferredRuntime.test.ts new file mode 100644 index 0000000000..383d2e047f --- /dev/null +++ b/tests/boundary/mcp/deferredRuntime.test.ts @@ -0,0 +1,798 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { spawnSync } from "node:child_process"; +import { + cp, + mkdir, + readFile, + readdir, + symlink, + writeFile, +} from "node:fs/promises"; +import { resolve, join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { GENERATED_MCP_TOOL_CATALOG } from "../../fixtures/mcpToolCatalog.js"; +import { PROMPT_CONTRACTS } from "../../../src/contracts/promptContracts.js"; +import { evidenceSchema } from "../../../src/domain/evidence.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { SERVER_START_FAILED } from "../../../src/main/messages.js"; + +const connect = async ( + mode: "legacy" | "auto", + environment: Record = {}, + nodeArguments: readonly string[] = [], +) => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [...nodeArguments, resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { + PATH: process.env.PATH ?? "", + HOME: process.env.HOME ?? "", + REA_LOG_LEVEL: "silent", + ...environment, + }, + stderr: "pipe", + }); + let stderr = ""; + transport.stderr?.on("data", (value) => { + stderr += value; + }); + const client = new Client( + { name: "deferred-runtime-boundary", version: "1" }, + { versionNegotiation: { mode } }, + ); + onTestFinished(async () => { + await client.close(); + await transport.close(); + }); + await client.connect(transport); + return { client, stderr: () => stderr }; +}; + +it.each([ + { name: "deferred", environment: {}, eager: false }, + { + name: "compact deferred", + environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "compact" }, + eager: false, + }, + { + name: "full deferred", + environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, + eager: false, + }, + { + name: "output-off eager", + environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: "off" }, + eager: true, + }, +])( + "loads compiled analysis at the intended phase for $name startup", + async ({ environment, eager }) => { + const directory = await createTestTempDirectory("rea-mcp-module-loads-"); + const log = join(directory, "loaded.jsonl"); + await writeFile(log, ""); + const modules = [ + "main/eagerRuntime.js", + "server/createServer.js", + "composition/binary.js", + "contracts/toolContracts.js", + ]; + const hook = `data:text/javascript,${encodeURIComponent(` +import { registerHooks } from "node:module"; +import { appendFileSync } from "node:fs"; +const modules = ${JSON.stringify(modules)}; +registerHooks({ + load(url, context, next) { + const loaded = next(url, context); + const source = loaded.source?.toString() ?? ""; + const names = modules.filter((name) => + url.endsWith("/dist/" + name) || source.includes("//#region dist/" + name + "\\n")); + if (url.endsWith("/scripts/rea.mjs")) names.push("entrypoint"); + for (const name of names) + appendFileSync(${JSON.stringify(log)}, JSON.stringify(name) + "\\n"); + return loaded; + }, +}); +`)}`; + const loadedModules = async () => + new Set( + (await readFile(log, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map((line): unknown => JSON.parse(line)), + ); + const { client, stderr } = await connect("legacy", environment, [ + `--import=${hook}`, + ]); + await client.listTools(); + await client.listPrompts(); + expect(await loadedModules()).toEqual( + new Set(eager ? ["entrypoint", ...modules] : ["entrypoint"]), + ); + const result = await client.callTool({ + name: "list_unknowns", + arguments: {}, + }); + expect(result.isError).not.toBe(true); + expect(await loadedModules()).toEqual(new Set(["entrypoint", ...modules])); + expect(stderr()).toBe(""); + }, +); + +it.each([ + { mode: "legacy", profile: "full" }, + { mode: "auto", profile: "full" }, + { mode: "legacy", profile: "compact" }, + { mode: "auto", profile: "compact" }, +] as const)( + "serves $profile discovery and canonical handlers over $mode negotiation", + async ({ mode, profile }) => { + const recorded = JSON.parse( + await readFile( + profile === "full" + ? "dist/mcp-catalog-full.json" + : "dist/mcp-catalog.json", + "utf8", + ), + ); + const { client, stderr } = await connect(mode, { + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + }); + const before = await client.listTools(); + expect(before.tools).toEqual(recorded.tools); + const contractProjection = (tool: { + name: string; + title?: string | undefined; + description?: string | undefined; + inputSchema: unknown; + outputSchema?: unknown; + annotations?: unknown; + }) => ({ + name: tool.name, + title: tool.title, + description: tool.description, + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + annotations: tool.annotations, + }); + if (profile === "full") + expect( + new Map( + before.tools.map((tool) => [tool.name, contractProjection(tool)]), + ), + ).toEqual( + new Map( + GENERATED_MCP_TOOL_CATALOG.map((tool) => [ + tool.name, + contractProjection(tool), + ]), + ), + ); + expect((await client.listPrompts()).prompts).toEqual( + PROMPT_CONTRACTS.map((prompt) => ({ + name: prompt.name, + title: prompt.title, + description: prompt.description, + arguments: Object.entries(prompt.arguments).map(([name, argument]) => ({ + name, + description: argument.description, + required: argument.required, + })), + })), + ); + + expect((await client.listPrompts()).prompts).toEqual( + recorded.prompts.map( + ({ parameters: _parameters, ...prompt }: Record) => + prompt, + ), + ); + const results = await Promise.all([ + client.callTool({ name: "list_unknowns", arguments: {} }), + client.callTool({ name: "list_unknowns", arguments: { limit: 1 } }), + ]); + expect(results.every((result) => result.isError !== true)).toBe(true); + expect(results[0]?.structuredContent).toEqual({ + result: { items: [], total: 0, offset: 0, next_offset: null }, + }); + expect(await client.listTools()).toEqual(before); + const invalid = await client.callTool({ + name: "list_unknowns", + arguments: { limit: 1.5 }, + }); + expect(invalid.isError).toBe(true); + expect(invalid._meta?.provenance).toEqual({ + untrusted_content: true, + source: "analysis_target", + }); + expect(client.getProtocolEra()).toBe(mode === "auto" ? "modern" : "legacy"); + if (mode === "auto") + expect(invalid._meta?.["io.modelcontextprotocol/serverInfo"]).toEqual( + recorded.info, + ); + expect(stderr()).toBe(""); + }, +); + +it("preserves connection creation time across deferred materialization", async () => { + const directory = await createTestTempDirectory("rea-mcp-started-at-"); + const clock = join(directory, "clock"); + const startedAt = "2020-01-01T00:00:00.000Z"; + await writeFile(clock, startedAt); + const hook = `data:text/javascript,${encodeURIComponent(` +import { readFileSync } from "node:fs"; +const NativeDate = Date; +globalThis.Date = class extends NativeDate { + constructor(...args) { + super(...(args.length === 0 ? [readFileSync(${JSON.stringify(clock)}, "utf8")] : args)); + } +}; +`)}`; + const { client } = await connect("legacy", {}, [`--import=${hook}`]); + await client.listTools(); + await writeFile(clock, "2021-01-01T00:00:00.000Z"); + const first = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(first.isError).not.toBe(true); + expect(first.structuredContent).toMatchObject({ + result: { server_identity: { server: { started_at: startedAt } } }, + }); + const again = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(again.structuredContent).toMatchObject({ + result: { server_identity: { server: { started_at: startedAt } } }, + }); +}); + +it.each([ + { level: "info", asynchronous: false, errorObject: true }, + { level: "info", asynchronous: true, errorObject: false }, + { level: "silent", asynchronous: false, errorObject: true }, +])( + "logs deferred startup failure with $level logging and asynchronous=$asynchronous", + ({ level, asynchronous, errorObject }) => { + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` +import assert from "node:assert/strict"; +import { runDeferred } from "./dist/main/deferredRuntime.js"; +import { parseConfig } from "./dist/config/parseConfig.js"; +const config = parseConfig({ HOME: process.env.HOME, REA_LOG_LEVEL: ${JSON.stringify(level)} }); +assert.equal(config.ok, true); +const output = []; +let closeCalls = 0; +let unregistrations = 0; +const result = await runDeferred({ + env: {}, + writeStderr: (text) => output.push(text), + setExitCode: () => {}, + registerShutdown: () => () => { unregistrations++; }, + serve: (factory) => { + const server = factory({}); + const close = server.server.close.bind(server.server); + server.server.close = async () => { closeCalls++; await close(); }; + const cause = ${errorObject ? 'new Error("transport setup reason")' : '"transport setup reason"'}; + if (${asynchronous}) return Promise.reject(cause); + throw cause; + }, +}, config.value); +console.log(JSON.stringify({ result, output, closeCalls, unregistrations })); +`, + ], + { cwd: process.cwd(), encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(JSON.parse(child.stdout)).toEqual({ + result: 1, + output: [`${SERVER_START_FAILED}\n`], + closeCalls: 1, + unregistrations: 1, + }); + if (level === "silent") { + expect(child.stderr).toBe(""); + } else { + expect(JSON.parse(child.stderr)).toMatchObject({ + level: 50, + application: "rea", + mode: "mcp", + layer: "server", + error: "transport setup reason", + msg: SERVER_START_FAILED, + }); + } + }, +); + +it("retains process cleanup ownership after partial startup and failed cleanup", () => { + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` +import assert from "node:assert/strict"; +import { runDeferred } from "./dist/main/deferredRuntime.js"; +import { parseConfig } from "./dist/config/parseConfig.js"; +const config = parseConfig({ HOME: process.env.HOME, REA_LOG_LEVEL: "silent" }); +assert.equal(config.ok, true); +const handlers = []; +const calls = [0, 0]; +let unregistrations = 0; +let captured; +let registeredBeforeServe; +const result = await runDeferred({ + env: {}, + writeStderr: () => {}, + setExitCode: () => {}, + registerShutdown: (handler) => { + handlers.push(handler); + return () => { unregistrations++; }; + }, + captureShutdown: (close) => { captured = close; }, + serve: (factory) => { + registeredBeforeServe = handlers.length; + for (let index = 0; index < calls.length; index++) { + const server = factory({}); + const close = server.server.close.bind(server.server); + server.server.close = async () => { + calls[index]++; + if (index === 0 && calls[index] === 1) throw new Error("close denied"); + await close(); + }; + } + throw new Error("serve failed after creating servers"); + }, +}, config.value).then(() => "unexpected success", (cause) => cause); +assert.ok(result instanceof AggregateError); +assert.equal(result.errors[0].message, "serve failed after creating servers"); +assert.ok(result.errors[1] instanceof AggregateError); +assert.match(String(result.errors[1].errors[0].errors[0]), /close denied/); +assert.equal(registeredBeforeServe, 1); +assert.deepEqual(calls, [1, 1]); +assert.equal(unregistrations, 0); +assert.equal(typeof captured, "function"); +const retry = captured(); +handlers[0](); +await retry; +await new Promise((resolve) => setImmediate(resolve)); +assert.deepEqual(calls, [2, 1]); +assert.equal(unregistrations, 1); +await captured(); +assert.deepEqual(calls, [2, 1]); +assert.equal(unregistrations, 1); +console.log("STARTUP_CLEANUP_RETRIED"); +`, + ], + { cwd: process.cwd(), encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe("STARTUP_CLEANUP_RETRIED\n"); + expect(child.stderr).toBe(""); +}); + +it.each(["handle", "server"])( + "retries only the failed %s cleanup owner", + (failedOwner) => { + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` +import assert from "node:assert/strict"; +import { runDeferred } from "./dist/main/deferredRuntime.js"; +import { parseConfig } from "./dist/config/parseConfig.js"; +const config = parseConfig({ HOME: process.env.HOME }); +assert.equal(config.ok, true); +const failedOwner = ${JSON.stringify(failedOwner)}; +const calls = { handle: 0, server: 0 }; +let captured; +let unregistrations = 0; +const result = await runDeferred({ + env: {}, writeStderr: () => {}, setExitCode: () => {}, + registerShutdown: () => () => { unregistrations++; }, + captureShutdown: (close) => { captured = close; }, + serve: (factory) => { + const server = factory({}); + const close = server.server.close.bind(server.server); + server.server.close = async () => { + calls.server++; + if (failedOwner === "server" && calls.server === 1) throw new Error("server denied"); + await close(); + }; + return { close: () => { + calls.handle++; + if (failedOwner === "handle" && calls.handle === 1) throw new Error("handle denied"); + // Deliberately not idempotent: a successful handle must never be retried. + assert.ok(calls.handle <= (failedOwner === "handle" ? 2 : 1)); + return Promise.resolve(); + } }; + }, +}, config.value); +assert.equal(result, 0); +await assert.rejects(captured(), AggregateError); +assert.deepEqual(calls, { handle: 1, server: 1 }); +assert.equal(unregistrations, 0); +await captured(); +assert.deepEqual(calls, { handle: failedOwner === "handle" ? 2 : 1, server: failedOwner === "server" ? 2 : 1 }); +assert.equal(unregistrations, 1); +await captured(); +assert.equal(unregistrations, 1); +console.log("MIXED_CLEANUP_RETRIED"); +`, + ], + { cwd: process.cwd(), encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe("MIXED_CLEANUP_RETRIED\n"); + expect(child.stderr).toBe(""); + }, +); + +it("joins shutdown with a transport and server published after shutdown starts", () => { + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` +import assert from "node:assert/strict"; +import { runDeferred } from "./dist/main/deferredRuntime.js"; +import { parseConfig } from "./dist/config/parseConfig.js"; +const config = parseConfig({ HOME: process.env.HOME }); +assert.equal(config.ok, true); +const transport = Promise.withResolvers(); +let factory; +let captured; +let signal; +let unregistrations = 0; +let handleCalls = 0; +let serverCalls = 0; +const running = runDeferred({ + env: {}, writeStderr: () => {}, setExitCode: () => {}, + registerShutdown: (handler) => { + signal = handler; + return () => { unregistrations++; }; + }, + captureShutdown: (close) => { captured = close; }, + serve: (create) => { factory = create; return transport.promise; }, +}, config.value); +signal(); +const closing = captured(); +let closed = false; +void closing.then(() => { closed = true; }); +await new Promise((resolve) => setImmediate(resolve)); +assert.equal(closed, false); +assert.equal(unregistrations, 0); +const server = factory({}); +const close = server.server.close.bind(server.server); +server.server.close = async () => { serverCalls++; await close(); }; +transport.resolve({ close: async () => { handleCalls++; } }); +assert.equal(await running, 0); +await closing; +assert.deepEqual([handleCalls, serverCalls, unregistrations], [1, 1, 1]); +await captured(); +assert.deepEqual([handleCalls, serverCalls, unregistrations], [1, 1, 1]); +console.log("LATE_TRANSPORT_CLOSED"); +`, + ], + { cwd: process.cwd(), encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe("LATE_TRANSPORT_CLOSED\n"); + expect(child.stderr).toBe(""); +}); + +it("refuses server publication after the shutdown snapshot and completed close", () => { + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` +import assert from "node:assert/strict"; +import { runDeferred } from "./dist/main/deferredRuntime.js"; +import { parseConfig } from "./dist/config/parseConfig.js"; +const config = parseConfig({ HOME: process.env.HOME }); +assert.equal(config.ok, true); +const closingHandle = Promise.withResolvers(); +const releaseHandle = Promise.withResolvers(); +let factory; +let captured; +let handleCalls = 0; +let serverCalls = 0; +let unregistrations = 0; +const result = await runDeferred({ + env: {}, writeStderr: () => {}, setExitCode: () => {}, + registerShutdown: () => () => { unregistrations++; }, + captureShutdown: (close) => { captured = close; }, + serve: (create) => { + factory = create; + const server = factory({}); + const close = server.server.close.bind(server.server); + server.server.close = async () => { serverCalls++; await close(); }; + return { close: async () => { + handleCalls++; + closingHandle.resolve(); + await releaseHandle.promise; + } }; + }, +}, config.value); +assert.equal(result, 0); +const closing = captured(); +await closingHandle.promise; +assert.throws(() => factory({}), /MCP connection is closed/); +releaseHandle.resolve(); +await closing; +assert.throws(() => factory({}), /MCP connection is closed/); +await captured(); +assert.deepEqual([handleCalls, serverCalls, unregistrations], [1, 1, 1]); +console.log("LATE_PUBLICATION_REFUSED"); +`, + ], + { cwd: process.cwd(), encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe("LATE_PUBLICATION_REFUSED\n"); + expect(child.stderr).toBe(""); +}); + +it.each([ + {}, + { REA_MCP_INPUT_SCHEMA_PROFILE: "compact" }, + { REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, + { REA_MCP_OUTPUT_SCHEMA_PROFILE: "off" }, +])("preserves the configured catalog profile %j", async (environment) => { + const { client } = await connect("legacy", environment); + const before = await client.listTools(); + const catalog = JSON.parse( + await readFile( + environment.REA_MCP_INPUT_SCHEMA_PROFILE === "full" + ? "dist/mcp-catalog-full.json" + : "dist/mcp-catalog.json", + "utf8", + ), + ); + expect(before.tools).toHaveLength(catalog.tools.length); + const expected = + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE === "off" + ? catalog.tools.map( + ({ outputSchema: _output, ...tool }: Record) => tool, + ) + : catalog.tools; + expect(before.tools).toEqual(expected); + const result = await client.callTool({ + name: "list_unknowns", + arguments: {}, + }); + expect(result.isError).not.toBe(true); + expect(await client.listTools()).toEqual(before); +}); + +it("runs the real static artifact workflow after deferred initialization", async () => { + const { client } = await connect("legacy"); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: resolve("scripts/rea.mjs") }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + const evidence = evidenceSchema.parse(inspected.structuredContent); + expect(evidence.operation).toBe("inspect_analysis_view"); + const view = analysisViewResultSchema.parse(evidence.normalized_result); + expect(view.parent_operation).toBe("inspect_artifact"); + const retained = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: view.parent_evidence_id, + }, + view: { kind: "summary" }, + }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + const closed = await client.callTool({ name: "close_binary", arguments: {} }); + expect(closed.isError, JSON.stringify(closed)).not.toBe(true); +}); + +const copyCompiledPackage = async (prefix: string): Promise => { + const directory = await createTestTempDirectory(prefix); + await cp(resolve("dist"), join(directory, "dist"), { recursive: true }); + await mkdir(join(directory, "scripts")); + await cp(resolve("scripts/rea.mjs"), join(directory, "scripts/rea.mjs")); + await cp(resolve("package.json"), join(directory, "package.json")); + await symlink( + resolve("node_modules"), + join(directory, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + return directory; +}; + +it("records the canonical catalog without a host account or ambient configuration", async () => { + const directory = await copyCompiledPackage("rea-catalog-without-account-"); + await cp( + resolve("scripts/record-mcp-catalog.mjs"), + join(directory, "scripts/record-mcp-catalog.mjs"), + ); + const temporary = join(directory, "temporary"); + await mkdir(temporary); + const hook = `data:text/javascript,${encodeURIComponent(` +import os from "node:os"; +import { syncBuiltinESMExports } from "node:module"; +os.userInfo = () => { throw new Error("Host account is unavailable"); }; +syncBuiltinESMExports(); +`)}`; + const child = spawnSync( + process.execPath, + [`--import=${hook}`, join(directory, "scripts/record-mcp-catalog.mjs")], + { + cwd: directory, + env: { + ...process.env, + HOME: join(directory, "ambient-home"), + USERPROFILE: join(directory, "ambient-home"), + TMPDIR: temporary, + TMP: temporary, + TEMP: temporary, + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_OUTPUT_SCHEMA_PROFILE: "off", + }, + encoding: "utf8", + timeout: 30000, + }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe(""); + expect(child.stderr).toBe(""); + expect(await readFile(join(directory, "dist/mcp-catalog.json"), "utf8")).toBe( + await readFile("dist/mcp-catalog.json", "utf8"), + ); + expect( + await readFile(join(directory, "dist/recordedMcpCatalog.js"), "utf8"), + ).toBe(await readFile("dist/recordedMcpCatalog.js", "utf8")); + expect(await readdir(temporary)).toEqual([]); +}); + +it.each([ + { failures: ["client"], producerFails: false }, + { failures: ["client", "server", "session"], producerFails: false }, + { failures: ["client"], producerFails: true }, + { failures: [], producerFails: true }, +])( + "preserves producer failure=$producerFails and settles every recorder owner with cleanup failures=$failures", + async ({ failures, producerFails }) => { + const directory = await copyCompiledPackage("rea-catalog-cleanup-"); + await cp( + resolve("scripts/record-mcp-catalog.mjs"), + join(directory, "scripts/record-mcp-catalog.mjs"), + ); + const log = join(directory, "cleanup.jsonl"); + const temporary = join(directory, "temporary"); + await mkdir(temporary); + // Fault injection tests recorder ownership, not provider or SDK behavior. + const close = (owner: string) => `async () => { + const { appendFileSync } = await import("node:fs"); + appendFileSync(${JSON.stringify(log)}, JSON.stringify(${JSON.stringify(owner)}) + "\\n"); + if (${JSON.stringify(failures)}.includes(${JSON.stringify(owner)})) + throw new Error(${JSON.stringify(`${owner} cleanup failed`)}); + return { ok: true }; + }`; + const replacements = { + "/dist/server/createServer.js": `export const createServer = () => ({ connect: async () => {}, close: ${close("server")} });`, + "/dist/composition/binary.js": `export const createBinarySession = () => ({ close: ${close("session")} });`, + "/dist/config/parseConfig.js": + "export const parseConfig = () => ({ ok: true, value: {} });", + "/dist/logger.js": "export const silentLogger = {};", + "/dist/contracts/promptContracts.js": + "export const PROMPT_CONTRACTS = [];", + }; + const clientModule = `data:text/javascript,${encodeURIComponent(` +export class Client { + async connect() {} + async listTools() { + if (${JSON.stringify(producerFails)}) throw new Error("catalog generation failed"); + return { tools: [] }; + } + async listPrompts() { return { prompts: [] }; } + getServerVersion() {} + getServerCapabilities() {} + getInstructions() {} + close = ${close("client")}; +} +export const InMemoryTransport = { createLinkedPair: () => [null, null] }; +`)}`; + const hook = `data:text/javascript,${encodeURIComponent(` +import { registerHooks } from "node:module"; +const replacements = ${JSON.stringify(replacements)}; +registerHooks({ + resolve(specifier, context, next) { + return specifier === "@modelcontextprotocol/client" + ? { url: ${JSON.stringify(clientModule)}, shortCircuit: true } + : next(specifier, context); + }, + load(url, context, next) { + const replacement = Object.entries(replacements).find(([path]) => url.endsWith(path)); + return replacement === undefined ? next(url, context) + : { source: replacement[1], format: "module", shortCircuit: true }; + }, +}); +`)}`; + const child = spawnSync( + process.execPath, + [`--import=${hook}`, join(directory, "scripts/record-mcp-catalog.mjs")], + { + cwd: directory, + env: { + ...process.env, + TMPDIR: temporary, + TMP: temporary, + TEMP: temporary, + }, + encoding: "utf8", + timeout: 10000, + }, + ); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(1); + expect( + new Set( + (await readFile(log, "utf8")) + .trim() + .split("\n") + .map((line): unknown => JSON.parse(line)), + ), + ).toEqual(new Set(["client", "server", "session"])); + for (const owner of failures) + expect(child.stderr).toContain(`${owner} cleanup failed`); + if (producerFails) { + expect(child.stderr).toContain("Error: catalog generation failed"); + if (failures.length > 0) + expect( + child.stderr.indexOf("Error: catalog generation failed"), + ).toBeLessThan(child.stderr.indexOf(`${failures[0]} cleanup failed`)); + } + if (failures.length > 0) + expect(child.stderr).toContain("AggregateError: Catalog cleanup failed"); + else expect(child.stderr).not.toContain("AggregateError"); + expect(await readdir(temporary)).toHaveLength(failures.length > 0 ? 1 : 0); + }, +); + +it("rejects a corrupted build-owned catalog before starting the connection", async () => { + const directory = await copyCompiledPackage("rea-corrupted-mcp-catalog-"); + await writeFile(join(directory, "dist/mcp-catalog.json"), "{}"); + const child = spawnSync( + process.execPath, + [join(directory, "scripts/rea.mjs"), "mcp"], + { cwd: directory, input: "", encoding: "utf8", timeout: 10000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(1); + expect(child.stdout).toBe(""); + expect(child.stderr).toBe( + "REA could not start. Run `rea doctor`, then restart REA from your MCP client.\n", + ); +}); diff --git a/tests/boundary/mcp/optionalProviderShutdown.test.ts b/tests/boundary/mcp/optionalProviderShutdown.test.ts index 142af125f0..f3e1bfb931 100644 --- a/tests/boundary/mcp/optionalProviderShutdown.test.ts +++ b/tests/boundary/mcp/optionalProviderShutdown.test.ts @@ -2,7 +2,10 @@ import { afterAll, expect, it, vi } from "vitest"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { resolve } from "node:path"; -import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + createDeferred, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; import { createBrowserObservationProvider } from "../../../src/composition/browserObservation.js"; import { createJavaScriptRuntimeObservationProvider } from "../../../src/composition/javascriptRuntimeObservation.js"; import { run } from "../../../src/main.js"; @@ -25,6 +28,62 @@ for (const path of [ mkdirSync(path); afterAll(() => rmSync(root, { recursive: true, force: true })); +it("refuses eager server publication after the shutdown snapshot and completed close", async () => { + const closingHandle = createDeferred(); + const releaseHandle = createDeferred(); + const acquired: ReturnType[] = []; + let publish: (() => unknown) | undefined; + let shutdown: (() => Promise) | undefined; + let closing: Promise | undefined; + let transportCloses = 0; + try { + expect( + await run({ + env: environment, + loadOptionalProviders: async () => ({}), + createServer: (source, options) => { + const server = createServer(source, options); + acquired.push(server); + return server; + }, + serve: (factory) => { + publish = () => factory({ era: "legacy" }); + publish(); + return { + close: async () => { + transportCloses++; + closingHandle.resolve(); + await releaseHandle.promise; + }, + }; + }, + captureShutdown: (close) => { + shutdown = close; + }, + registerShutdown: () => () => {}, + writeStderr: () => {}, + setExitCode: () => {}, + }), + ).toBe(0); + if (shutdown === undefined || publish === undefined) + throw new Error("Runtime ownership hooks were not installed"); + closing = shutdown(); + await closingHandle.promise; + expect(publish).toThrow("MCP connection is closed"); + releaseHandle.resolve(); + await closing; + expect(publish).toThrow("MCP connection is closed"); + await shutdown(); + expect(transportCloses).toBe(1); + expect(acquired).toHaveLength(1); + } finally { + releaseHandle.resolve(); + await closing; + await shutdown?.(); + for (const server of acquired) await server.close(); + } +}); + it("retries failed process-owned optional cleanup without reclosing completed peers", async () => { const runtime = createJavaScriptRuntimeObservationProvider(); const browser = createBrowserObservationProvider(); diff --git a/turbo.json b/turbo.json index e3be6d9d9d..c8530d501d 100644 --- a/turbo.json +++ b/turbo.json @@ -21,6 +21,7 @@ "scripts/clean-build-output.mjs", "scripts/bundle-runtime.mjs", "scripts/record-catalog-digests.mjs", + "scripts/record-mcp-catalog.mjs", "scripts/generate-package-metadata.mjs", "src/**/*.{cts,mts,ts,tsx}", "src/**/package.json",