diff --git a/src/server/promptCompletion.ts b/src/server/promptCompletion.ts index 995eecb958..21d67ededc 100644 --- a/src/server/promptCompletion.ts +++ b/src/server/promptCompletion.ts @@ -8,7 +8,6 @@ import type { BinarySessionPort } from "../application/binary/BinarySessionPort. import type { PromptCompletionKind } from "../contracts/promptContracts.js"; import { currentMcpRequestSignal } from "./EvidenceMcpServer.js"; import { parseArtifactInventoryEvidence } from "../domain/artifactInventoryEvidence.js"; -import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; import { parseProcessCapture } from "../domain/process/processCaptureParsing.js"; @@ -181,17 +180,11 @@ const evidenceValues = ( if (kind === "evidence") return [evidence.evidence_id]; if (kind === "capture") return isProcessCaptureEvidence(evidence) ? [evidence.evidence_id] : []; - if (evidence.operation === "inventory_artifact") + if ( + evidence.operation === "inventory_artifact" || + evidence.operation === "inspect_artifact" + ) return artifactInventoryValues(kind, evidence); - if (evidence.operation === "inspect_artifact") { - const inspection = artifactInspectionResultSchema.safeParse( - evidence.normalized_result, - ); - if (!inspection.success) return []; - return inspection.data.substeps.flatMap(({ evidence: inventoryEvidence }) => - artifactInventoryValues(kind, inventoryEvidence), - ); - } return []; }; diff --git a/tests/boundary/mcp/promptCompletion.test.ts b/tests/boundary/mcp/promptCompletion.test.ts index 789f07c9db..8d919c3c7a 100644 --- a/tests/boundary/mcp/promptCompletion.test.ts +++ b/tests/boundary/mcp/promptCompletion.test.ts @@ -1,5 +1,5 @@ import { ok as resultOk } from "../../../src/domain/result.js"; -import { rm, writeFile } from "node:fs/promises"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -14,16 +14,19 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; import { PROCESS_CAPTURE_REFERENCE } from "../../../src/contracts/investigationExamples.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; -import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; import { artifactInspectionResultSchema, createArtifactInspection, } from "../../../src/domain/artifactInspection.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { digestCanonicalValue } from "../../../src/domain/canonicalDigest.js"; import { createPromptCompletionSource } from "../../../src/server/promptCompletion.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { fragmentInventoryEvidence } from "../../fixtures/artifactEvidence.js"; +import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; let directory: string | undefined; @@ -210,69 +213,74 @@ describe("guided prompt completion from investigation records", () => { }); describe("guided prompt completion from retained records", () => { - it("offers identifiers from every retained inspection inventory substep", async () => { + it("offers identifiers from every authenticated page in one inspection inventory", async () => { const session = createTestBinarySession(() => client([])); - const inspectionEvidence = createCompositeInspectionEvidence(); + const { evidence: inspectionEvidence, inventory: completeInventory } = + await createPagedInspectionEvidence(); expect(session.recordEvidence(inspectionEvidence).ok).toBe(true); const completion = createPromptCompletionSource(session, session); - const expectedManifests = [ - ARTIFACT_COMPARISON_EXAMPLE.left, - ARTIFACT_COMPARISON_EXAMPLE.right, - ] - .map( - ({ normalized_result }) => - artifactInventoryResultSchema.parse(normalized_result).manifest - .manifest_id, - ) - .sort(); - const expectedOccurrences = [ - ARTIFACT_COMPARISON_EXAMPLE.left, - ARTIFACT_COMPARISON_EXAMPLE.right, - ] - .flatMap(({ normalized_result }) => - artifactInventoryResultSchema - .parse(normalized_result) - .occurrences.map(({ occurrence_id }) => occurrence_id), - ) - .sort(); + const pages = artifactInspectionResultSchema.parse( + inspectionEvidence.normalized_result, + ).substeps; + const firstPage = pages[0]?.evidence.normalized_result; + const laterPage = pages[1]?.evidence.normalized_result; + if (firstPage === undefined || laterPage === undefined) + throw new Error("paged inspection fixture must contain two pages"); + expect( + artifactInventoryResultSchema.parse(firstPage).occurrences.length, + ).toBeLessThan(completeInventory.occurrences.length); + expect( + artifactInventoryResultSchema.parse(laterPage).occurrences.length, + ).toBeGreaterThan(0); - expect(await completion.complete("manifest", "agm_")).toEqual( - expectedManifests, - ); + expect(await completion.complete("manifest", "agm_")).toEqual([ + completeInventory.manifest.manifest_id, + ]); expect(await completion.complete("occurrence", "occ_")).toEqual( - expectedOccurrences, + completeInventory.occurrences + .map(({ occurrence_id }) => occurrence_id) + .sort(), ); await session.close(); }); - it("skips an inner inventory whose payload no longer matches its Evidence ID", async () => { + it.each([ + "child", + "coverage", + "substep-link", + "declared-subject", + "carrier-subject", + ] as const)( + "rejects inspection completions when %s is invalid", + async (corruption) => { + const session = createTestBinarySession(() => client([])); + const { evidence: inspectionEvidence } = + await createPagedInspectionEvidence(corruption); + expect(session.recordEvidence(inspectionEvidence).ok).toBe(true); + const completion = createPromptCompletionSource(session, session); + + expect(await completion.complete("manifest", "agm_")).toEqual([]); + expect(await completion.complete("occurrence", "occ_")).toEqual([]); + + await session.close(); + }, + ); + + it("still completes a valid standalone inventory Evidence", async () => { const session = createTestBinarySession(() => client([])); - const inspectionEvidence = createCompositeInspectionEvidence(true); - const corruptedInner = artifactInspectionResultSchema.parse( - inspectionEvidence.normalized_result, - ).substeps[1]?.evidence; - if (corruptedInner === undefined) - throw new Error("composite fixture must contain a second inventory"); - expect(corruptedInner.evidence_id).toBe( - ARTIFACT_COMPARISON_EXAMPLE.right.evidence_id, - ); - expect( - artifactInventoryResultSchema.safeParse(corruptedInner.normalized_result) - .success, - ).toBe(true); - expect(() => parseEvidence(corruptedInner)).toThrow(); - expect(session.recordEvidence(inspectionEvidence).ok).toBe(true); + const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left; + expect(session.recordEvidence(inventoryEvidence).ok).toBe(true); const completion = createPromptCompletionSource(session, session); - const firstInventory = artifactInventoryResultSchema.parse( - ARTIFACT_COMPARISON_EXAMPLE.left.normalized_result, + const inventory = artifactInventoryResultSchema.parse( + inventoryEvidence.normalized_result, ); expect(await completion.complete("manifest", "agm_")).toEqual([ - firstInventory.manifest.manifest_id, + inventory.manifest.manifest_id, ]); expect(await completion.complete("occurrence", "occ_")).toEqual( - firstInventory.occurrences.map(({ occurrence_id }) => occurrence_id), + inventory.occurrences.map(({ occurrence_id }) => occurrence_id).sort(), ); await session.close(); @@ -351,52 +359,145 @@ const fixtureEnvironment = { isolation: "process" as const, }; -const createCompositeInspectionEvidence = (corruptSecondInventory = false) => { - const leftInspection = createArtifactInspection( - ARTIFACT_COMPARISON_EXAMPLE.left, +type InspectionCorruption = + | "child" + | "coverage" + | "substep-link" + | "declared-subject" + | "carrier-subject"; + +const createPagedInspectionEvidence = async ( + corruption?: InspectionCorruption, +) => { + directory = await createTestTempDirectory("rea-prompt-inspection-pages-"); + const path = join(directory, "fixture.app"); + await mkdir(path); + await Promise.all([ + writeFile(join(path, "first.js"), "first();"), + writeFile(join(path, "second.js"), "second();"), + ]); + const inventoryResult = await inventoryArtifact(path); + const source = createEvidence( + { + path, + sha256: inventoryResult.manifest.root_sha256, + format: inventoryResult.manifest.root_format, + }, + fixtureProvider, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse(inventoryResult), + confidence: "observed", + authority: "shipped-artifact", + }, ); - const rightInspection = createArtifactInspection( - ARTIFACT_COMPARISON_EXAMPLE.right, - ); - const substeps = [...leftInspection.substeps, ...rightInspection.substeps]; - if (corruptSecondInventory) { - const secondSubstep = substeps[1]; - if (secondSubstep === undefined) - throw new Error("composite fixture must contain a second inventory"); + const pages = fragmentInventoryEvidence(source, 2); + const firstPage = pages[0]; + if (firstPage === undefined) throw new Error("missing first inventory page"); + const base = createArtifactInspection(firstPage); + const { inspection_id: _inspectionId, ...baseSemantic } = base; + const substeps = pages.map((evidence) => ({ + substep_id: `ais_${digestCanonicalValue( + { operation: "inventory_artifact", evidence_id: evidence.evidence_id }, + "Artifact inspection", + )}`, + operation: "inventory_artifact" as const, + status: "completed" as const, + evidence_id: evidence.evidence_id, + evidence, + })); + const linkedEvidenceIds = [ + ...base.evidence_links, + ...pages.map(({ evidence_id }) => evidence_id), + ].sort(); + let inspection = { + ...baseSemantic, + substeps, + evidence_links: [...new Set(linkedEvidenceIds)], + coverage: { + ...base.coverage, + substeps_completed: pages.length, + }, + }; + + if (corruption === "child") { + const first = inspection.substeps[0]; + if (first === undefined) throw new Error("missing first inspection step"); const inventory = artifactInventoryResultSchema.parse( - secondSubstep.evidence.normalized_result, + first.evidence.normalized_result, ); - substeps[1] = { - ...secondSubstep, - evidence: { - ...secondSubstep.evidence, - normalized_result: { - ...inventory, - coverage: { ...inventory.coverage, status: "unknown" }, + inspection = { + ...inspection, + substeps: [ + { + ...first, + evidence: { + ...first.evidence, + normalized_result: { + ...inventory, + limitations: [...inventory.limitations, "Changed after sealing"], + }, + }, }, + ...inspection.substeps.slice(1), + ], + }; + } else if (corruption === "coverage") { + inspection = { + ...inspection, + coverage: { + ...inspection.coverage, + substeps_completed: inspection.substeps.length + 1, + }, + }; + } else if (corruption === "substep-link") { + const first = inspection.substeps[0]; + if (first === undefined) throw new Error("missing first inspection step"); + inspection = { + ...inspection, + substeps: [ + { ...first, evidence_id: `ev_${"f".repeat(64)}` }, + ...inspection.substeps.slice(1), + ], + }; + } else if (corruption === "declared-subject") { + inspection = { + ...inspection, + subject: { + ...inspection.subject, + root_artifact_id: `art_${"f".repeat(64)}`, }, }; } - const inspection = artifactInspectionResultSchema.parse({ - ...leftInspection, - substeps, - evidence_links: [ - ...leftInspection.evidence_links, - ...rightInspection.evidence_links, - ], - coverage: { - ...leftInspection.coverage, - substeps_completed: substeps.length, + + const semantic = inspection; + const result = { + ...semantic, + inspection_id: `ai_${digestCanonicalValue(semantic, "Artifact inspection")}`, + }; + const target = source.subject; + if (target === null) throw new Error("fixture inventory has no subject"); + const evidence = createEvidence( + { + path: target.local_path, + sha256: + corruption === "carrier-subject" + ? "a".repeat(64) + : target.digest.sha256, + format: target.format, }, - }); - return createEvidence(undefined, fixtureProvider, { - operation: "inspect_artifact", - parameters: {}, - result: jsonValueSchema.parse(inspection), - confidence: "observed", - authority: "shipped-artifact", - evidenceLinks: inspection.evidence_links, - }); + fixtureProvider, + { + operation: "inspect_artifact", + parameters: {}, + result: jsonValueSchema.parse(result), + confidence: "observed", + authority: "shipped-artifact", + evidenceLinks: result.evidence_links, + }, + ); + return { evidence, inventory: inventoryResult }; }; const client = (