From be6863e9a16bcc3a09d87c50981e69292795a512 Mon Sep 17 00:00:00 2001 From: kvalls <95490470+kvalls@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:11:06 +0100 Subject: [PATCH 01/74] feat(ghidra): support native PE DLLs on Windows --- .github/workflows/real-ghidra-windows.yml | 20 +++ AGENTS.md | 2 +- docs/installation.md | 2 +- docs/managed-code-analysis.md | 5 +- docs/provider-evaluation.md | 9 +- docs/roadmap.md | 2 +- docs/windows-ghidra-p0.md | 15 +- scripts/create-ghidra-windows-fixture.mjs | 155 +++++++++++++-------- scripts/verify-packaged-ghidra-windows.mjs | 114 ++++++++++++--- scripts/verify-real-ghidra-windows.mjs | 27 +++- src/ghidra/GhidraAnalysisProfile.ts | 6 + src/ghidra/GhidraProvider.behavior.test.ts | 84 ++++++----- src/ghidra/GhidraProvider.ts | 9 +- src/ghidra/GhidraProviderCapabilities.ts | 2 +- 14 files changed, 316 insertions(+), 136 deletions(-) diff --git a/.github/workflows/real-ghidra-windows.yml b/.github/workflows/real-ghidra-windows.yml index 42dc38fa46..d6de7469f7 100644 --- a/.github/workflows/real-ghidra-windows.yml +++ b/.github/workflows/real-ghidra-windows.yml @@ -64,6 +64,14 @@ jobs: env: REA_ANALYSIS_PROVIDER: ghidra run: npm run verify:ghidra:windows -- --x86 | Tee-Object -FilePath windows-ghidra-x86-proof.log + - name: Verify native x64 DLL Windows Ghidra operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows -- --dll | Tee-Object -FilePath windows-ghidra-dll-proof.log + - name: Verify native x86 DLL Windows Ghidra operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows -- --x86 --dll | Tee-Object -FilePath windows-ghidra-x86-dll-proof.log - name: Verify ordinary-user CLI and MCP operations env: REA_ANALYSIS_PROVIDER: ghidra @@ -72,6 +80,14 @@ jobs: env: REA_ANALYSIS_PROVIDER: ghidra run: npm run verify:ghidra:windows:package -- --x86 | Tee-Object -FilePath windows-ghidra-x86-package-proof.log + - name: Verify ordinary-user x64 DLL CLI and MCP operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows:package -- --dll | Tee-Object -FilePath windows-ghidra-dll-package-proof.log + - name: Verify ordinary-user x86 DLL CLI and MCP operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows:package -- --x86 --dll | Tee-Object -FilePath windows-ghidra-x86-dll-package-proof.log - uses: actions/upload-artifact@v4 if: always() with: @@ -81,4 +97,8 @@ jobs: windows-ghidra-package-proof.log windows-ghidra-x86-proof.log windows-ghidra-x86-package-proof.log + windows-ghidra-dll-proof.log + windows-ghidra-x86-dll-proof.log + windows-ghidra-dll-package-proof.log + windows-ghidra-x86-dll-package-proof.log retention-days: 30 diff --git a/AGENTS.md b/AGENTS.md index 1acb825527..01ecf3d91d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Product Direction -REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64/arm64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. +REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64/arm64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications and DLLs; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. Prioritize: diff --git a/docs/installation.md b/docs/installation.md index a0be329205..dfa3bc84bf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -452,7 +452,7 @@ entry comments atomically and returns refreshed analysis. These session metadata edits leave executable bytes unchanged and are discarded on close. GUI controls require Hopper; Windows P0 remains read-only. -Windows P0 admits native x86 and x86-64 PE applications on fixed local NTFS volumes. +Windows P0 admits native x86 and x86-64 PE applications and DLLs on fixed local NTFS volumes. The npm package bundles native Job Object ownership, protected private runtime DACLs, and handle-based path admission; no separate addon installation is needed. See the [Windows Ghidra P0 guide](windows-ghidra-p0.md) for verified scope and diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 92bc230609..bb87a7a635 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -125,8 +125,9 @@ The upstream analyzer currently gates its own analysis to x86-64. NativeAOT's broader platform matrix does not establish REA coverage: PE/COFF, ELF, Mach-O, shared libraries, target architectures, and runtime metadata versions each need their own provider verification. REA's experimental Windows Ghidra boundary is -limited to admitted native x86 and x86-64 PE applications, so it does not currently -establish coverage for PE DLLs. Mobile NativeAOT targets should remain +limited to admitted native x86 and x86-64 PE applications and DLLs for static +analysis. This does not establish Windows NativeAOT metadata-recovery coverage. +Mobile NativeAOT targets should remain experimental until REA has a matching provider and package-level verification. For REA's native analysis, preserve the same artifact path and digest used by diff --git a/docs/provider-evaluation.md b/docs/provider-evaluation.md index 8aa655e52a..a18303a92b 100644 --- a/docs/provider-evaluation.md +++ b/docs/provider-evaluation.md @@ -18,7 +18,7 @@ comments atomically, verifies readback, and returns refreshed function analysis. Changes live only in the session database and leave executable bytes unchanged. Ghidra does not expose GUI controls; Windows P0 remains read-only. -Windows x64 P0 supports native x86 and x86-64 PE applications on local NTFS with +Windows x64 P0 supports native x86 and x86-64 PE applications and DLLs on local NTFS with bundled Job Object ownership, protected runtime DACLs, and handle-based path admission. Real ordinary-user verification covers all 25 read-only operations through the packaged CLI and MCP, target integrity, and cleanup. This remains @@ -62,7 +62,7 @@ maximum, when the release leaves those at the 12.1 defaults). Otherwise doctor probes `java`/`javac` or `java.exe`/`javac.exe` from `PATH`. Supported Linux and macOS hosts accept compatible ELF, PE, and Mach-O executable targets. Host admission and target compatibility are separate checks. Windows x64 P0 admits only native, -non-managed, non-DLL x86 and x86-64 PE applications on fixed local NTFS, with the packaged +non-managed x86 and x86-64 PE applications and DLLs on fixed local NTFS, with the packaged native authority available before launch. The launcher creates one ephemeral runtime root with project, @@ -136,8 +136,9 @@ Mach-O target coverage. It requires `clang`, LLD, and `lld-link`; set these tools before compilation. Keeping this matrix separate lets Linux host/provider acceptance run with only the host compiler. -`npm run verify:ghidra:windows` and its `-- --x86` variant check the source-owned native x86 and x86-64 PE -fixture, all 25 read-only operations, target/snapshot/import SHA-256 linkage, +`npm run verify:ghidra:windows` and its `-- --x86` variant check source-owned native x86 and x86-64 PE +applications. Adding `-- --dll` or `-- --x86 --dll` selects entry-point-free DLLs +with exported caller/callee functions. These lanes check all 25 read-only operations, target/snapshot/import SHA-256 linkage, and project, endpoint, process, and runtime cleanup. The independent native lane checks DACLs, handle admission, cancellation, and Job Object lifecycle. `npm run verify:ghidra:windows:package` packs and installs REA into an isolated diff --git a/docs/roadmap.md b/docs/roadmap.md index b4ac0ba545..077efa6325 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -87,7 +87,7 @@ and exact commands. Experimental Windows x64 P0 now supplies bundled Job Object ownership, protected runtime DACLs, and handle-based admission for native x86 and x86-64 PE -applications on local NTFS. Real ordinary-user CLI/MCP verification covers +applications and DLLs on local NTFS. Real ordinary-user CLI/MCP verification covers all 25 read-only operations and cleanup. The [Windows P0 guide](windows-ghidra-p0.md) describes this boundary and unverified broader coverage. diff --git a/docs/windows-ghidra-p0.md b/docs/windows-ghidra-p0.md index ec54f5908a..5f50d832c2 100644 --- a/docs/windows-ghidra-p0.md +++ b/docs/windows-ghidra-p0.md @@ -17,7 +17,7 @@ additional permission flag or degraded mode is required. - Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+; - an operator-installed Ghidra 12.1.x distribution (verified with 12.1.4); - a 64-bit full JDK inside that installation's declared Java range (JDK 21 or newer, with no maximum, for current 12.1 releases); -- an explicit native, non-managed, non-DLL x86 or x86-64 PE application; and +- an explicit native, non-managed x86 or x86-64 PE application or DLL; and - the 25 read-only Ghidra inventory, memory, and function-analysis operations. Loaded memory reads and file offsets preserve Ghidra's observed source mappings. @@ -38,6 +38,13 @@ REA selects its packaged bridge scripts by explicit path. Same-named files or directories in the caller's working directory do not select a different script or require users to clean that directory before analysis. +Native DLL analysis preserves the PE shared-library role in the analysis profile. +REA imports an immutable copy for static analysis; it does not load the library +into the analyst process or execute its entry routine. The source-owned DLL +fixtures have no entry routine or imports and expose two exported functions. +The same native snapshot, path-admission and cleanup requirements apply to EXEs +and DLLs. + ## Configuration ```powershell @@ -100,7 +107,7 @@ and artifact SHA-256 before loading the package-owned addon. Native failure reasons remain distinct from a valid installation and an unsupported target. PE header classification and SHA-256 come from the same open file. The Windows -provider admits native x86 and x86-64 PE applications and rejects unsupported roles, +provider admits native x86 and x86-64 PE applications and DLLs and rejects unsupported roles, architectures, managed images, and malformed headers. The original selected source coordinate is preserved for native admission. @@ -159,7 +166,11 @@ artifact and manifest. npm run verify:windows-native npm run verify:ghidra:windows npm run verify:ghidra:windows -- --x86 +npm run verify:ghidra:windows -- --dll +npm run verify:ghidra:windows -- --x86 --dll npm run verify:ghidra:windows:package -- --x86 +npm run verify:ghidra:windows:package -- --dll +npm run verify:ghidra:windows:package -- --x86 --dll npm run verify:ghidra:windows:package -- C:\fixtures\installed-rea C:\fixtures\sample.exe ``` diff --git a/scripts/create-ghidra-windows-fixture.mjs b/scripts/create-ghidra-windows-fixture.mjs index 6538ec57f8..e610b46880 100644 --- a/scripts/create-ghidra-windows-fixture.mjs +++ b/scripts/create-ghidra-windows-fixture.mjs @@ -3,68 +3,103 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +// Source-owned static fixtures: no compiler, target loading or execution. for (const architecture of ["x86_64", "x86"]) { - const x86 = architecture === "x86"; - const output = resolve( - root, - "build", - "fixtures", - x86 ? "rea-ghidra-windows-x86.exe" : "rea-ghidra-windows.exe", - ); - const bytes = Buffer.alloc(1024); - const peOffset = 0x80; - const optionalHeader = peOffset + 24; - const optionalHeaderSize = x86 ? 0xe0 : 0xf0; - const sectionTable = optionalHeader + optionalHeaderSize; + for (const dll of [false, true]) { + const x86 = architecture === "x86"; + const output = resolve( + root, + "build", + "fixtures", + `rea-ghidra-windows${x86 ? "-x86" : ""}.${dll ? "dll" : "exe"}`, + ); + const bytes = Buffer.alloc(1024); + const peOffset = 0x80; + const optionalHeader = peOffset + 24; + const optionalHeaderSize = x86 ? 0xe0 : 0xf0; + const sectionTable = optionalHeader + optionalHeaderSize; + const directories = optionalHeader + (x86 ? 96 : 112); - bytes.write("MZ", 0, "ascii"); - bytes.writeUInt32LE(peOffset, 0x3c); - bytes.write("PE\0\0", peOffset, "binary"); - bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4); - bytes.writeUInt16LE(1, peOffset + 6); - bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20); - bytes.writeUInt16LE(x86 ? 0x102 : 0x22, peOffset + 22); + bytes.write("MZ", 0, "ascii"); + bytes.writeUInt32LE(peOffset, 0x3c); + bytes.write("PE\0\0", peOffset, "binary"); + bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4); + bytes.writeUInt16LE(1, peOffset + 6); + bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20); + bytes.writeUInt16LE( + (x86 ? 0x102 : 0x22) | (dll ? 0x2000 : 0), + peOffset + 22, + ); - bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader); - bytes.writeUInt32LE(0x200, optionalHeader + 4); - bytes.writeUInt32LE(0x1000, optionalHeader + 16); - bytes.writeUInt32LE(0x1000, optionalHeader + 20); - if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28); - else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24); - bytes.writeUInt32LE(0x1000, optionalHeader + 32); - bytes.writeUInt32LE(0x200, optionalHeader + 36); - bytes.writeUInt16LE(6, optionalHeader + 40); - bytes.writeUInt16LE(6, optionalHeader + 48); - bytes.writeUInt32LE(0x2000, optionalHeader + 56); - bytes.writeUInt32LE(0x200, optionalHeader + 60); - bytes.writeUInt16LE(3, optionalHeader + 68); - bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70); - if (x86) { - bytes.writeUInt32LE(0x10_0000, optionalHeader + 72); - bytes.writeUInt32LE(0x1000, optionalHeader + 76); - bytes.writeUInt32LE(0x10_0000, optionalHeader + 80); - bytes.writeUInt32LE(0x1000, optionalHeader + 84); - bytes.writeUInt32LE(16, optionalHeader + 92); - } else { - bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72); - bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80); - bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88); - bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96); - bytes.writeUInt32LE(16, optionalHeader + 108); + bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader); + bytes.writeUInt32LE(0x200, optionalHeader + 4); + bytes.writeUInt32LE(dll ? 0 : 0x1000, optionalHeader + 16); + bytes.writeUInt32LE(0x1000, optionalHeader + 20); + if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28); + else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24); + bytes.writeUInt32LE(0x1000, optionalHeader + 32); + bytes.writeUInt32LE(0x200, optionalHeader + 36); + bytes.writeUInt16LE(6, optionalHeader + 40); + bytes.writeUInt16LE(6, optionalHeader + 48); + bytes.writeUInt32LE(0x2000, optionalHeader + 56); + bytes.writeUInt32LE(0x200, optionalHeader + 60); + bytes.writeUInt16LE(3, optionalHeader + 68); + bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70); + if (x86) { + bytes.writeUInt32LE(0x10_0000, optionalHeader + 72); + bytes.writeUInt32LE(0x1000, optionalHeader + 76); + bytes.writeUInt32LE(0x10_0000, optionalHeader + 80); + bytes.writeUInt32LE(0x1000, optionalHeader + 84); + bytes.writeUInt32LE(16, optionalHeader + 92); + } else { + bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72); + bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80); + bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88); + bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96); + bytes.writeUInt32LE(16, optionalHeader + 108); + } + + bytes.write(".text", sectionTable, "ascii"); + bytes.writeUInt32LE(dll ? 0x180 : 0x16, sectionTable + 8); + bytes.writeUInt32LE(0x1000, sectionTable + 12); + bytes.writeUInt32LE(0x200, sectionTable + 16); + bytes.writeUInt32LE(0x200, sectionTable + 20); + bytes.writeUInt32LE(0x6000_0020, sectionTable + 36); + + // EXE entry returns zero. DLL caller adds one to the callee's 42. + bytes.set( + dll + ? [0xe8, 0x0b, 0, 0, 0, 0x83, 0xc0, 1, 0xc3] + : [0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], + 0x200, + ); + bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210); + + if (dll) { + // Export both functions; an entry-point-free library is discovered by exports. + bytes.writeUInt32LE(0x1040, directories); + bytes.writeUInt32LE(0x130, directories + 4); + const exports = 0x240; + bytes.writeUInt32LE(0x1100, exports + 12); + bytes.writeUInt32LE(1, exports + 16); + bytes.writeUInt32LE(2, exports + 20); + bytes.writeUInt32LE(2, exports + 24); + bytes.writeUInt32LE(0x1080, exports + 28); + bytes.writeUInt32LE(0x1088, exports + 32); + bytes.writeUInt32LE(0x1090, exports + 36); + bytes.writeUInt32LE(0x1000, 0x280); + bytes.writeUInt32LE(0x1010, 0x284); + bytes.writeUInt32LE(0x1120, 0x288); + bytes.writeUInt32LE(0x1140, 0x28c); + bytes.writeUInt16LE(0, 0x290); + bytes.writeUInt16LE(1, 0x292); + bytes.write("rea-fixture.dll\0", 0x300, "binary"); + bytes.write("fixture_caller\0", 0x320, "binary"); + bytes.write("fixture_return42\0", 0x340, "binary"); + } + + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, bytes); + if (!x86 && !dll) process.stdout.write(`${output}\n`); } - - bytes.write(".text", sectionTable, "ascii"); - bytes.writeUInt32LE(0x16, sectionTable + 8); - bytes.writeUInt32LE(0x1000, sectionTable + 12); - bytes.writeUInt32LE(0x200, sectionTable + 16); - bytes.writeUInt32LE(0x200, sectionTable + 20); - bytes.writeUInt32LE(0x6000_0020, sectionTable + 36); - - // entry calls the second function, returns zero; second returns 42. - bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200); - bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210); - - await mkdir(dirname(output), { recursive: true }); - await writeFile(output, bytes); - if (!x86) process.stdout.write(`${output}\n`); } diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 17d29c4ada..268d949d9e 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -15,7 +15,7 @@ import { import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -24,10 +24,43 @@ if (process.platform !== "win32" || process.arch !== "x64") throw new Error( "Packaged Windows Ghidra acceptance requires a Windows x64 host.", ); -const workspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); const exec = promisify(execFile); -const x86Fixture = process.argv[2] === "--x86"; -const packageRootArgument = x86Fixture ? undefined : process.argv[2]; +const arguments_ = process.argv.slice(2); +// Windows locks the loaded package-owned addon until the verifier exits. +// Keep installation cleanup in a parent that never loads that addon. +if (arguments_[0] !== "--internal-workspace") { + const ownedWorkspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); + try { + const result = await exec( + process.execPath, + [ + fileURLToPath(import.meta.url), + "--internal-workspace", + ownedWorkspace, + ...arguments_, + ], + { timeout: 900_000, maxBuffer: 8 * 1024 * 1024 }, + ); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + } catch (cause) { + if (typeof cause.stdout === "string") process.stdout.write(cause.stdout); + if (typeof cause.stderr === "string") process.stderr.write(cause.stderr); + throw cause; + } finally { + await rm(ownedWorkspace, { recursive: true, force: true }); + } + process.exit(0); +} +assert.ok(arguments_[1] !== undefined, "Missing verifier workspace."); +const workspace = resolve(arguments_[1]); +const fixtureArguments = arguments_.slice(2); +const x86Fixture = fixtureArguments.includes("--x86"); +const dllFixture = fixtureArguments.includes("--dll"); +const [packageRootArgument, targetArgument, reportArgument] = + fixtureArguments.filter( + (argument) => argument !== "--x86" && argument !== "--dll", + ); let packageRoot = resolve(packageRootArgument ?? "."); // The default lane verifies the npm artifact in an isolated prefix. Explicit // package roots support an already-installed artifact without a second install. @@ -75,17 +108,17 @@ if (packageRootArgument === undefined) { ); packageRoot = join(prefix, "node_modules", "rea-agents"); } catch (cause) { - await rm(workspace, { recursive: true, force: true }); - throw cause; + throw new Error("Packaged Windows verifier installation failed.", { + cause, + }); } } const target = resolve( - process.argv[3] ?? - (x86Fixture - ? "build/fixtures/rea-ghidra-windows-x86.exe" - : "build/fixtures/rea-ghidra-windows.exe"), + targetArgument ?? + `build/fixtures/rea-ghidra-windows${x86Fixture ? "-x86" : ""}.${dllFixture ? "dll" : "exe"}`, ); let TOOL_CONTRACTS, native, token, sha256; +let parsedTarget; try { ({ TOOL_CONTRACTS } = await import( pathToFileURL(join(packageRoot, "dist/contracts/toolContracts.js")) @@ -107,9 +140,25 @@ try { ); const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); sha256 = digest(await readFile(target)); + const { parseBinaryTarget } = await import( + pathToFileURL(join(packageRoot, "dist/application/BinaryTargetResolver.js")) + ); + parsedTarget = await parseBinaryTarget(target); + assert.equal(parsedTarget.ok, true); + assert.equal(parsedTarget.value.format, "pe"); + assert.equal(parsedTarget.value.managed, false); + if (targetArgument === undefined) { + assert.equal( + parsedTarget.value.architecture, + x86Fixture ? "x86" : "x86_64", + ); + assert.equal( + parsedTarget.value.executableRole, + dllFixture ? "shared-library" : "application", + ); + } } catch (cause) { - await rm(workspace, { recursive: true, force: true }); - throw cause; + throw new Error("Packaged Windows verifier prerequisites failed.", { cause }); } const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); const runtimeParent = join(workspace, "runtime with spaces"); @@ -145,6 +194,8 @@ const report = { architecture: "x64", nonAdmin: true, targetSha256: sha256, + targetArchitecture: parsedTarget.value.architecture, + targetRole: parsedTarget.value.executableRole, cli: {}, mcpOperations: [], nativeControls: "available", @@ -208,8 +259,8 @@ try { }); const call = async (name, arguments_) => { report.lastOperation = name; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); const contract = TOOL_CONTRACTS.find((value) => value.name === name); assert.ok(contract !== undefined, `Missing canonical contract: ${name}`); const argumentsParsed = contract.inputSchema.parse(arguments_); @@ -230,6 +281,25 @@ try { await call("list_names", {}); const procedures = await call("list_procedures", {}); assert.ok(procedures.length > 0); + if (targetArgument === undefined) { + const calleeAddress = x86Fixture ? "0x401010" : "0x140001010"; + const calleeCode = await call("procedure_pseudo_code", { + procedure: calleeAddress, + }); + assert.match(calleeCode, /\b(?:42|0x2a)\b/iu); + if (dllFixture) { + const callerAddress = x86Fixture ? "0x401000" : "0x140001000"; + assert.ok( + ( + await call("procedure_callees", { procedure: callerAddress }) + ).includes(calleeAddress), + ); + assert.match( + await call("procedure_pseudo_code", { procedure: callerAddress }), + /\+\s*1\b/u, + ); + } + } assert.ok((await call("list_segments", {})).length > 0); await call("list_strings", {}); const procedure = @@ -296,6 +366,12 @@ try { "Packaged verification omitted an admitted operation.", ); const session = await call("binary_session", {}); + assert.equal( + session.analysis_provider_binding?.analysis_profile?.parameters + ?.executable_role, + parsedTarget.value.executableRole, + "MCP profile must preserve the selected PE role.", + ); assert.equal( session.analysis_provider_binding?.selection_source, "auto-single-candidate", @@ -327,8 +403,8 @@ try { report.callerScriptCollisionsPreserved = true; report.runtimeCleanup = true; report.ok = true; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); process.stdout.write(`${JSON.stringify(report)}\n`); } catch (cause) { report.failure = { @@ -338,9 +414,7 @@ try { .replaceAll(target, "") .replaceAll(workspace, ""), }; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); throw cause; -} finally { - await rm(workspace, { recursive: true, force: true }); } diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index 1c5b3b829a..3332ea4e2d 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -62,21 +62,21 @@ if ( ); const root = resolve(fileURLToPath(new URL("..", import.meta.url))); -const architecture = process.argv[2] === "--x86" ? "x86" : "x86_64"; +const architecture = process.argv.includes("--x86") ? "x86" : "x86_64"; +const dllFixture = process.argv.includes("--dll"); +const role = dllFixture ? "shared-library" : "application"; const targetPath = resolve( root, "build", "fixtures", - architecture === "x86" - ? "rea-ghidra-windows-x86.exe" - : "rea-ghidra-windows.exe", + `rea-ghidra-windows${architecture === "x86" ? "-x86" : ""}.${dllFixture ? "dll" : "exe"}`, ); const target = await parseBinaryTarget(targetPath); if ( !target.ok || target.value.format !== "pe" || target.value.architecture !== architecture || - target.value.executableRole !== "application" || + target.value.executableRole !== role || target.value.managed !== false ) throw new Error( @@ -89,6 +89,8 @@ const profile = await resolveGhidraAnalysisProfile( ); if (!profile.ok || profile.value.profile === null) throw new Error("Windows Ghidra profile could not be committed"); +if (profile.value.profile.parameters.executable_role !== role) + throw new Error("Windows Ghidra profile lost the observed PE role"); const client = new GhidraClient({ platform: installation.platform, @@ -162,6 +164,19 @@ try { throw new Error( "Windows fixture decompilation lost its known return value", ); + if (dllFixture) { + const callerAddress = architecture === "x86" ? "0x401000" : "0x140001000"; + const callees = await functionOperation("procedure_callees", { + procedure: callerAddress, + }); + if (!callees.includes(callee.address)) + throw new Error("DLL export caller lost its controlled call reference"); + const callerCode = await functionOperation("procedure_pseudo_code", { + procedure: callerAddress, + }); + if (typeof callerCode !== "string" || !/\+\s*1\b/u.test(callerCode)) + throw new Error("DLL export caller lost its addition of one"); + } await inventory("inspect_native_load_image", {}); const memory = await inventory("read_bytes", { @@ -254,7 +269,7 @@ try { limitations: [ "approved-non-sensitive-fixtures-only", "windows-x64-local-ntfs-only", - "native-x86-and-x86-64-pe-applications-only", + "native-x86-and-x86-64-pe-applications-and-dlls-only", "no-gui-or-mutation-authority", ], }; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 1af7469fa0..eeb8646a41 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -34,6 +34,12 @@ export const resolveGhidraAnalysisProfile = ( profile: createAnalysisProfile(provider, { target_kind: target.kind, target_format: target.format, + ...(target.format === "pe" + ? { + executable_role: target.executableRole ?? null, + managed: target.managed ?? null, + } + : {}), architecture: target.architecture ?? null, available_architectures: [ ...(target.availableArchitectures ?? []), diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 7bbbc8c89d..88e1eb07fe 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -270,7 +270,7 @@ describe("Ghidra platform support", () => { expect( ghidra.inspectTargetSupport({ ...nativeApplication, - executableRole: "shared-library", + executableRole: "non-executable", }), ).toMatchObject({ status: "unsupported", @@ -295,45 +295,59 @@ describe("Ghidra platform support", () => { code: "architecture_unsupported", }); }); - it("admits generated PE32 applications while rejecting unsupported x86 roles", async () => { +}); + +describe("Ghidra Windows PE fixture admission", () => { + it("admits generated PE32 and PE32+ DLLs and applications while preserving native-only Windows admission", async () => { const root = fileURLToPath(new URL("../../", import.meta.url)); await promisify(execFile)(process.execPath, [ join(root, "scripts/create-ghidra-windows-fixture.mjs"), ]); - const path = join(root, "build/fixtures/rea-ghidra-windows-x86.exe"); - const bytes = await readFile(path); - const metadata = parseExecutableHeader(bytes, "x64"); - expect(metadata).toMatchObject({ - ok: true, - value: { - format: "pe", - architecture: "x86", - executableRole: "application", - managed: false, - }, - }); - if (!metadata.ok) throw new Error(metadata.error); - if (metadata.value.format !== "pe") - throw new Error("Expected a PE fixture"); - const target: BinaryTarget = { - path, - sha256: createHash("sha256").update(bytes).digest("hex"), - kind: "executable", - ...metadata.value, - }; const ghidra = provider({ ...installationHost(), platform: "win32" }); - expect(ghidra.inspectTargetSupport(target).status).toBe("supported"); - for (const [candidate, code] of [ - [ - { ...target, executableRole: "shared-library" }, - "target_role_unsupported", - ], - [{ ...target, managed: true }, "managed_target_unsupported"], - ] as const) - expect(ghidra.inspectTargetSupport(candidate)).toMatchObject({ - status: "unsupported", - code, - }); + for (const architecture of ["x86", "x86_64"] as const) { + for (const dll of [false, true]) { + const path = join( + root, + "build/fixtures", + `rea-ghidra-windows${architecture === "x86" ? "-x86" : ""}.${dll ? "dll" : "exe"}`, + ); + const bytes = await readFile(path); + const metadata = parseExecutableHeader(bytes, "x64"); + const role = dll ? "shared-library" : "application"; + expect(metadata).toMatchObject({ + ok: true, + value: { + format: "pe", + architecture, + executableRole: role, + managed: false, + }, + }); + if (!metadata.ok) throw new Error(metadata.error); + if (metadata.value.format !== "pe") + throw new Error("Expected a PE fixture"); + const target: BinaryTarget = { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + kind: "executable", + ...metadata.value, + }; + expect(ghidra.inspectTargetSupport(target).status).toBe("supported"); + for (const [candidate, code] of [ + [ + { ...target, executableRole: "non-executable" }, + "target_role_unsupported", + ], + [{ ...target, managed: true }, "managed_target_unsupported"], + [{ ...target, architecture: "arm64" }, "architecture_unsupported"], + ] as const) { + expect(ghidra.inspectTargetSupport(candidate)).toMatchObject({ + status: "unsupported", + code, + }); + } + } + } }); }); diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index d657a156e3..d782827d96 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -247,12 +247,15 @@ const inspectWindowsP0TargetSupport = ( reason: "Windows Ghidra P0 accepts x86 and x86-64 PE targets only.", diagnostics, }; - if (target.executableRole !== "application") + if ( + target.executableRole !== "application" && + target.executableRole !== "shared-library" + ) return { status: "unsupported", code: "target_role_unsupported", reason: - "Windows Ghidra P0 accepts PE applications, not DLL or non-executable images.", + "Windows Ghidra P0 accepts PE applications and DLLs, not non-executable or unclassified images.", diagnostics, }; if (target.managed !== false) @@ -260,7 +263,7 @@ const inspectWindowsP0TargetSupport = ( status: "unsupported", code: "managed_target_unsupported", reason: - "Windows Ghidra P0 accepts native PE applications; managed or unclassified PE targets are unsupported.", + "Windows Ghidra P0 accepts native PE applications and DLLs; managed or unclassified PE targets are unsupported.", diagnostics, }; return { diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index edbbb11c31..5b9a6ca8ba 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -31,7 +31,7 @@ export const healthLimitations = Object.freeze([ /** Additional limitations applied to the experimental Windows x64 P0 boundary. */ export const windowsP0Limitations = Object.freeze([ - "Windows Ghidra P0 accepts approved native x86 and x86-64 PE applications only; DLL, managed, hostile, sensitive, and mutable-path targets are unsupported.", + "Windows Ghidra P0 accepts approved native x86 and x86-64 PE applications and DLLs; managed, hostile, sensitive, and mutable-path targets are unsupported.", "The Windows bridge uses authenticated IPv4 loopback because Node path-based IPC does not expose Java AF_UNIX sockets; the endpoint file contains no bearer token.", "Windows sessions require the matching packaged Windows x64 native addon, local NTFS targets and runtimes, and Windows 10 or later. Native handles enforce path admission, private DACLs, and Job Object ownership automatically.", ]); From e10c934d5a756c1a4c7b547ebd02c403ddfde7b8 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 01:36:11 +0200 Subject: [PATCH 02/74] fix(ghidra): keep the cross-format branch callee multi-block The cross-format oracle requires a callee whose recovered control flow has more than one basic block, and rea_cross_branch provided it through a symmetric if/else of tail calls. Clang 21 folds that shape into a conditional negate plus one tail branch, a single basic block, so the lane rejected a correct Ghidra analysis of the fixture. Give the branch a loop whose trip count depends on a volatile global: no compiler can eliminate the backedge, and the callee stays multi-block at every optimization level on every toolchain. --- tests/conformance/ghidra/cross-format.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/conformance/ghidra/cross-format.c b/tests/conformance/ghidra/cross-format.c index f718b9b540..22efa61597 100644 --- a/tests/conformance/ghidra/cross-format.c +++ b/tests/conformance/ghidra/cross-format.c @@ -8,10 +8,15 @@ __attribute__((noinline, used)) int rea_cross_leaf(int value) { } __attribute__((noinline, used)) int rea_cross_branch(int value) { - if (value > 10) { - return rea_cross_leaf(value); + /* A volatile-dependent loop keeps this callee multi-block at every + optimization level: newer clang folds a symmetric if/else of tail calls + into a conditional negate, and cross-format verification requires a + recovered multi-block callee. */ + int adjusted = value > 10 ? value + 1 : -value; + while (adjusted > 100) { + adjusted -= rea_cross_global; } - return rea_cross_leaf(-value); + return rea_cross_leaf(adjusted); } __attribute__((noinline, used)) int rea_cross_indirect( From 40f632a155bff2defdea09e467229c703bc0b0e9 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 02:06:15 +0200 Subject: [PATCH 03/74] ci: re-run the suite after the main-branch shard flake The failed shard is the JavaScriptSemanticGraphBuilder short-circuit mutation test, which also failed on main run 38003459598 at the same base commit (08ec2bf8) before this branch started CI, and passed on main run 38003459600 minutes later at the same commit. From e17290cd2a17931d7a2af70f74b76b94276bd788 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 02:34:52 +0200 Subject: [PATCH 04/74] ci: re-run the suite on the corrected main checkpoint b8a8190b restored the guarded-mutation uncertainty the failing test encodes, so the shard should pass against the new merge base. From e4a8e0b728154fe0bf181684ccd0d73df45cf4bf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:44:32 +0000 Subject: [PATCH 05/74] fix(javascript): recognize Electron APIs through renamed bindings Electron recognition matched callee spellings, so a bundler rename such as esbuild's `import { ipcMain as ipcMain2 } from "electron"`, or a destructured `const { contextBridge: bridge } = require("electron")`, hid every IPC handler, BrowserWindow, contextBridge and utility process behind it. Resolve each call and construction root lexically to its Electron import or require origin, and classify the callee under that export's name. Shadowed locals keep their own meaning. On Hermes Desktop v0.21.6, whose main process is one esbuild ESM bundle, recognized main IPC handlers rise from 1 to 204, BrowserWindow constructions from 0 to 11, and paired renderer transmissions from 1 to 201. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XPY2GmufMNGQKoEgRUx1pd --- .../electronStaticAnalysisBrowser.ts | 8 +- .../javascript/electronStaticAnalysisIpc.ts | 3 +- .../electronStaticAnalysisValues.ts | 21 ++++- .../javascript/javascriptSemanticAnalysis.ts | 79 +++++++++++++++++++ .../javascript/javascriptStaticAnalysis.ts | 27 +++---- .../javascriptStaticAnalysisState.ts | 2 + .../electronApplicationAnalysis.test.ts | 66 ++++++++++++++++ 7 files changed, 186 insertions(+), 20 deletions(-) diff --git a/src/domain/javascript/electronStaticAnalysisBrowser.ts b/src/domain/javascript/electronStaticAnalysisBrowser.ts index f29e7cf8f4..a44baf89a7 100644 --- a/src/domain/javascript/electronStaticAnalysisBrowser.ts +++ b/src/domain/javascript/electronStaticAnalysisBrowser.ts @@ -12,6 +12,7 @@ import type { import { boundedExpression, collectContextBridgeMembers, + electronCalleeName, electronStaticValue, objectProperty, } from "./electronStaticAnalysisValues.js"; @@ -19,7 +20,6 @@ import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { addLocatedFinding } from "./javascriptStaticAnalysisFindings.js"; import { argumentNode, - calleeName, range, staticPath, staticPathResolutionContext, @@ -42,7 +42,7 @@ const inspectBrowserWindow = ( node: t.NewExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); if (name !== "BrowserWindow" && !name.endsWith(".BrowserWindow")) return; const options = argumentNode(node.arguments[0]); const collected = collectWindowOptions(context.source, options); @@ -165,7 +165,7 @@ const inspectContextBridge = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); const main = name === "contextBridge.exposeInMainWorld" || name.endsWith(".contextBridge.exposeInMainWorld"); @@ -213,7 +213,7 @@ const inspectUtilityProcess = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); if (name !== "utilityProcess.fork" && !name.endsWith(".utilityProcess.fork")) return; const moduleNode = argumentNode(node.arguments[0]); diff --git a/src/domain/javascript/electronStaticAnalysisIpc.ts b/src/domain/javascript/electronStaticAnalysisIpc.ts index 9efc2670b8..a560ea1844 100644 --- a/src/domain/javascript/electronStaticAnalysisIpc.ts +++ b/src/domain/javascript/electronStaticAnalysisIpc.ts @@ -7,6 +7,7 @@ import type { } from "./electronStaticAnalysisTypes.js"; import { boundedExpression, + electronCalleeName, electronStaticValue, handlerKind, } from "./electronStaticAnalysisValues.js"; @@ -36,7 +37,7 @@ const inspectIpcCall = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); const descriptor = ipcDescriptor(name); if (descriptor === undefined) return; const channelNode = argumentNode(node.arguments[0]); diff --git a/src/domain/javascript/electronStaticAnalysisValues.ts b/src/domain/javascript/electronStaticAnalysisValues.ts index 2a800e6bbd..c2bb6e76a2 100644 --- a/src/domain/javascript/electronStaticAnalysisValues.ts +++ b/src/domain/javascript/electronStaticAnalysisValues.ts @@ -6,7 +6,26 @@ import { readExactJavaScriptLiteral, semanticStaticPropertyName, } from "./javascriptAstValues.js"; -import { sourceSlice } from "./javascriptStaticAnalysisHelpers.js"; +import { calleeName, sourceSlice } from "./javascriptStaticAnalysisHelpers.js"; +import type { JavaScriptFindingContext } from "./javascriptStaticAnalysisState.js"; + +/** Name a callee, spelling a root renamed from Electron as its export. */ +export const electronCalleeName = ( + callee: t.Node, + context: JavaScriptFindingContext, +): string => { + const name = calleeName(callee); + let root = callee; + while (t.isMemberExpression(root) || t.isOptionalMemberExpression(root)) { + if (!t.isNode(root.object)) return name; + root = root.object; + } + if (!t.isIdentifier(root)) return name; + const exported = context.electronBindings?.get(root.start ?? -1); + return exported === undefined + ? name + : `${exported}${name.slice(root.name.length)}`; +}; /** Preserve one literal value or the exact inert expression. */ export const electronStaticValue = ( diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 5a873429e1..b5c1c7f5df 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -29,6 +29,7 @@ import { semanticVariableScope, } from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js"; import { collectSemanticReturns, @@ -282,6 +283,84 @@ const resolveGlobalAliasFact = ( return openGlobalFact(name); }; +const ELECTRON_MODULE = /^electron(?:\/(?:common|main|renderer|utility))?$/u; + +/** + * Electron export paths for call and construction roots bound to Electron + * under another name, such as esbuild's `import { ipcMain as ipcMain2 }`. + * Keys are root identifier offsets; roots spelled as their export are omitted. + */ +export const classifyParsedJavaScriptElectronBindings = ( + file: ParsedJavaScriptSource, +): ReadonlyMap => { + const facts = new Map(); + const state = createState(file.program); + collectDefinitions(file.program, state); + const exportsByBinding = new Map(); + traverseJavaScriptAst(file.program, { + enter: (node) => { + if ( + !t.isCallExpression(node) && + !t.isOptionalCallExpression(node) && + !t.isNewExpression(node) + ) + return; + const root = calleeRoot(node.callee); + if (root === undefined) return; + const binding = resolveSemanticBindingState(state, root, root.name); + if (binding === undefined) return; + let exported = exportsByBinding.get(binding.bindingId); + if (exported === undefined) { + exported = electronExport(binding, state); + exportsByBinding.set(binding.bindingId, exported); + } + if (exported !== null && exported !== root.name) + facts.set(root.start ?? -1, exported); + }, + }); + return facts; +}; + +const calleeRoot = (node: t.Node): t.Identifier | undefined => { + while (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + if (!t.isNode(node.object)) return undefined; + node = node.object; + } + return t.isIdentifier(node) ? node : undefined; +}; + +// Read import and require origins directly; general value evaluation is +// unnecessary here and costly across large vendor bundles. +const electronExport = ( + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): string | null => { + const origin = bindingOrigin(binding, state); + return origin !== undefined && + ELECTRON_MODULE.test(origin.specifier) && + origin.importedPath.length > 0 + ? origin.importedPath.join(".") + : null; +}; + +const bindingOrigin = ( + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): JavaScriptModuleOrigin | undefined => { + if (binding.directOrigins.length > 0) + return binding.directOrigins.length === 1 + ? binding.directOrigins[0] + : undefined; + const [initializer] = binding.initializers; + if (binding.mutable || binding.initializers.length !== 1) return undefined; + const origin = semanticRequireOrigin(initializer?.node, state); + const projection = initializer?.projection ?? []; + return origin === undefined || + !projection.every((key): key is string => typeof key === "string") + ? undefined + : { ...origin, importedPath: [...origin.importedPath, ...projection] }; +}; + const createState = (program: t.Program): JavaScriptSemanticAnalysisState => { const root: JavaScriptSemanticScopeState = { scopeId: semanticScopeId("program", program), diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index af20eaa926..524af811eb 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -25,9 +25,11 @@ import { finalizeLocatedFindings } from "./javascriptStaticAnalysisFindings.js"; import { createJavaScriptAnalysisAccumulator, type JavaScriptAnalysisAccumulator as AnalysisAccumulator, + type JavaScriptFindingContext, } from "./javascriptStaticAnalysisState.js"; import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; import { + classifyParsedJavaScriptElectronBindings, classifyParsedJavaScriptOpenReceivers, type JavaScriptOpenReceiverFact, } from "./javascriptSemanticAnalysis.js"; @@ -57,21 +59,24 @@ export const analyzeParsedJavaScriptStaticSource = ( openReceiverFacts?: ReadonlyMap, ): JavaScriptStaticAnalysis => { const accumulator = createJavaScriptAnalysisAccumulator(source.length); - traverseStaticSource(source, file, accumulator, openReceiverFacts); + traverseStaticSource(file, { + source, + accumulator, + ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), + electronBindings: classifyParsedJavaScriptElectronBindings(file), + }); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); }; const traverseStaticSource = ( - source: string, file: ParsedJavaScriptSource, - accumulator: AnalysisAccumulator, - openReceiverFacts?: ReadonlyMap, + findings: JavaScriptFindingContext, ): void => { traverseJavaScriptAst(file, { enter: (node) => { - accumulator.visitedNodes += 1; - inspectNode(source, node, accumulator, openReceiverFacts); + findings.accumulator.visitedNodes += 1; + inspectNode(node, findings); return undefined; }, }); @@ -156,16 +161,10 @@ const finalizeStaticAnalysis = ( }; const inspectNode = ( - source: string, node: t.Node, - accumulator: AnalysisAccumulator, - openReceiverFacts?: ReadonlyMap, + findings: JavaScriptFindingContext, ): void => { - const findings = { - source, - accumulator, - ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), - }; + const { source, accumulator } = findings; inspectElectronStaticNode(node, findings); if (t.isCallExpression(node)) { inspectBundlerRegistration(source, node, accumulator); diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 23de6952bf..47e25bccdb 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -65,6 +65,8 @@ export interface JavaScriptFindingContext { | "local-indexed-db" | "local-cache-storage" >; + /** Electron export paths for renamed call roots, by identifier offset. */ + readonly electronBindings?: ReadonlyMap; } /** Candidate import, require, worker, or service-worker reference. */ diff --git a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts index 3484f98a24..4180e1818d 100644 --- a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts +++ b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts @@ -89,6 +89,33 @@ describe("static Electron application analysis", () => { expect(requestedMembers).toEqual(["*"]); }); + it("maps Electron boundaries through bundler-renamed bindings", async () => { + const root = await renamedBindingFixtureDirectory(); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const graph = parseJavaScriptApplicationGraph(result.graph); + const handledChannels = graph.nodes.flatMap(({ kind, observations }) => + kind === "ipc-handler" + ? observations.flatMap(({ properties }) => + properties.side === "main" ? [properties.channel] : [], + ) + : [], + ); + + expect(result.electron_summary).toMatchObject({ + browser_windows: 1, + context_bridge_apis: 1, + exposed_api_members: 2, + ipc: { + main_handlers: 2, + paired_renderer_transmissions: 2, + unpaired_literal_renderer_transmissions: 0, + }, + utility_processes: 1, + }); + expect(handledChannels.sort()).toEqual(["rea:read", "rea:write"]); + }); + it("returns a tagged cancellation without executing application code", async () => { const root = await fixtureDirectory(); const controller = new AbortController(); @@ -299,6 +326,45 @@ const fixtureDirectory = async (): Promise => { return root; }; +const renamedBindingFixtureDirectory = async (): Promise => { + const root = await createTestTempDirectory("rea-electron-renamed-"); + await Promise.all([ + writeFile( + join(root, "package.json"), + JSON.stringify({ name: "rea-electron-renamed", main: "main.mjs" }), + ), + // Bundlers rename colliding imports, e.g. esbuild's ipcMain2. + writeFile( + join(root, "main.mjs"), + String.raw` +import { BrowserWindow as BrowserWindow2, ipcMain as ipcMain2 } from "electron"; +import { ipcMain as ipcMain3, utilityProcess as utility } from "electron/main"; + +new BrowserWindow2({ webPreferences: { preload: "./preload.cjs" } }); +ipcMain2.handle("rea:read", async () => "value"); +ipcMain3.on("rea:write", () => undefined); +utility.fork("./worker.js"); +function register(ipcMain2) { + ipcMain2.handle("rea:shadowed", async () => "local"); +} +export { register }; +`, + ), + writeFile( + join(root, "preload.cjs"), + String.raw` +const { contextBridge: bridge, ipcRenderer: ipc } = require("electron"); +bridge.exposeInMainWorld("reaApi", { + read: () => ipc.invoke("rea:read"), + write: (value) => ipc.send("rea:write", value), +}); +`, + ), + writeFile(join(root, "worker.js"), "module.exports = {};"), + ]); + return root; +}; + const computedMemberFixtureDirectory = async (): Promise => { const root = await createTestTempDirectory("rea-electron-computed-members-"); await mkdir(join(root, "native")); From 7623e21ff02cf4eb00ccb4fa357936e4ed01425e Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:00:11 -0700 Subject: [PATCH 06/74] perf(native): record pwntools diagnostics once in their Evidence (#1404) inspect_binary_layout and inspect_recorded_crash passed the decoded report as both result and rawResult, so every Evidence record carried the whole report twice (and an MCP response four times, counting its text copy). The decoded report is the only representation, so raw_result is now null, as for other single-representation operations. The real pwntools verifiers asserted the duplicate; they now assert null. Co-authored-by: Claude Opus 5.5 --- scripts/verify-binary-layout.mjs | 2 +- scripts/verify/recorded-crash/public.mjs | 2 +- .../binaryDiagnostics/BinaryLayoutService.test.ts | 14 ++++++++++++++ .../binaryDiagnostics/BinaryLayoutService.ts | 3 ++- .../binaryDiagnostics/RecordedCrashService.test.ts | 1 + .../binaryDiagnostics/RecordedCrashService.ts | 3 ++- tests/fixtures/analysisView.ts | 2 +- 7 files changed, 22 insertions(+), 5 deletions(-) diff --git a/scripts/verify-binary-layout.mjs b/scripts/verify-binary-layout.mjs index 996eb39666..1f3b5e4bc4 100644 --- a/scripts/verify-binary-layout.mjs +++ b/scripts/verify-binary-layout.mjs @@ -1283,7 +1283,7 @@ async function inspect( evidence.provider.version, "pwntools@4.15.0;pyelftools@0.33;unicorn@2.1.2", ); - assert.deepEqual(evidence.normalized_result, evidence.raw_result); + assert.equal(evidence.raw_result, null); assert.equal(evidence.normalized_result.diagnostics.truncated, false); return evidence.normalized_result; } diff --git a/scripts/verify/recorded-crash/public.mjs b/scripts/verify/recorded-crash/public.mjs index 76713bf9f4..ecdfd14cd2 100644 --- a/scripts/verify/recorded-crash/public.mjs +++ b/scripts/verify/recorded-crash/public.mjs @@ -80,7 +80,7 @@ export async function connectRecordedCrash({ entrypoint, environment }) { evidence.confidence, debuggerContext ? "derived" : "observed", ); - assert.deepEqual(evidence.raw_result, report); + assert.equal(evidence.raw_result, null); assert.equal(evidence.subject.local_path, path); assert.equal(report.artifact.path, path); assert.equal( diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts index e8f3e2c6dd..e65da65f67 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts @@ -5,6 +5,7 @@ import { BINARY_LAYOUT_TEST_PROVIDER, } from "../../../tests/fixtures/binaryDiagnostics/layout.js"; import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { parseEvidence } from "../../domain/evidence.js"; import { err, ok } from "../../domain/result.js"; it.each([ @@ -28,6 +29,19 @@ it.each([ }, ); +it("records the decoded layout once in its Evidence", async () => { + const layout = binaryLayoutFixture(); + const service = new BinaryLayoutService({ + identity: BINARY_LAYOUT_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(layout)), + }); + const result = await service.inspect({ path: layout.artifact.path }); + if (!result.ok) throw result.error; + const evidence = parseEvidence(result.value); + expect(evidence.normalized_result).toEqual(layout); + expect(evidence.raw_result).toBeNull(); +}); + it("preserves a provider's actionable unsupported reason", async () => { const failure = new AnalysisCapabilityUnavailableError( "fixture-layout", diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.ts b/src/application/binaryDiagnostics/BinaryLayoutService.ts index 11bfeb76e9..bdb07da56a 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.ts @@ -78,7 +78,8 @@ export class BinaryLayoutService { operation: OPERATION, parameters: input.data, result: value, - rawResult: value, + // The decoded report is the only representation; do not repeat it. + rawResult: null, confidence: "observed", limitations: value.limitations, locations: [{ kind: "artifact-path", path: value.artifact.path }], diff --git a/src/application/binaryDiagnostics/RecordedCrashService.test.ts b/src/application/binaryDiagnostics/RecordedCrashService.test.ts index 6060aef332..0ab49ee1ba 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.test.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.test.ts @@ -24,6 +24,7 @@ it("labels an envelope containing debugger-derived maps as derived", async () => const evidence = parseEvidence(result.value); expect(evidence.confidence).toBe("derived"); expect(evidence.normalized_result).toEqual(fixture); + expect(evidence.raw_result).toBeNull(); }); it.each([ diff --git a/src/application/binaryDiagnostics/RecordedCrashService.ts b/src/application/binaryDiagnostics/RecordedCrashService.ts index cfc9278dc1..5a4fff645d 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.ts @@ -83,7 +83,8 @@ export class RecordedCrashService { operation: OPERATION, parameters: input.data, result: value, - rawResult: value, + // The decoded report is the only representation; do not repeat it. + rawResult: null, confidence: value.debugger.status === "available" ? "derived" : "observed", limitations: value.limitations, diff --git a/tests/fixtures/analysisView.ts b/tests/fixtures/analysisView.ts index 0f928a22a2..d3be2b1e3b 100644 --- a/tests/fixtures/analysisView.ts +++ b/tests/fixtures/analysisView.ts @@ -99,7 +99,7 @@ export const analysisViewLayoutEvidence = ( operation: "inspect_binary_layout", parameters: { path: layout.artifact.path }, result: layout, - rawResult: layout, + rawResult: null, confidence: "observed", limitations: layout.limitations, locations: [{ kind: "artifact-path", path: layout.artifact.path }], From 1f0f728c63a4d9f32fc183c6a41627eedb8d1fe2 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:00:35 -0700 Subject: [PATCH 07/74] perf(hopper): stop repeating Hopper replies as raw_result (#1405) HopperProvider passed the bridge reply as rawResult and, for every operation except address_to_file_offset, the same object as the result. Every Hopper Evidence record (decompilation, procedure and string listings, xrefs, ...) therefore carried its complete result twice, and an MCP response four times counting its text copy. raw_result is now null unless REA transformed the reply, which only the file-offset mapping does; that case keeps Hopper's original reply. Co-authored-by: Claude Opus 5.5 --- src/hopper/HopperProvider.ts | 3 ++- tests/acceptance/applications/runtime.test.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 5b1da0ba9b..6c314ddb68 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -249,7 +249,8 @@ export class HopperProvider implements AnalysisProviderCandidate { ? { ok: true, value: createAnalysisExecution(mapped.value, executionProvider, { - rawResult: result.value, + // Only the file-offset mapping differs from Hopper's reply. + rawResult: mapped === result ? null : result.value, ...(profile === undefined ? {} : { analysisProfile: profile }), limitations: [ ...(preparedImage === undefined diff --git a/tests/acceptance/applications/runtime.test.ts b/tests/acceptance/applications/runtime.test.ts index 1a50c3fc10..bfa75b7219 100644 --- a/tests/acceptance/applications/runtime.test.ts +++ b/tests/acceptance/applications/runtime.test.ts @@ -144,6 +144,8 @@ describe("production stdio runtime", () => { expect(evidence.operation).toBe("current_document"); expect(evidence.provider.id).toBe("hopper"); expect(evidence.normalized_result).toBe(observation.normalized_result); + // Hopper's reply is the result; it is not repeated as raw_result. + expect(evidence.raw_result).toBeNull(); expect(records).toContainEqual( expect.objectContaining({ application: "rea", From 8550913cda311e5340cf012b9c4afd622e36e35b Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:00:53 +0800 Subject: [PATCH 08/74] fix(javascript)!: stop copying literals into semantic metadata (#1407) Commit literal values with canonical JSON SHA-256 in semantic role keys and use a type label for strings. Preserve complete values, provenance and literal queries while avoiding two payload copies in each string node. Verify real compiled CLI/default stdio MCP, typed union seeds, empty strings and trace continuation with a large source-owned fixture. Refs #723 BREAKING CHANGE: Newly produced semantic literal node IDs use value-sha256 role keys, and string labels read "string literal". Read exact values from properties.value and obtain node IDs from the selected parent graph. Previously stored Evidence and its original IDs remain valid. --- docs/javascript-application-workflows.md | 12 ++ .../JavaScriptSemanticGraphValueProjection.ts | 7 +- .../mcp/javascriptLiteralPayload.test.ts | 154 ++++++++++++++++++ 3 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 tests/boundary/mcp/javascriptLiteralPayload.test.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index cb3e1e5e9c..c257464164 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -90,6 +90,18 @@ keep the result ambiguous. Runtime Evidence can later corroborate an exact mapped candidate, but structural reachability, semantic influence, runtime observation, and causal proof remain separate claims. +Semantic literal nodes keep their complete value in `properties.value`; literal +queries match that field. String labels read `string literal` rather than +repeating the payload. A literal's `identity.role_key` commits its canonical +JSON value as `value-sha256:`, together with its binding and property +role. This keeps large values out of identity and display metadata without +truncating their Evidence. + +New analyses therefore produce different literal node IDs from the earlier +payload-bearing role keys. Use IDs returned by the selected parent graph and +read literal values from `properties.value`, rather than decoding role keys or +labels. Previously stored graphs and their original IDs remain valid inputs. + ## Version comparison REA pairs entities only when a tier produces one unique candidate on each side. diff --git a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts index b8002fcd0c..dfe8267b6f 100644 --- a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts @@ -2,6 +2,7 @@ import { semanticContainer, semanticPropertyPointer, } from "../../domain/javascript/javascriptSemanticSlots.js"; +import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; import type { JavaScriptSemanticBinding } from "../../domain/javascript/javascriptSemanticIr.js"; import type { JavaScriptSemanticProperty, @@ -156,9 +157,11 @@ const addLiteralNode = ( ) => retainSemanticGraphNode(context.state, context.file, { kind: "literal", - roleKey: `literal:${binding.bindingId}:${role}:${JSON.stringify(value)}`, + // Identity needs an exact value commitment, not another payload copy. + // Keep the complete literal in properties for queries and Evidence export. + roleKey: `literal:${binding.bindingId}:${role}:value-sha256:${digestCanonicalValue(value, "Semantic literal")}`, location: binding.definitions[0]?.location ?? null, - label: JSON.stringify(value), + label: typeof value === "string" ? "string literal" : JSON.stringify(value), functionNodeId: context.bindingNodes.get(binding.bindingId)?.function_node_id ?? null, properties: { value }, diff --git a/tests/boundary/mcp/javascriptLiteralPayload.test.ts b/tests/boundary/mcp/javascriptLiteralPayload.test.ts new file mode 100644 index 0000000000..b7fcbb8764 --- /dev/null +++ b/tests/boundary/mcp/javascriptLiteralPayload.test.ts @@ -0,0 +1,154 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "delivers complete large literals through CLI and default stdio MCP without metadata copies", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-literal-payload-"); + const value = 'rea_literal_payload_ğ\\"\0'.padEnd(24, "x").repeat(65_536); + const source = [ + `export const payload = ${JSON.stringify(value)};`, + 'export const choice = flag ? 1 : "1";', + 'export const empty = "";', + ].join("\n"); + await writeFile(join(root, "main.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + timeoutMs: 60_000, + }); + expect(cliResponse.exitCode).toBe(0); + const cliEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(cliResponse.json); + + const client = new Client({ name: "literal-payload-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + // The old payload-bearing role key/label exceed the default wire budget. + expect(response.isError).not.toBe(true); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + const textEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(text.text)); + expect(textEvidence.evidence_id).toBe(evidence.evidence_id); + const graph = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ).semantic_graph; + const literal = graph.nodes.find( + (node) => node.kind === "literal" && node.properties.value === value, + ); + if (literal === undefined) throw new Error("Complete literal was lost"); + const sourceSha256 = createHash("sha256").update(source).digest("hex"); + expect(literal.identity).toMatchObject({ + module_path: "main.js", + artifact_sha256: sourceSha256, + }); + expect(literal.evidence.location).toMatchObject({ + available: true, + value: { kind: "source-range", source: "main.js" }, + }); + const choices = graph.nodes.filter( + (node) => + node.kind === "literal" && + (node.properties.value === 1 || node.properties.value === "1"), + ); + expect(choices.length).toBe(2); + expect(new Set(choices.map((node) => node.node_id)).size).toBe(2); + expect( + graph.nodes.some( + (node) => node.kind === "literal" && node.properties.value === "", + ), + ).toBe(true); + + const traceResponse = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + query: { + seed: { kind: "semantic-node", node_id: literal.node_id }, + direction: "backward-provenance", + }, + }, + }); + expect(traceResponse.isError).not.toBe(true); + const traceEvidence = toolContract( + "trace_javascript_semantics", + ).outputSchema.parse(traceResponse.structuredContent); + const trace = javaScriptSemanticTraceResultSchema.parse( + traceEvidence.normalized_result, + ); + expect( + trace.nodes.some( + (node) => + node.node_id === literal.node_id && node.properties.value === value, + ), + ).toBe(true); + const valueTraceResponse = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + query: { + seed: { kind: "literal", value: "1" }, + direction: "backward-provenance", + }, + }, + }); + expect(valueTraceResponse.isError).not.toBe(true); + const valueTraceEvidence = toolContract( + "trace_javascript_semantics", + ).outputSchema.parse(valueTraceResponse.structuredContent); + const valueTrace = javaScriptSemanticTraceResultSchema.parse( + valueTraceEvidence.normalized_result, + ); + expect(valueTrace.seed_node_ids).toEqual( + choices + .filter((node) => node.properties.value === "1") + .map((node) => node.node_id), + ); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } + }, + 60_000, +); From de8fc6719596305b1d76204e6b052cf46d7e7fc3 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:23:47 +0800 Subject: [PATCH 09/74] fix(javascript): retain shorter literal metadata encodings (#1408) Choose the shorter complete JSON or digest key and string display label. Short values keep their existing identities and labels, while large values retain the payload-copy reduction without an arbitrary truncation limit. --- docs/javascript-application-workflows.md | 18 ++++++++++-------- .../JavaScriptSemanticGraphValueProjection.ts | 17 +++++++++++++---- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index c257464164..0093710351 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -91,16 +91,18 @@ mapped candidate, but structural reachability, semantic influence, runtime observation, and causal proof remain separate claims. Semantic literal nodes keep their complete value in `properties.value`; literal -queries match that field. String labels read `string literal` rather than -repeating the payload. A literal's `identity.role_key` commits its canonical -JSON value as `value-sha256:`, together with its binding and property -role. This keeps large values out of identity and display metadata without +queries match that field. String labels use `string literal` when that is +shorter than the complete JSON value. A literal's `identity.role_key` commits +its canonical JSON value as `value-sha256:` when the digest key is +shorter; smaller values keep their existing JSON key. This keeps large values +out of identity and display metadata without expanding short values or truncating their Evidence. -New analyses therefore produce different literal node IDs from the earlier -payload-bearing role keys. Use IDs returned by the selected parent graph and -read literal values from `properties.value`, rather than decoding role keys or -labels. Previously stored graphs and their original IDs remain valid inputs. +New analyses therefore produce different IDs for hashed literals from the +earlier payload-bearing role keys. Use IDs returned by the selected parent +graph and read literal values from `properties.value`, rather than decoding +role keys or labels. Previously stored graphs and their original IDs remain +valid inputs. ## Version comparison diff --git a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts index dfe8267b6f..febe23d247 100644 --- a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts @@ -154,18 +154,27 @@ const addLiteralNode = ( binding: JavaScriptSemanticBinding, value: string | number | boolean | null, role: string, -) => - retainSemanticGraphNode(context.state, context.file, { +) => { + const encodedValue = JSON.stringify(value); + const valueDigest = `value-sha256:${digestCanonicalValue(value, "Semantic literal")}`; + const valueKey = + encodedValue.length <= valueDigest.length ? encodedValue : valueDigest; + const stringLabel = "string literal"; + return retainSemanticGraphNode(context.state, context.file, { kind: "literal", // Identity needs an exact value commitment, not another payload copy. // Keep the complete literal in properties for queries and Evidence export. - roleKey: `literal:${binding.bindingId}:${role}:value-sha256:${digestCanonicalValue(value, "Semantic literal")}`, + roleKey: `literal:${binding.bindingId}:${role}:${valueKey}`, location: binding.definitions[0]?.location ?? null, - label: typeof value === "string" ? "string literal" : JSON.stringify(value), + label: + typeof value === "string" && encodedValue.length > stringLabel.length + ? stringLabel + : encodedValue, functionNodeId: context.bindingNodes.get(binding.bindingId)?.function_node_id ?? null, properties: { value }, }); +}; /** Create one canonical slot for a root binding and exact static property path. */ export const semanticPropertySlot = ( From ca3131b36ccf025f64b9e43ecae2698d37747de7 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:29:48 -0700 Subject: [PATCH 10/74] perf(cli): write compact JSON when output is not a terminal (#1413) `--json` / `--format json` always indented its streamed result. Agents and scripts read REA through pipes and redirects, where indentation carries no information: on real application results it added 16-45% model tokens (o200k proxy) and 25-73% bytes. The skill's CLI fallback uses `--json`. Indent only when stdout is a TTY, as `gh --json` does; piped or redirected JSON is compact. The JSON value, `jsonl`, `--filter-output`, `--full-output` and the token-window paths are unchanged. `jq .` restores indentation for files. Co-authored-by: Claude Opus 5.5 --- docs/cli.md | 5 +++- scripts/verify/javascript/output.mjs | 6 ++-- src/cli/jsonOutput.ts | 18 ++++++++++-- .../tools/historicalCaptureResources.test.ts | 28 +++++++++++-------- tests/boundary/cli/jsonOutput.test.ts | 15 ++++++++++ 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index ed5f810e98..2a9863c1e9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -179,7 +179,10 @@ user permissions; consult the chosen guide for host requirements and effects. ## Output and exit status The default terminal format is TOON. Use `--json` when saving results for a -JSON consumer. Output selection and formatting do not change operation status. +JSON consumer. `--json` is indented on a terminal and compact when piped or +redirected, which keeps large results smaller for agents and scripts; pipe +through `jq .` for indented files. Output selection and formatting do not change +operation status. | Status | Meaning | | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjs index 13dfa916c7..dbdfdc3402 100644 --- a/scripts/verify/javascript/output.mjs +++ b/scripts/verify/javascript/output.mjs @@ -54,15 +54,15 @@ try { data: { ...evidence, normalized_result: { payload: [marker] } }, meta: metadata, }; - const skeleton = JSON.stringify(template, null, format === "json" ? 2 : 0); + // The destination is a redirected file, so both formats are compact. + const skeleton = JSON.stringify(template); const encodedMarker = JSON.stringify(marker); const markerIndex = skeleton.indexOf(encodedMarker); assert.ok(markerIndex >= 0); assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker)); const prefix = skeleton.slice(0, markerIndex); const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`; - const indentation = prefix.slice(prefix.lastIndexOf("\n") + 1); - const separator = format === "json" ? `,\n${indentation}` : ","; + const separator = ","; const expectedHash = createHash("sha256"); expectedHash.update(prefix); for (let index = 0; index < count; index += 1) { diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts index b42bd82bdf..f7422a7bf2 100644 --- a/src/cli/jsonOutput.ts +++ b/src/cli/jsonOutput.ts @@ -4,16 +4,27 @@ import { bufferedJsonParts, jsonParts } from "../domain/jsonSerialization.js"; type JsonOutputFormat = "json" | "jsonl"; -/** Write ordinary JSON without constructing a document-sized string. */ +/** + * Write ordinary JSON without constructing a document-sized string. + * + * `json` is indented for a terminal reader. Piped or redirected output, which + * agents and scripts consume, is compact: indentation adds no information and + * measured 16-45% more model tokens on real results. The value is identical. + */ export const writeJsonOutput = async ( value: unknown, destination: Writable, format: JsonOutputFormat = "json", ): Promise => { - for (const part of bufferedJsonParts(jsonOutputParts(value, format))) + for (const part of bufferedJsonParts( + jsonOutputParts(value, format, isTerminal(destination)), + )) await writePart(destination, part); }; +const isTerminal = (destination: Writable): boolean => + "isTTY" in destination && destination.isTTY === true; + const writePart = (destination: Writable, part: string): Promise => new Promise((resolve, reject) => { const onError = (cause: unknown): void => reject(cause); @@ -39,6 +50,7 @@ const writePart = (destination: Writable, part: string): Promise => export function* jsonOutputParts( value: unknown, format: JsonOutputFormat, + pretty = true, ): Generator { if (format === "jsonl" && Array.isArray(value)) { for (const item of value) { @@ -47,6 +59,6 @@ export function* jsonOutputParts( } return; } - yield* jsonParts(value, format === "json"); + yield* jsonParts(value, pretty && format === "json"); yield "\n"; } diff --git a/tests/acceptance/tools/historicalCaptureResources.test.ts b/tests/acceptance/tools/historicalCaptureResources.test.ts index 7aea0fcce2..87e509e652 100644 --- a/tests/acceptance/tools/historicalCaptureResources.test.ts +++ b/tests/acceptance/tools/historicalCaptureResources.test.ts @@ -83,18 +83,22 @@ it("reports real HAR heap exhaustion through CLI and MCP and preserves server us }); expect(JSON.stringify(response)).not.toContain("caller-marked"); expect(JSON.stringify(response)).not.toContain("rea doctor"); - await expect( - execute(process.execPath, [ - cli, - "inspect-web-network-capture", - path, - "har", - "--json", - ]), - ).rejects.toMatchObject({ - code: 1, - stdout: expect.stringContaining('"code": "resource_constraint"'), - }); + const failed: unknown = await execute(process.execPath, [ + cli, + "inspect-web-network-capture", + path, + "har", + "--json", + ]).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failed).toMatchObject({ code: 1 }); + const stdout = + typeof failed === "object" && failed !== null && "stdout" in failed + ? String(failed.stdout) + : ""; + expect(JSON.parse(stdout)).toMatchObject({ code: "resource_constraint" }); const small = join(root, "small.har"); await writeFile(small, capture(2)); await client.ping(); diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 0294073885..fd03f0e920 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -16,6 +16,7 @@ import { silentLogger } from "../../../src/logger.js"; const collect = async ( value: unknown, format: "json" | "jsonl" = "json", + terminal = true, ): Promise => { const chunks: Buffer[] = []; const destination = new Writable({ @@ -28,6 +29,7 @@ const collect = async ( callback(); }, }); + if (terminal) Object.defineProperty(destination, "isTTY", { value: true }); await writeJsonOutput(value, destination, format); return Buffer.concat(chunks).toString("utf8"); }; @@ -52,6 +54,16 @@ describe("incremental CLI JSON", () => { expect(await collect(value)).toBe(`${JSON.stringify(value, null, 2)}\n`); }); + it("indents JSON only for a terminal and writes piped JSON compactly", async () => { + const value = { nested: [{ value: null }], text: "雪𝟠" }; + expect(await collect(value, "json", true)).toBe( + `${JSON.stringify(value, null, 2)}\n`, + ); + expect(await collect(value, "json", false)).toBe( + `${JSON.stringify(value)}\n`, + ); + }); + it("writes compact JSONL records and preserves an empty record set", async () => { const value = [{ text: "first\nrecord" }, { text: "雪𝟠" }]; expect(await collect(value, "jsonl")).toBe( @@ -78,6 +90,7 @@ describe("incremental CLI JSON", () => { }, 1); }, }); + Object.defineProperty(destination, "isTTY", { value: true }); await writeJsonOutput({ leaf }, destination); expect(largestChunk).toBeLessThan(Buffer.byteLength(leaf)); const actual = Buffer.concat(chunks).toString("utf8"); @@ -212,6 +225,8 @@ describe("streamed CLI result surface", () => { expect(operation).toBe("inspect_managed_members"); return value; }); + // A terminal destination exercises the indented encoding past the limit. + Object.defineProperty(destination, "isTTY", { value: true }); const argv = ["inspect-managed-members", "fixture.dll", "--json"]; const output = createStreamedCliJsonOutput(argv, destination); if (output === undefined) throw new Error("Missing JSON output surface"); From 4d8ef3f35022f286e5a6b2ee250cfb336e14a4ad Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:30:42 +0800 Subject: [PATCH 11/74] perf(javascript): aggregate semantic family coverage once (#1410) Reduce repeated semantic relation and unknown family scans while preserving exact complete Evidence and coverage. --- .../JavaScriptSemanticGraphBuilder.ts | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index a951a7a55e..45ded50518 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -175,7 +175,21 @@ export const createJavaScriptSemanticGraphProjection = ): Value => { bindSemanticGraphApplicationNodes(state, applicationGraph); if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); + const relations = [...state.relations.values()]; const unknowns = [...state.unknowns.values()]; + type Family = (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number]; + const retainedByFamily = new Map(); + const unknownIdsByFamily = new Map(); + for (const relation of relations) { + const family = JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation]; + retainedByFamily.set(family, (retainedByFamily.get(family) ?? 0) + 1); + } + for (const unknown of unknowns) { + const identifiers = unknownIdsByFamily.get(unknown.family); + if (identifiers === undefined) + unknownIdsByFamily.set(unknown.family, [unknown.unknown_id]); + else identifiers.push(unknown.unknown_id); + } const graph = factory({ schema: "JavaScriptSemanticRelationGraph", root_artifact_sha256: rootArtifactSha256, @@ -183,7 +197,7 @@ export const createJavaScriptSemanticGraphProjection = root_node_ids: [...state.roots], evidence_contexts: state.evidenceContexts.contexts, nodes: [...state.nodes.values()], - relations: [...state.relations.values()], + relations, fingerprints, unknowns, coverage: { @@ -204,15 +218,9 @@ export const createJavaScriptSemanticGraphProjection = families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ family, status: semanticFamilyStatus(family), - retained_relations: [...state.relations.values()].filter( - (relation) => - JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation] === - family, - ).length, + retained_relations: retainedByFamily.get(family) ?? 0, omitted_relations: truncatedFiles > 0 ? null : 0, - unknown_ids: unknowns - .filter((unknown) => unknown.family === family) - .map(({ unknown_id: identifier }) => identifier), + unknown_ids: unknownIdsByFamily.get(family) ?? [], })), }, limitations: [ From 81cdafc132c7ba3bd6e12b1169f9e3687025d07b Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:45:42 +0800 Subject: [PATCH 12/74] =?UTF-8?q?docs:=20celebrate=2050,000=20GitHub=20sta?= =?UTF-8?q?rs=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 2 +- README_es.md | 2 +- README_fa.md | 2 +- README_fr.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 2 +- README_pt-BR.md | 2 +- README_ru.md | 2 +- README_th.md | 2 +- README_tr.md | 2 +- README_uk.md | 2 +- README_vi.md | 2 +- README_zh-TW.md | 2 +- README_zh.md | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7509d81ea7..e345a56f11 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ Report vulnerabilities through [SECURITY.md](SECURITY.md). ## Star history -🎉 **40,000 GitHub stars — thank you!** +🎉 **50,000 GitHub stars — thank you!** Thanks to everyone using REA, reporting bugs, requesting features, testing builds, and contributing fixes. diff --git a/README_ar.md b/README_ar.md index 92151b9c7e..43bb3adc93 100644 --- a/README_ar.md +++ b/README_ar.md @@ -258,7 +258,7 @@ npx rea-agents@latest setup ## سجل النجوم -🎉 **40,000 نجمة على GitHub — شكرًا لكم!** +🎉 **50,000 نجمة على GitHub — شكرًا لكم!** شكرًا لكل من يستخدم REA، ويبلّغ عن الأخطاء، ويقترح ميزات، ويختبر إصدارات البناء، ويساهم بالإصلاحات. diff --git a/README_de.md b/README_de.md index b71795bc0c..776aa649d1 100644 --- a/README_de.md +++ b/README_de.md @@ -256,7 +256,7 @@ Melde Sicherheitslücken gemäß [SECURITY.md](SECURITY.md). ## Sternverlauf -🎉 **40.000 GitHub-Sterne – vielen Dank!** +🎉 **50.000 GitHub-Sterne – vielen Dank!** Danke an alle, die REA nutzen, Fehler melden, Funktionswünsche äußern, Builds testen und Fehlerbehebungen beitragen. diff --git a/README_es.md b/README_es.md index ac86b1f4db..1dafea412d 100644 --- a/README_es.md +++ b/README_es.md @@ -256,7 +256,7 @@ Informa de vulnerabilidades siguiendo [SECURITY.md](SECURITY.md). ## Historial de estrellas -🎉 **40.000 estrellas en GitHub: ¡gracias!** +🎉 **50.000 estrellas en GitHub: ¡gracias!** Gracias a quienes usan REA, informan de errores, solicitan funciones, prueban compilaciones y contribuyen con correcciones. diff --git a/README_fa.md b/README_fa.md index 7b2b307d97..f7c70ca1d4 100644 --- a/README_fa.md +++ b/README_fa.md @@ -241,7 +241,7 @@ npx rea-agents@latest setup ## تاریخچهٔ ستاره‌ها -🎉 **۴۰٬۰۰۰ ستارهٔ GitHub — سپاسگزاریم!** +🎉 **۵۰٬۰۰۰ ستارهٔ GitHub — سپاسگزاریم!** از همهٔ کسانی که از REA استفاده می‌کنند، خطاها را گزارش می‌دهند، قابلیت‌های جدید پیشنهاد می‌کنند، buildها را آزمایش می‌کنند و در بهبود پروژه مشارکت دارند سپاسگزاریم. diff --git a/README_fr.md b/README_fr.md index 8fa07803b9..00f22dfd9b 100644 --- a/README_fr.md +++ b/README_fr.md @@ -257,7 +257,7 @@ Signalez les vulnérabilités via [SECURITY.md](SECURITY.md). ## Historique des étoiles -🎉 **40 000 étoiles GitHub — merci !** +🎉 **50 000 étoiles GitHub — merci !** Merci à toutes les personnes qui utilisent REA, signalent des bugs, proposent des fonctionnalités, testent les builds et contribuent des correctifs. diff --git a/README_ja.md b/README_ja.md index b827c3c040..c100c1f16d 100644 --- a/README_ja.md +++ b/README_ja.md @@ -258,7 +258,7 @@ npx rea-agents@latest setup ## スター履歴 -🎉 **GitHub スター 40,000 件、ありがとうございます！** +🎉 **GitHub スター 50,000 件、ありがとうございます！** REA の利用、バグ報告、機能の要望、ビルドのテスト、修正への貢献に感謝します。 diff --git a/README_ko.md b/README_ko.md index 8767920595..9dda2ee1bd 100644 --- a/README_ko.md +++ b/README_ko.md @@ -258,7 +258,7 @@ npx rea-agents@latest setup ## 스타 기록 -🎉 **GitHub 스타 40,000개, 감사합니다!** +🎉 **GitHub 스타 50,000개, 감사합니다!** REA를 사용하고, 버그를 보고하고, 기능을 요청하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. diff --git a/README_pl.md b/README_pl.md index bdd062da9b..f234fe0341 100644 --- a/README_pl.md +++ b/README_pl.md @@ -256,7 +256,7 @@ Zgłaszaj podatności zgodnie z [SECURITY.md](SECURITY.md). ## Historia gwiazdek -🎉 **40 000 gwiazdek na GitHubie — dziękujemy!** +🎉 **50 000 gwiazdek na GitHubie — dziękujemy!** Dziękujemy wszystkim, którzy używają REA, zgłaszają błędy, proponują nowe funkcje, testują kompilacje i przesyłają poprawki. diff --git a/README_pt-BR.md b/README_pt-BR.md index 711f7d7881..a788e29858 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -256,7 +256,7 @@ Relate vulnerabilidades conforme [SECURITY.md](SECURITY.md). ## Histórico de estrelas -🎉 **40.000 estrelas no GitHub — muito obrigado!** +🎉 **50.000 estrelas no GitHub — muito obrigado!** Obrigado a todos que usam o REA, relatam bugs, sugerem funcionalidades, testam builds e contribuem com correções. diff --git a/README_ru.md b/README_ru.md index 6b44940e96..cc845a6227 100644 --- a/README_ru.md +++ b/README_ru.md @@ -256,7 +256,7 @@ npx rea-agents@latest setup ## История звёзд -🎉 **40 000 звёзд на GitHub — спасибо!** +🎉 **50 000 звёзд на GitHub — спасибо!** Спасибо всем, кто использует REA, сообщает об ошибках, предлагает новые функции, тестирует сборки и помогает с исправлениями. diff --git a/README_th.md b/README_th.md index c66e1a3932..2dd8b83e5e 100644 --- a/README_th.md +++ b/README_th.md @@ -256,7 +256,7 @@ npx rea-agents@latest setup ## ประวัติดาว -🎉 **40,000 ดาวบน GitHub — ขอบคุณทุกคน!** +🎉 **50,000 ดาวบน GitHub — ขอบคุณทุกคน!** ขอบคุณทุกคนที่ใช้ REA รายงานบั๊ก เสนอฟีเจอร์ ทดสอบบิลด์ และช่วยแก้ไข diff --git a/README_tr.md b/README_tr.md index ff5ae32399..6bf9ffd0da 100644 --- a/README_tr.md +++ b/README_tr.md @@ -256,7 +256,7 @@ Güvenlik açıklarını [SECURITY.md](SECURITY.md) üzerinden bildirin. ## Yıldız geçmişi -🎉 **GitHub'da 40.000 yıldız — teşekkürler!** +🎉 **GitHub'da 50.000 yıldız — teşekkürler!** REA'yı kullanan, hata bildiren, özellik talep eden, derlemeleri test eden ve düzeltmelere katkıda bulunan herkese teşekkürler. diff --git a/README_uk.md b/README_uk.md index 973ff9b417..c59b27becc 100644 --- a/README_uk.md +++ b/README_uk.md @@ -256,7 +256,7 @@ npx rea-agents@latest setup ## Історія зірок -🎉 **40 000 зірок на GitHub — дякуємо!** +🎉 **50 000 зірок на GitHub — дякуємо!** Дякуємо всім, хто використовує REA, повідомляє про помилки, пропонує нові функції, тестує збірки й допомагає з виправленнями. diff --git a/README_vi.md b/README_vi.md index 6f198f90fc..cb33dd266c 100644 --- a/README_vi.md +++ b/README_vi.md @@ -256,7 +256,7 @@ Báo cáo lỗ hổng theo [SECURITY.md](SECURITY.md). ## Lịch sử sao -🎉 **40.000 sao trên GitHub — xin cảm ơn!** +🎉 **50.000 sao trên GitHub — xin cảm ơn!** Cảm ơn mọi người đã sử dụng REA, báo lỗi, đề xuất tính năng, kiểm thử bản dựng và đóng góp bản sửa lỗi. diff --git a/README_zh-TW.md b/README_zh-TW.md index adef7444c9..ec44d4c3e0 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -258,7 +258,7 @@ npx rea-agents@latest setup ## Star 歷史 -🎉 **GitHub Star 達到 40,000 個，感謝大家！** +🎉 **GitHub Star 達到 50,000 個，感謝大家！** 感謝每一位使用 REA、報告錯誤、提出功能需求、測試建置和貢獻修復的朋友。 diff --git a/README_zh.md b/README_zh.md index baf67a1451..6dca111fb6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -258,7 +258,7 @@ npx rea-agents@latest setup ## Star 历史 -🎉 **GitHub Star 达到 40,000 个，感谢大家！** +🎉 **GitHub Star 达到 50,000 个，感谢大家！** 感谢每一位使用 REA、报告错误、提出功能需求、测试构建和贡献修复的朋友。 From 98022a835275375ee0ffa88b7c8983ba6f080040 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:02:24 +0800 Subject: [PATCH 13/74] fix(javascript)!: avoid endpoint payload copies in graph metadata (#1423) Keep exact endpoint observations and request properties while using compact display labels and collision-safe canonical value digests for long artifact-local endpoint keys. Verify real CLI/stdio MCP tracing and version comparison without truncating evidence. BREAKING CHANGE: New analyses use compact endpoint labels and digest-based IDs for long or reserved-prefix endpoint keys. Select IDs from the parent graph and read full values from endpoint properties. Previously stored graphs retain their original identities and remain valid workflow inputs. --- docs/javascript-application-workflows.md | 14 + .../JavaScriptArtifactGraphFindings.ts | 19 +- .../JavaScriptSemanticGraphDataProjection.ts | 6 +- .../javascriptEndpointPayload.test.ts | 263 ++++++++++++++++++ 4 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 tests/acceptance/applications/javascriptEndpointPayload.test.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 0093710351..592276e15e 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -104,6 +104,20 @@ graph and read literal values from `properties.value`, rather than decoding role keys or labels. Previously stored graphs and their original IDs remain valid inputs. +Endpoint nodes keep the exact endpoint in each observation's `properties.value`; +semantic request and response nodes keep it in `properties.endpoint`. Endpoint +identity keys use `value-sha256:` over the canonical JSON string when +that is shorter, or when the original value starts with the reserved digest +prefix. Other short endpoint keys remain unchanged. Endpoint labels use the +shorter of the exact value and ` endpoint`; semantic request/response +labels use the shorter of the endpoint and operation method. These labels are +display metadata, not endpoint lookup values. + +New analyses therefore change hashed endpoint IDs and longer endpoint labels. +Use IDs from the selected graph and read the complete endpoint properties; +previously stored graphs still support their original IDs. Literal endpoint +search, semantic tracing, and version comparison use the preserved values. + ## Version comparison REA pairs entities only when a tier produces one unique candidate on each side. diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index cff140220a..580a4d0355 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; +import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js"; import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -162,16 +163,22 @@ const addEndpoint = ( const source = sourceNodeFor(context, input.file.path, input.value.module_key) ?? input.asset; + const digestPrefix = "value-sha256:"; + const valueKey = `${digestPrefix}${digestCanonicalValue(input.value.value, "JavaScript endpoint")}`; + // Reserve the digest prefix so an exact caller value cannot alias a digest key. + const key = + input.value.value.length <= valueKey.length && + !input.value.value.startsWith(digestPrefix) + ? input.value.value + : valueKey; + const label = `${input.value.kind} endpoint`; const node = context.accumulator.addNode({ kind: "endpoint", - identity: artifactLocalIdentity( - input.file.sha256, - input.value.kind, - input.value.value, - ), + identity: artifactLocalIdentity(input.file.sha256, input.value.kind, key), observations: [ { - label: input.value.value, + label: + input.value.value.length <= label.length ? input.value.value : label, properties: { endpoint_kind: input.value.kind, value: input.value.value, diff --git a/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts b/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts index 51efcd2ec6..b1c44e8f8c 100644 --- a/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts @@ -137,7 +137,11 @@ const addRequestNode = ( kind: operation.kind === "request" ? "request" : "response", roleKey: operation.requestId, location: operation.location, - label: operation.endpoint ?? operation.method, + label: + operation.endpoint !== null && + operation.endpoint.length <= operation.method.length + ? operation.endpoint + : operation.method, functionNodeId: operation.ownerCallableId === null ? null diff --git a/tests/acceptance/applications/javascriptEndpointPayload.test.ts b/tests/acceptance/applications/javascriptEndpointPayload.test.ts new file mode 100644 index 0000000000..6a980aa7ad --- /dev/null +++ b/tests/acceptance/applications/javascriptEndpointPayload.test.ts @@ -0,0 +1,263 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { applicationVersionComparisonResultSchema } from "../../../src/domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; +import { applicationFeatureTraceResultSchema } from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "keeps full endpoint evidence usable without copying large values into metadata", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-endpoint-payload-"); + const endpoint = `https://example.invalid/${"part/ğ".repeat(160_000)}`; + const digestLookalike = `value-sha256:${createHash("sha256").update(JSON.stringify(endpoint)).digest("hex")}`; + // UTF-8 replacement would conflate these distinct JavaScript strings. + const surrogate = "\ud800".repeat(100); + const replacement = "\ufffd".repeat(100); + const values = [ + endpoint, + digestLookalike, + "/ok", + "/ok", + surrogate, + replacement, + ]; + const source = values + .map((value) => `fetch(${JSON.stringify(value)});`) + .join("\n"); + const sourcePath = join(root, "main.js"); + await writeFile(sourcePath, source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + timeoutMs: 60_000, + }); + expect(cliResponse.exitCode).toBe(0); + const cliEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(cliResponse.json); + + const client = new Client({ name: "endpoint-payload-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + const textEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(text.text)); + expect(textEvidence.evidence_id).toBe(evidence.evidence_id); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + const endpoints = result.graph.nodes.filter( + (node) => node.kind === "endpoint", + ); + const nodeFor = (value: string) => { + const node = endpoints.find((candidate) => + candidate.observations.some( + ({ properties }) => properties.value === value, + ), + ); + if (node === undefined) throw new Error("Complete endpoint was lost"); + return node; + }; + const endpointNode = nodeFor(endpoint); + expect( + new Set( + [endpoint, digestLookalike, surrogate, replacement].map( + (value) => nodeFor(value).node_id, + ), + ).size, + ).toBe(4); + const repeatedNode = nodeFor("/ok"); + expect( + endpoints.filter((node) => + node.observations.some( + ({ properties }) => properties.value === "/ok", + ), + ), + ).toHaveLength(1); + expect(repeatedNode.observations).toHaveLength(2); + expect(repeatedNode.identity).toMatchObject({ key: "/ok" }); + expect( + repeatedNode.observations.every(({ label }) => label === "/ok"), + ).toBe(true); + expect(endpointNode.identity).toMatchObject({ + artifact_sha256: createHash("sha256").update(source).digest("hex"), + }); + expect(endpointNode.observations[0]?.evidence.location).toMatchObject({ + available: true, + value: { kind: "source-range", source: "main.js" }, + }); + const requests = result.semantic_graph.nodes.filter( + (node) => + node.kind === "request" && node.properties.endpoint === endpoint, + ); + expect(requests).toHaveLength(1); + + await verifyEndpointWorkflows({ + client, + evidenceId: evidence.evidence_id, + endpoint, + endpointNodeId: endpointNode.node_id, + requestNodeIds: requests.map(({ node_id }) => node_id), + root, + source, + sourcePath, + }); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } + }, + 120_000, +); + +interface EndpointWorkflowInput { + readonly client: Client; + readonly evidenceId: string; + readonly endpoint: string; + readonly endpointNodeId: string; + readonly requestNodeIds: readonly string[]; + readonly root: string; + readonly source: string; + readonly sourcePath: string; +} + +const verifyEndpointWorkflows = async ({ + client, + evidenceId, + endpoint, + endpointNodeId, + requestNodeIds, + root, + source, + sourcePath, +}: EndpointWorkflowInput): Promise => { + const application = { + kind: "retained-evidence", + evidence_id: evidenceId, + }; + const featureResponse = await client.callTool({ + name: "trace_application_feature", + arguments: { + application, + seed: { + kind: "string", + value: endpoint, + match: "exact", + case_sensitive: true, + }, + }, + }); + expect(featureResponse.isError).not.toBe(true); + const featureEvidence = toolContract( + "trace_application_feature", + ).outputSchema.parse(featureResponse.structuredContent); + const feature = applicationFeatureTraceResultSchema.parse( + featureEvidence.normalized_result, + ); + expect(feature.seed_matches.map(({ node_id }) => node_id)).toContain( + endpointNodeId, + ); + + const traceResponse = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application, + query: { + seed: { kind: "endpoint", value: endpoint }, + direction: "backward-provenance", + }, + }, + }); + expect(traceResponse.isError).not.toBe(true); + const traceEvidence = toolContract( + "trace_javascript_semantics", + ).outputSchema.parse(traceResponse.structuredContent); + const trace = javaScriptSemanticTraceResultSchema.parse( + traceEvidence.normalized_result, + ); + expect(trace.seed_node_ids).toEqual(requestNodeIds); + expect( + trace.nodes.some((node) => node.properties.endpoint === endpoint), + ).toBe(true); + + await writeFile(sourcePath, `// next artifact version\n${source}`); + const nextResponse = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(nextResponse.isError).not.toBe(true); + const nextEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(nextResponse.structuredContent); + const next = javascriptApplicationAnalysisResultSchema.parse( + nextEvidence.normalized_result, + ); + const nextEndpoint = next.graph.nodes.find( + (node) => + node.kind === "endpoint" && + node.observations.some(({ properties }) => properties.value === endpoint), + ); + if (nextEndpoint === undefined) throw new Error("Next endpoint was lost"); + const comparisonResponse = await client.callTool({ + name: "compare_application_versions", + arguments: { + left: application, + right: { + kind: "retained-evidence", + evidence_id: nextEvidence.evidence_id, + }, + }, + }); + expect(comparisonResponse.isError).not.toBe(true); + const comparisonEvidence = toolContract( + "compare_application_versions", + ).outputSchema.parse(comparisonResponse.structuredContent); + const comparison = applicationVersionComparisonResultSchema.parse( + comparisonEvidence.normalized_result, + ); + expect(comparison.items).toContainEqual( + expect.objectContaining({ + node_kind: "endpoint", + left_node_id: endpointNodeId, + right_node_id: nextEndpoint.node_id, + match: expect.objectContaining({ + status: "matched", + basis: "semantic-key", + }), + }), + ); +}; From 28595276faac6a01dd4930313a91fab717d29513 Mon Sep 17 00:00:00 2001 From: Marnix Pluim Date: Sat, 10 Oct 2026 06:03:13 +0200 Subject: [PATCH 14/74] fix(javascript): bound member-mutation paths through reassignment loops Fixes #1401. Refs #1065. Preserve exact member facts and equal-depth write effects while bounding the observed reassignment-loop expansion. Validated with the JavaScript semantic/application suite, focused alias regressions, full static checks, real compiled CLI and SDK MCP Notion workflows, and all 20 checks on the final PR head. Co-Authored-By: Marnix Pluim <286664873+marnixpluim@users.noreply.github.com> Co-Authored-By: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Co-Authored-By: Claude --- ...manticAnalysis.reference_lifetimes.test.ts | 157 ++++++++++ .../javascriptSemanticMemberMutations.ts | 270 +++++++++++++++++- .../javascriptSemanticPropertyPaths.ts | 86 ++++++ .../javascript/javascriptSemanticValues.ts | 25 +- 4 files changed, 524 insertions(+), 14 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts index 2795427bd4..73c7897b0f 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts @@ -9,6 +9,163 @@ const resultValue = (body: string) => "result", ).returnSites[0]?.value; +describe("bounded member path propagation", () => { + it("bounds cyclic member paths with no observed container origin", () => { + const branches = Array.from( + { length: 16 }, + (_, index) => `if (flag === ${index}) node = node.m${index};`, + ).join(" else "); + const start = performance.now(); + const callable = onlyCallable( + analyzeJavaScriptSemantics( + `export function result(seed) { let node=seed; while(node) { ${branches} use(node.state); } return seed.keep; }`, + ), + "result", + ); + expect(callable.returnSites[0]?.value.status).toBe("unknown"); + expect(performance.now() - start).toBeLessThan(2000); + }, 30000); + + it("preserves Array prototype effects through an otherwise unknown alias", () => { + expect( + resultValue( + "const values=[1]; const prototype=Array.prototype; prototype[Symbol.iterator]=custom; for(const item of values) { mutate(item); } return values[0];", + )?.status, + ).toBe("unknown"); + }); + + it("bounds loops derived from a scalar member while preserving sibling facts", () => { + const branches = Array.from( + { length: 16 }, + (_, index) => `if (flag === ${index}) node = node.m${index};`, + ).join(" else "); + const start = performance.now(); + expect( + resultValue( + `const source={child:1,keep:{value:3}}; let node=source.child; while(node) { ${branches} use(node.state); } return source.keep.value;`, + ), + ).toEqual({ status: "literal", value: 3 }); + expect(performance.now() - start).toBeLessThan(2000); + }, 30000); + + it.each([ + ["child:external,", "child", "present"], + ["", "missing", "unknown-coverage"], + ])( + "retains slot presence below an unobserved container: %s", + (property, name, presence) => { + expect( + resultValue( + `const source={${property}keep:3}; const alias=source.${name}; alias.deep.value=2; return source;`, + ), + ).toMatchObject({ + status: "object", + properties: expect.arrayContaining([ + { + name, + presence, + value: { + status: "unknown", + reason: "This property may have been mutated.", + }, + }, + { + name: "keep", + presence: "present", + value: { status: "literal", value: 3 }, + }, + ]), + }); + }, + ); + + it.each([ + ["use(node);", "m0.value", "unknown"], + ["node.value = 2;", "m0.value", "unknown"], + ["node.value = 2;", "mode", "literal"], + ])( + "bounds a loop that reassigns one reference through many members: %s seed.%s", + (effect, read, status) => { + const branches = Array.from( + { length: 9 }, + (_, index) => `if (flag === ${index}) node = node.m${index};`, + ).join(" else "); + const start = performance.now(); + const value = resultValue( + `const seed = { mode: "initial", m0: { value: 1 } }; let node = seed; while (node) { ${branches} ${effect} } return seed.${read};`, + ); + expect(performance.now() - start).toBeLessThan(2000); + expect(value?.status).toBe(status); + }, + 30000, + ); + + it.each([ + ["const alias = flag ? owner.child : owner; alias.value = 2;", "value"], + ["const alias = flag ? owner : owner.child; alias.value = 2;", "value"], + ["const alias = flag ? owner.child : owner; alias[key] = 2;", "value"], + [ + "const alias = flag ? owner.child : owner; alias.child = 2;", + "child.value", + ], + [ + "let alias = owner; let i = 0; while (i++ < 1) { alias = flag ? alias.child : alias; alias.value = 2; }", + "value", + ], + ])( + "keeps a write that reaches a child through a longer path: %s", + (body, read) => { + expect( + resultValue( + `const shared = { value: 1, child: { value: 1 } }; const owner = { child: shared }; ${body} return shared.${read};`, + )?.status, + ).toBe("unknown"); + }, + ); + + it.each([ + "mutate(source.left, source.right);", + "const alias = flag ? source.left : source.right; mutate(alias);", + "const copy = flag ? [source.left] : [source.right]; mutate(copy);", + "const alias = flag ? source.left : source.right; alias.value = 2;", + ])("keeps members outside the mutated paths: %s", (body) => { + expect( + resultValue( + `const source = { left: { value: 1 }, right: { value: 2 }, keep: { value: 3 } }; ${body} return source.keep.value;`, + ), + ).toEqual({ status: "literal", value: 3 }); + }); +}); + +describe("distinct observable mutation paths", () => { + it.each([ + ["alias.value = 2;", "left"], + ["alias.value = 2;", "right"], + ["mutate(alias);", "left"], + ["mutate(alias);", "right"], + ])( + "propagates both conditional paths to shared bindings: %s %s", + (effect, target) => { + expect( + resultValue( + `const left={value:1}; const right={value:2}; const source={left,right}; const alias=flag?source.left:source.right; ${effect} return ${target}.value;`, + )?.status, + ).toBe("unknown"); + }, + ); + + it.each(["mutate(alias);", "alias.value = 2;"])( + "preserves sibling facts below a shared prefix: %s", + (effect) => { + expect( + resultValue( + `const shared={left:{value:1},right:{value:2},keep:{value:3}}; const source={nested:shared}; const alias=flag?source.nested.left:source.nested.right; ${effect} return shared.keep.value;`, + ), + ).toEqual({ status: "literal", value: 3 }); + }, + ); +}); + describe("reference lifetimes across calls and shallow copies", () => { it("bounds dynamic destructuring across shared alias branches", () => { const declarations = Array.from( diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index ae887b9c79..21a6aded04 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -18,6 +18,7 @@ import { semanticIterationSources } from "./javascriptSemanticIterationSources.j import { semanticMutationInitializers } from "./javascriptSemanticMutationInitializers.js"; import { semanticArrayIndex, + SemanticPropertyPathCoverage, semanticPropertyPathKeyMatches, type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; @@ -42,8 +43,10 @@ export const collectSemanticMemberMutations = ( if (parent !== null) parents.set(node, parent); }, }); + const normalizePath = observableMutationPaths(state); const recordedEffects = new Set(); const selections = new Map(); + const coveringPaths = new Map(); let arrayIterationUnknown = false; const pendingReferences: { readonly initializer: JavaScriptSemanticBindingState["referenceInitializers"][number]; @@ -93,8 +96,10 @@ export const collectSemanticMemberMutations = ( originAt: t.Node | undefined = mutation, ): void => { const pending = [{ node, path, bindings, originAt }]; - while (pending.length > 0) { - const current = pending.pop(); + // Visit shallower alias traversals first so their exact prefixes suppress + // longer loop paths before those paths enumerate every assignment order. + for (let cursor = 0; cursor < pending.length; cursor++) { + const current = pending[cursor]; if (current === undefined) break; const expression = unwrapJavaScriptExpression(current.node).node; if (t.isIdentifier(expression)) { @@ -126,10 +131,13 @@ export const collectSemanticMemberMutations = ( if (current.bindings.has(lifetime)) continue; // Union compatible copied-slot selections at each binding. Carrying every // combination of exclusions through branch diamonds is exponential. - const selectorIndex = current.path.findIndex( + // Preserve paths that can change observed slots or Array prototype + // effects; canonicalize suffixes that cannot distinguish lattice facts. + let path = normalizePath(binding.bindingId, current.path, effect); + if (path === null) continue; + const selectorIndex = path.findIndex( (key) => key !== null && typeof key === "object", ); - let path = current.path; const selector = path[selectorIndex]; if (selector !== null && typeof selector === "object") { const selectionIdentity = JSON.stringify([ @@ -157,6 +165,25 @@ export const collectSemanticMemberMutations = ( index === selectorIndex ? { ...selector, excludedKeys } : key, ); } + // Keep an antichain of exact paths. Widening distinct member keys would + // also invalidate unrelated siblings that never reach this effect. + const pathState = JSON.stringify([ + binding.bindingId, + effect, + // Short writes stop at owned literal slots, whereas longer writes + // reach shared children. Only equal-depth writes cover one another. + effect === "write" ? path.length : null, + mutation?.start, + mutation?.end, + current.originAt?.start, + current.originAt?.end, + ]); + let covering = coveringPaths.get(pathState); + if (covering === undefined) { + covering = new SemanticPropertyPathCoverage(); + coveringPaths.set(pathState, covering); + } + if (!covering.retain(path)) continue; const identity = JSON.stringify([ binding.bindingId, effect, @@ -544,6 +571,241 @@ const referencedValues = ( return []; }; +type MutationObservableOrigin = + | t.ObjectExpression + | t.ArrayExpression + | "ambient" + | "array" + | "array-prototype" + | "opaque"; + +interface MutationObserverNode { + readonly origins: Set; + readonly sources: Set; + readonly dependents: Set; + readonly projections: Map; + readonly read?: { + readonly source: MutationObserverNode; + readonly key: PropertyPath[number]; + }; +} + +const observableMutationPaths = ( + state: JavaScriptSemanticAnalysisState, +): (( + bindingId: string, + path: PropertyPath, + effect: ValueEffect, +) => PropertyPath | null) => { + const bindingNodes = new Map(); + const expressions = new WeakMap(); + const pendingExpressions: { + readonly expression: t.Node; + readonly observer: MutationObserverNode; + }[] = []; + const pending: MutationObserverNode[] = []; + const queued = new Set(); + const schedule = (node: MutationObserverNode): void => { + if (queued.has(node)) return; + queued.add(node); + pending.push(node); + }; + const create = ( + read?: MutationObserverNode["read"], + ): MutationObserverNode => ({ + origins: new Set(), + sources: new Set(), + dependents: new Set(), + projections: new Map(), + ...(read === undefined ? {} : { read }), + }); + const link = ( + source: MutationObserverNode, + target: MutationObserverNode, + ): void => { + if (target.sources.has(source)) return; + target.sources.add(source); + source.dependents.add(target); + schedule(target); + }; + const expressionNode = (node: t.Node): MutationObserverNode => { + const expression = unwrapJavaScriptExpression(node).node; + const existing = expressions.get(expression); + if (existing !== undefined) return existing; + const observer = create(); + expressions.set(expression, observer); + pendingExpressions.push({ expression, observer }); + return observer; + }; + const project = ( + source: MutationObserverNode, + path: PropertyPath, + ): MutationObserverNode => { + let observer = source; + for (const key of path) { + const identity = JSON.stringify( + key === null || typeof key === "object" ? key : String(key), + ); + let selected = observer.projections.get(identity); + if (selected === undefined) { + selected = create({ source: observer, key }); + observer.projections.set(identity, selected); + observer.dependents.add(selected); + schedule(selected); + } + observer = selected; + } + return observer; + }; + for (const binding of state.bindingsById.values()) + bindingNodes.set(binding.bindingId, create()); + for (const binding of state.bindingsById.values()) { + const observer = bindingNodes.get(binding.bindingId); + if (observer === undefined) continue; + for (const initializer of binding.initializers) + link( + project(expressionNode(initializer.node), initializer.projection), + observer, + ); + // Preserve exact paths where copies, defaults or iteration need their + // reference-specific projections; an opaque origin prevents pruning. + if (binding.referenceInitializers.length > 0) + observer.origins.add("opaque"); + for (const initializer of binding.referenceInitializers) { + for (const source of [ + initializer, + ...(initializer.requiredSources ?? []), + ...(initializer.fallbackSources ?? []), + ]) + link(expressionNode(source.node), observer); + } + } + const settle = (): void => { + while (pendingExpressions.length > 0 || pending.length > 0) { + while (pendingExpressions.length > 0) { + const selected = pendingExpressions.pop(); + if (selected === undefined) break; + const { expression, observer } = selected; + if (t.isObjectExpression(expression) || t.isArrayExpression(expression)) + observer.origins.add(expression); + else if (t.isIdentifier(expression)) { + const binding = resolveSemanticBindingState( + state, + expression, + expression.name, + ); + if (binding !== undefined) { + const origin = bindingNodes.get(binding.bindingId); + if (origin !== undefined) link(origin, observer); + } else if (expression.name === "Array") observer.origins.add("array"); + else if ( + ["globalThis", "global", "window", "self"].includes(expression.name) + ) + observer.origins.add("ambient"); + } else if ( + t.isMemberExpression(expression) || + t.isOptionalMemberExpression(expression) + ) + link( + project(expressionNode(expression.object), [ + semanticStaticPropertyKey( + expression.property, + expression.computed, + ), + ]), + observer, + ); + else + for (const reference of referencedValues(expression, [], "escape")) + link( + project(expressionNode(reference.node), reference.path), + observer, + ); + schedule(observer); + } + const observer = pending.pop(); + if (observer === undefined) continue; + queued.delete(observer); + let changed = false; + const add = (origin: MutationObservableOrigin): void => { + if (observer.origins.has(origin)) return; + observer.origins.add(origin); + changed = true; + }; + for (const source of observer.sources) + for (const origin of source.origins) add(origin); + if (observer.read !== undefined) { + const { source, key } = observer.read; + if (key === null || typeof key === "object") { + // Dynamic keys and copied-slot masks retain their original paths. + // Enumerating their exclusion combinations would recreate the growth + // that the mutation collector's selection union already prevents. + if (source.origins.size > 0) add("opaque"); + } else + for (const origin of source.origins) { + if (typeof origin === "string") { + if (origin === "array-prototype" || origin === "opaque") + add(origin); + else if ( + origin === "ambient" && + semanticPropertyPathKeyMatches(key, "Array") + ) + add("array"); + else if ( + origin === "array" && + semanticPropertyPathKeyMatches(key, "prototype") + ) + add("array-prototype"); + } else { + for (const reference of referencedValues( + origin, + [key], + "escape", + )) { + link( + project(expressionNode(reference.node), reference.path), + observer, + ); + if (reference.iterableFallbackPath !== undefined) + link( + project( + expressionNode(reference.node), + reference.iterableFallbackPath, + ), + observer, + ); + } + } + } + } + if (changed) + for (const dependent of observer.dependents) schedule(dependent); + } + }; + settle(); + return (bindingId, path, effect) => { + let observer = bindingNodes.get(bindingId); + if (observer === undefined) return path; + if (path.some((key) => key !== null && typeof key === "object")) + return path; + if (effect === "escape") { + observer = project(observer, path); + settle(); + return observer.origins.size === 0 ? null : path; + } + if (observer.origins.size === 0) return path.length === 0 ? path : [null]; + for (let offset = 0; offset < path.length; offset++) { + const key = path[offset]; + if (key === undefined) break; + observer = project(observer, [key]); + settle(); + if (observer.origins.size === 0 && offset + 1 < path.length) + return [...path.slice(0, offset + 1), null]; + } + return path; + }; +}; + const objectReferencedValues = ( node: t.ObjectExpression, path: PropertyPath, diff --git a/src/domain/javascript/javascriptSemanticPropertyPaths.ts b/src/domain/javascript/javascriptSemanticPropertyPaths.ts index 9bee0e9b7a..95cbeef974 100644 --- a/src/domain/javascript/javascriptSemanticPropertyPaths.ts +++ b/src/domain/javascript/javascriptSemanticPropertyPaths.ts @@ -35,3 +35,89 @@ export const semanticPropertyPathKeyMatches = ( const index = semanticArrayIndex(name); return index !== null && index >= key.startIndex; }; + +interface PropertyPathCoverageNode { + terminal: boolean; + readonly exact: Map; + readonly selectors: Map< + string, + { + readonly key: JavaScriptSemanticPropertyPath[number]; + readonly node: PropertyPathCoverageNode; + } + >; +} + +const coverageNode = (): PropertyPathCoverageNode => ({ + terminal: false, + exact: new Map(), + selectors: new Map(), +}); + +/** Index exact effect prefixes without widening distinct member paths. */ +export class SemanticPropertyPathCoverage { + readonly #root = coverageNode(); + + /** Retain a path only when no previously retained prefix covers it. */ + retain(path: JavaScriptSemanticPropertyPath): boolean { + const pending = [{ node: this.#root, offset: 0 }]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined) break; + if (current.node.terminal) return false; + const key = path[current.offset]; + if (key === undefined) continue; + if (key !== null && typeof key !== "object") { + const exact = current.node.exact.get(String(key)); + if (exact !== undefined) + pending.push({ node: exact, offset: current.offset + 1 }); + } + for (const selected of current.node.selectors.values()) { + if (keyCovers(selected.key, key)) + pending.push({ node: selected.node, offset: current.offset + 1 }); + } + } + let node = this.#root; + for (const key of path) { + if (key !== null && typeof key !== "object") { + const name = String(key); + let child = node.exact.get(name); + if (child === undefined) { + child = coverageNode(); + node.exact.set(name, child); + } + node = child; + } else { + const identity = JSON.stringify(key); + let selected = node.selectors.get(identity); + if (selected === undefined) { + selected = { key, node: coverageNode() }; + node.selectors.set(identity, selected); + } + node = selected.node; + } + } + node.terminal = true; + node.exact.clear(); + node.selectors.clear(); + return true; + } +} + +const keyCovers = ( + covering: JavaScriptSemanticPropertyPath[number], + key: JavaScriptSemanticPropertyPath[number], +): boolean => { + if (covering === null) return true; + if (key === null) return false; + if (typeof key !== "object") + return typeof covering === "object" + ? semanticPropertyPathKeyMatches(covering, String(key)) + : String(covering) === String(key); + return ( + typeof covering === "object" && + covering.excludedKeys.every((name) => key.excludedKeys.includes(name)) && + (covering.startIndex === undefined || + (key.startIndex !== undefined && covering.startIndex <= key.startIndex)) + ); +}; diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index 2f53626708..884915948c 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -211,12 +211,17 @@ const evaluateBinding = ( if (isPrimitive(captured)) value = captured; } } - const mutated = binding.mutatedPaths.reduce( - invalidateSemanticMutationPath, - value, - ); - const projected = binding.escapedPaths.reduce( - (value, escape) => + let projected = value; + for (const path of binding.mutatedPaths) { + projected = invalidateSemanticMutationPath(projected, path); + // Every subsequent write returns this same root unknown. Replaying those + // paths at every newly collected effect otherwise becomes quadratic. + if (projected.status === "unknown") break; + } + for (const escape of binding.escapedPaths) { + // Escapes only invalidate object references; other lattice values survive. + if (projected.status !== "object" && projected.status !== "array") break; + if ( context.capturePoint !== undefined && semanticEscapeFollowsCapture( binding, @@ -224,10 +229,10 @@ const evaluateBinding = ( escape.node, context.state.parentsByNode, ) - ? value - : invalidateSemanticEscapedPath(value, escape.path), - mutated, - ); + ) + continue; + projected = invalidateSemanticEscapedPath(projected, escape.path); + } // Point-local memoization also bounds failed speculative captures. Only a // primitive result is allowed back into ordinary value evaluation. if ( From 4d95f2b2e7a6fdc1ba9dfcbd90578f6bbc5e5e1e Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:15:11 +0800 Subject: [PATCH 15/74] fix(validation): retain unmatched union correction details Preserve actionable field paths and schema-declared variants for unknown or missing discriminators. Closes #1421. Refs #1065. --- src/domain/inputIssueProjection.ts | 19 ++++++++- tests/boundary/cli/analysisViewCli.test.ts | 45 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/domain/inputIssueProjection.ts b/src/domain/inputIssueProjection.ts index 25798721c9..1d5db83b65 100644 --- a/src/domain/inputIssueProjection.ts +++ b/src/domain/inputIssueProjection.ts @@ -35,7 +35,23 @@ const projectIssue = ( typeof part === "string" || typeof part === "number" ? [part] : [], ); // Branch issue paths are relative to the union; branches often agree. - if (issue.code === "invalid_union") + if (issue.code === "invalid_union") { + // Unmatched discriminators and ambiguous unions have no branch issues. + // Keep their outer failure instead of flattening it into an empty list. + if (issue.errors.length === 0) { + const expected = + "options" in issue ? issue.options?.filter(isSafeExpected) : undefined; + return [ + { + path, + reason: + valueAtPath(input, path) === undefined + ? "missing_argument" + : "invalid_value", + ...(expected === undefined ? {} : { expected }), + }, + ]; + } return uniqueIssues( issue.errors.flatMap((branch) => branch.flatMap((branchIssue) => @@ -46,6 +62,7 @@ const projectIssue = ( ), ), ); + } if (issue.code === "invalid_key") return issue.issues.flatMap((keyIssue) => projectIssue( diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index e6e6fb51d7..86e0d9b271 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -80,3 +80,48 @@ cliTest("rejects a request that omits source", async ({ cli }) => { details: { operation: "inspect-analysis-view" }, }); }); + +for (const [view, reason] of [ + [{ kind: "unknown-view" }, "invalid_value"], + [{}, "missing_argument"], +] as const) { + cliTest( + `reports correction details for ${reason} view kind`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-view-variant-"); + const input = join(root, "view.json"); + await writeFile( + input, + JSON.stringify({ + source: { + kind: "inline", + evidence: analysisViewLayoutEvidence(), + }, + view, + }), + ); + const result = await cli.run({ + arguments: ["inspect-analysis-view", input, "--json"], + environment: { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["view", "kind"], + reason, + expected: expect.arrayContaining(["summary", "page"]), + }, + ], + }, + }); + expect(JSON.stringify(result.json)).not.toContain("unknown-view"); + }, + ); +} From a9ee2360fd11332b2cc50ef7130a89321b828d28 Mon Sep 17 00:00:00 2001 From: jhuang-tw <77732008+jhuang-tw@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:18:50 +0800 Subject: [PATCH 16/74] feat(native): inspect oversized retained function dossiers via bounded views Add caller-selected native dossier facets with preserved Evidence identity, provenance, unknowns and transport recovery. Refs #1247 and #1065. Co-authored-by: jhuang-tw <77732008+jhuang-tw@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- docs/mcp-contracts.md | 7 +- .../references/native-and-artifacts.md | 8 + .../analysisView/AnalysisViewService.test.ts | 92 ++++++++ .../analysisView/AnalysisViewService.ts | 18 +- src/cli/analysisViewCommands.ts | 2 +- src/contracts/analysisViewToolContracts.ts | 19 +- src/domain/analysisView/analysisView.test.ts | 197 +++++++++++++++-- src/domain/analysisView/analysisView.ts | 63 +++++- src/domain/analysisView/binaryLayoutView.ts | 4 + .../analysisView/javascriptApplicationView.ts | 4 + src/domain/analysisView/nativeFunctionView.ts | 202 ++++++++++++++++++ tests/boundary/cli/analysisViewCli.test.ts | 44 ++++ tests/boundary/mcp/analysisViewMcp.test.ts | 131 +++++++++++- 13 files changed, 769 insertions(+), 22 deletions(-) create mode 100644 src/domain/analysisView/nativeFunctionView.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 5eedd9c14c..7bf2fb76c0 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -237,7 +237,12 @@ Ordinary responses keep their existing complete result contract. ## Retained application Evidence inputs `inspect_analysis_view` projects a caller-selected view of already completed -`inspect_binary_layout` or `analyze_javascript_application` Evidence. Source is +`inspect_binary_layout`, `analyze_javascript_application`, or `analyze_function` +Evidence. Native views select procedure, pseudocode, assembly, callers/callees, +references, and high-pcode facets without starting a provider. Native offset +and limit default to 0 and 64; pseudocode uses UTF-16 code units and never +splits surrogate pairs, while other pages count rows. Unavailable facts and +provider limitations remain explicit. Source is an exact same-session retained reference or portable inline Evidence. Views are a summary, a layout mitigations or linkage facet, one section/symbol/module, or a stable page with a caller-selected positive `limit`. Module pages include diff --git a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md b/skill-src/reverse-engineer-anything/references/native-and-artifacts.md index f47017171d..65b68c6b2d 100644 --- a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md +++ b/skill-src/reverse-engineer-anything/references/native-and-artifacts.md @@ -34,6 +34,14 @@ cross-references. Addresses and recovered pseudocode are analysis observations, not original source. Provider unavailability and unsupported metadata remain unknown rather than false. +For a completed `analyze_function` dossier, use `inspect_analysis_view` with +`view: {kind: "native", facet: "value_flow_operations", offset: 0, limit: 32}` +and the exact retained Evidence ID to read selected facts without reanalysis. +Native facets also include procedure, pseudocode, assembly, callers/callees, +references, and value-flow summaries. Pseudocode offsets and limits count UTF-16 +code units; use the returned `next_offset` for continuation. Provider limitations +and unavailable facts remain explicit; the complete parent Evidence is retained. + To see which functions or Objective-C methods a Mach-O actually calls in one run, use `observe_native_calls` with explicit breakpoints and a bounded `duration_ms`/`max_events`. It launches a new process under LLDB, so the target diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts index 29289efae8..b361e162fa 100644 --- a/src/application/analysisView/AnalysisViewService.test.ts +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -2,6 +2,8 @@ import { expect, it } from "vitest"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; import { createEvidence, parseEvidence } from "../../domain/evidence.js"; +import { functionDossierSchema } from "../../domain/hopperValues.js"; +import { ghidraFunctionDossier } from "../../domain/ghidraValues.fixture.js"; import { analysisViewJavaScriptAnalysisWithSource, analysisViewJavaScriptEvidence, @@ -9,6 +11,96 @@ import { } from "../../../tests/fixtures/analysisView.js"; import { inspectAnalysisView } from "./AnalysisViewService.js"; +it.each(["/fixtures/unknown.exe", ""])( + "preserves recorded empty native identity fields: path=%j", + (path) => { + const original = functionDossierSchema.parse(ghidraFunctionDossier()); + const parent = createEvidence( + { path, format: "pe", sha256: "c".repeat(64) }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: {}, + result: { + ...original, + procedure: { + ...original.procedure, + address: "", + body: { available: false, reason: "Not recorded" }, + }, + }, + }, + ); + const result = inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view: { + kind: "native", + facet: "procedure", + offset: 0, + limit: 64, + }, + }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toMatchObject({ + procedure_address: "", + artifact: { path }, + item: { address: "" }, + }); + expect(result.value.locations).toEqual( + path.length > 0 ? [{ kind: "artifact-path", path }] : [], + ); + }, +); + +it("projects a small native view from an authenticated >10 MiB retained dossier", () => { + const original = functionDossierSchema.parse(ghidraFunctionDossier()); + const parent = createEvidence( + { path: "/fixtures/large.exe", format: "pe", sha256: "c".repeat(64) }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: { address: "0x401000" }, + result: { ...original, pseudocode: "X".repeat(11 * 1024 * 1024) }, + limitations: ["Synthetic fixture"], + }, + ); + const input = { + source: { + kind: "retained-evidence" as const, + evidence_id: parent.evidence_id, + }, + view: { + kind: "native" as const, + facet: "pseudocode" as const, + offset: 0, + limit: 128, + }, + }; + const view = inspectAnalysisView(input, (id) => + id === parent.evidence_id ? parent : undefined, + ); + if (!view.ok) throw view.error; + expect(view.value.evidence_links).toEqual([parent.evidence_id]); + expect(view.value.normalized_result).toMatchObject({ + kind: "native", + parent_operation: "analyze_function", + parent_evidence_id: parent.evidence_id, + procedure_address: "0x401000", + artifact: { path: "/fixtures/large.exe", sha256: "c".repeat(64) }, + coverage: { total: 11 * 1024 * 1024, examined: 128, next_offset: 128 }, + }); + expect(view.value.locations).toEqual([ + { kind: "artifact-path", path: "/fixtures/large.exe" }, + { kind: "address", address: "0x401000" }, + ]); + expect(JSON.stringify(view.value).length).toBeLessThan(12000); + expect(JSON.stringify(parent).length).toBeGreaterThan(10 * 1024 * 1024); + expect(inspectAnalysisView(input, () => undefined)).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); +}); + it("projects inline layout Evidence and retains a derived view record", () => { const parent = analysisViewLayoutEvidence(); const result = inspectAnalysisView({ diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 0d10d5fa8d..7af6fb24a7 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -53,6 +53,13 @@ export const inspectAnalysisViewValidated = ( operation: parent.operation, normalizedResult: parent.normalized_result, limitations: parent.limitations, + artifact: + parent.subject === null + ? null + : { + path: parent.subject.local_path, + sha256: parent.subject.digest.sha256, + }, }, input.view, ); @@ -126,7 +133,16 @@ const createAnalysisViewEvidence = ( confidence: "derived", authority: parent.authority, limitations: result.limitations, - locations: [{ kind: "artifact-path", path: result.artifact.path }], + locations: [ + ...(result.artifact.path.length > 0 + ? [{ kind: "artifact-path" as const, path: result.artifact.path }] + : []), + ...(result.kind === "native" && + result.procedure_address !== null && + result.procedure_address.length > 0 + ? [{ kind: "address" as const, address: result.procedure_address }] + : []), + ], evidenceLinks: [parent.evidence_id], }); }; diff --git a/src/cli/analysisViewCommands.ts b/src/cli/analysisViewCommands.ts index a0deb02290..a193d27400 100644 --- a/src/cli/analysisViewCommands.ts +++ b/src/cli/analysisViewCommands.ts @@ -18,7 +18,7 @@ export const registerAnalysisViewCommands = ( ): void => { cli.command(CLI_COMMANDS.inspectAnalysisView, { description: - "Project a selected view of completed layout or JavaScript application Evidence JSON", + "Project a selected view of completed layout, JavaScript application or native function Evidence JSON", args: z.object({ inputJson: z.string().describe("Inline workflow JSON or JSON file path"), }), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index b8f455767a..be65b229f2 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -8,14 +8,14 @@ import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js"; const RETAINED_EXAMPLE_ID = `ev_${"a".repeat(64)}`; -/** Selected views of already completed layout and JavaScript analysis Evidence. */ +/** Selected views of already completed layout, JavaScript and native function Evidence. */ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ { name: "inspect_analysis_view", ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - "Inspect a selected view of completed binary-layout or JavaScript application Evidence without repeating analysis. Use an exact same-session retained evidence_id or portable inline Evidence. Select a summary, layout mitigations/linkage facet, one section/symbol/module, or a stable page with a positive limit. Module pages contain JavaScript assets, bundled modules and source modules; use node_id when paths are ambiguous or unavailable. Returns selected facts inline with artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the retained Evidence when needed.", + "Inspect selected views of completed binary-layout, JavaScript application or native analyze_function Evidence without repeating analysis. Use an exact same-session retained evidence_id or portable inline Evidence. Select a summary, layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Native views select procedure, pseudocode, assembly, callers/callees, references and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.", inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ @@ -43,6 +43,21 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ }, }, }, + { + title: "Page native high-pcode operations from retained Evidence", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { + kind: "native", + facet: "value_flow_operations", + offset: 0, + limit: 32, + }, + }, + }, { title: "Page module identities from retained application Evidence", input: { diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index e3aabcf143..1fd6fb7f1a 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -1,6 +1,8 @@ import { expect, it } from "vitest"; import { resolveJavaScriptSourceMapReference } from "../javascript/javascriptSourceMapPaths.js"; +import { functionDossierSchema } from "../hopperValues.js"; +import { ghidraFunctionDossier } from "../ghidraValues.fixture.js"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; import { analysisViewBindJavaScriptGraphs, @@ -48,21 +50,190 @@ const javascriptParent = ( limitations: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.limitations, }); -it("accepts caller-selected page sizes and rejects malformed bounds", () => { - const input = { - source: { kind: "retained-evidence", evidence_id: `ev_${"a".repeat(64)}` }, - view: { kind: "page", collection: "sections", offset: 0, limit: 500 }, - }; - expect(inspectAnalysisViewInputSchema.safeParse(input).success).toBe(true); - for (const limit of [0, -1, 0.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) - expect( - inspectAnalysisViewInputSchema.safeParse({ - ...input, - view: { ...input.view, limit }, - }).success, - ).toBe(false); +const nativeParent = ( + result: unknown = ghidraFunctionDossier(), +): AnalysisViewParent => ({ + evidenceId: "ev_" + "b".repeat(64), + operation: "analyze_function", + normalizedResult: result, + limitations: ["Retained provider limitation"], + artifact: { path: "/fixtures/example.exe", sha256: "a".repeat(64) }, }); +it("projects native procedure and bounded assembly and high-pcode with exact parent identity", () => { + const parent = nativeParent(); + const procedure = projectAnalysisView(parent, { + kind: "native", + facet: "procedure", + offset: 0, + limit: 64, + }); + if (!procedure.ok) throw procedure.error; + expect(procedure.value).toMatchObject({ + kind: "native", + parent_operation: "analyze_function", + artifact: parent.artifact, + item: { address: "0x401000" }, + }); + expect(procedure.value.view_digest).not.toBe(parent.evidenceId.slice(3)); + const assembly = projectAnalysisView(parent, { + kind: "native", + facet: "assembly", + offset: 0, + limit: 1, + }); + if (!assembly.ok) throw assembly.error; + expect(assembly.value).toMatchObject({ + coverage: { examined: 1, total: 2, next_offset: 1 }, + item: ["0x401000: CALL 0x401020"], + }); + const flow = projectAnalysisView(parent, { + kind: "native", + facet: "value_flow_operations", + offset: 0, + limit: 1, + }); + if (!flow.ok) throw flow.error; + expect(flow.value).toMatchObject({ item: [{ opcode: "COPY" }] }); +}); + +it("rejects invalid native parents, unsupported source operations and singleton offsets", () => { + const view = { + kind: "native", + facet: "assembly", + offset: 0, + limit: 4, + } as const; + expect(projectAnalysisView(nativeParent({}), view)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect( + projectAnalysisView({ ...nativeParent(), artifact: null }, view), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(projectAnalysisView(layoutParent(), view)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(projectAnalysisView(javascriptParent(), view)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect( + projectAnalysisView(nativeParent(), { + kind: "native", + facet: "procedure", + offset: 1, + limit: 2, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); +}); + +it("preserves native API limitations and unavailable residual unknowns", () => { + const original = functionDossierSchema.parse(ghidraFunctionDossier()); + const view = { + kind: "native", + facet: "procedure", + offset: 0, + limit: 64, + } as const; + const unavailable = projectAnalysisView( + nativeParent({ + ...original, + native_api: { + available: false, + reason: "No decompiler", + residual_unknowns: ["Calling convention unresolved"], + }, + }), + view, + ); + if (!unavailable.ok) throw unavailable.error; + expect(unavailable.value.unknowns).toContain("Calling convention unresolved"); + const available = projectAnalysisView(nativeParent(), view); + if (!available.ok) throw available.error; + if (!original.native_api?.available) throw new Error("expected API fixture"); + expect(available.value.limitations).toEqual( + expect.arrayContaining(original.native_api.limitations), + ); +}); + +it("reports the malformed native dossier field", () => { + const result = projectAnalysisView( + nativeParent({ + ...functionDossierSchema.parse(ghidraFunctionDossier()), + pseudocode: 12, + }), + { + kind: "native", + facet: "pseudocode", + offset: 0, + limit: 64, + }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["source", "normalized_result", "pseudocode"], + }), + ]), + }, + }); +}); + +it("native pseudocode pages do not split surrogate pairs", () => { + const original = functionDossierSchema.parse(ghidraFunctionDossier()); + const parent = nativeParent({ ...original, pseudocode: "ab😀cd" }); + expect( + projectAnalysisView(parent, { + kind: "native", + facet: "pseudocode", + offset: 3, + limit: 2, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + const safe = projectAnalysisView(parent, { + kind: "native", + facet: "pseudocode", + offset: 2, + limit: 3, + }); + if (!safe.ok) throw safe.error; + expect(safe.value).toMatchObject({ + item: { text: "😀c", unit: "utf16-code-units" }, + coverage: { examined: 3, next_offset: 5 }, + }); +}); + +it.each([ + { kind: "page", collection: "sections", offset: 0, limit: 500 }, + { kind: "native", facet: "assembly", offset: 0, limit: 500 }, +])( + "accepts caller-selected page sizes and rejects malformed bounds: $kind", + (view) => { + const input = { + source: { + kind: "retained-evidence", + evidence_id: `ev_${"a".repeat(64)}`, + }, + view, + }; + expect(inspectAnalysisViewInputSchema.safeParse(input).success).toBe(true); + for (const limit of [0, -1, 0.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) + expect( + inspectAnalysisViewInputSchema.safeParse({ + ...input, + view: { ...input.view, limit }, + }).success, + ).toBe(false); + }, +); + it("projects layout summary, facet, item, and stable pages", () => { const parent = layoutParent(); const layout = analysisViewLayoutFixture(); diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index efb9f5d44b..db2d2dbac3 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -12,6 +12,7 @@ import { jsonObjectSchema, jsonValueSchema } from "../jsonValue.js"; import { err, ok, type Result } from "../result.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { binaryLayoutSchema } from "../native/binaryLayout.js"; +import { functionDossierSchema } from "../hopperValues.js"; import { parseOwnedJavaScriptApplicationAnalysisSteps, type JavaScriptApplicationAnalysisResult, @@ -19,6 +20,7 @@ import { import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { projectBinaryLayoutView } from "./binaryLayoutView.js"; import { projectJavaScriptApplicationView } from "./javascriptApplicationView.js"; +import { projectNativeFunctionView } from "./nativeFunctionView.js"; /** Public name of the selected-view workflow. */ export const INSPECT_ANALYSIS_VIEW_OPERATION = "inspect_analysis_view" as const; @@ -73,12 +75,35 @@ const pageViewSchema = z.strictObject({ limit: z.number().int().positive(), }); +/** Bounded native function facets of already authenticated analyze_function Evidence. */ +const nativeViewSchema = z.strictObject({ + kind: z.literal("native"), + facet: z.enum([ + "procedure", + "pseudocode", + "assembly", + "callers", + "callees", + "incoming_references", + "outgoing_references", + "value_flow_summary", + "value_flow_operations", + "value_flow_def_use", + "value_flow_effects", + "value_flow_parameters", + "value_flow_parameter_uses", + ]), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}); + /** Caller-selected projection of already completed analysis Evidence. */ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ summaryViewSchema, facetViewSchema, itemViewSchema, pageViewSchema, + nativeViewSchema, ]); /** Inline Evidence or an exact same-session retained reference. */ @@ -108,7 +133,9 @@ const coverageSchema = z.strictObject({ }); const artifactSchema = z.strictObject({ - path: z.string().min(1), + path: z + .string() + .describe("Exact recorded artifact path, including an empty string."), sha256: digestSchema, }); @@ -150,6 +177,7 @@ const viewResultBase = { parent_operation: z.enum([ "inspect_binary_layout", "analyze_javascript_application", + "analyze_function", ]), parent_digest: digestSchema, view_digest: digestSchema, @@ -185,6 +213,13 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ items: z.array(jsonValueSchema), ...viewResultBase, }), + z.strictObject({ + kind: z.literal("native"), + view: nativeViewSchema, + procedure_address: z.string().nullable(), + item: jsonValueSchema, + ...viewResultBase, + }), ]); export type AnalysisViewRequest = z.output; @@ -200,6 +235,7 @@ export interface AnalysisViewParent { readonly operation: string; readonly normalizedResult: unknown; readonly limitations: readonly string[]; + readonly artifact?: { readonly path: string; readonly sha256: string } | null; } /** Construct a typed input failure for one selected-view constraint. */ @@ -216,7 +252,11 @@ export type UnsignedAnalysisView = > | Omit, "view_digest"> | Omit, "view_digest"> - | Omit, "view_digest">; + | Omit, "view_digest"> + | Omit< + Extract, + "view_digest" + >; /** Attach the content digest that must change when projected bytes change. */ export const sealAnalysisView = ( @@ -261,7 +301,7 @@ const unsupportedParent = ( new AnalysisUnsupportedTargetError( INSPECT_ANALYSIS_VIEW_OPERATION, parentArtifactPath(parent), - `Parent Evidence operation ${JSON.stringify(parent.operation)} is not inspect_binary_layout or analyze_javascript_application.`, + `Parent Evidence operation ${JSON.stringify(parent.operation)} is not inspect_binary_layout, analyze_javascript_application or analyze_function.`, ); const parentArtifactPath = (parent: AnalysisViewParent): string => { @@ -327,5 +367,22 @@ export const projectAnalysisView = ( ); return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; } + if (parent.operation === "analyze_function") { + const native = functionDossierSchema.safeParse(parent.normalizedResult); + if (!native.success) + return err( + analysisInputErrorFromIssues( + INSPECT_ANALYSIS_VIEW_OPERATION, + native.error.issues.map((issue) => ({ + ...issue, + path: ["source", "normalized_result", ...issue.path], + })), + { source: { normalized_result: parent.normalizedResult } }, + { cause: native.error }, + ), + ); + const projected = projectNativeFunctionView(parent, native.data, view); + return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; + } return err(unsupportedParent(parent)); }; diff --git a/src/domain/analysisView/binaryLayoutView.ts b/src/domain/analysisView/binaryLayoutView.ts index 5a52904a85..13a9b766f3 100644 --- a/src/domain/analysisView/binaryLayoutView.ts +++ b/src/domain/analysisView/binaryLayoutView.ts @@ -152,6 +152,10 @@ export const projectBinaryLayoutView = ( view: AnalysisViewRequest, ): Result => { const shared = parentFields(parent, layout); + if (view.kind === "native") + return err( + layoutIncompatible("Native views require analyze_function Evidence."), + ); if (view.kind === "summary") return ok({ kind: "summary", diff --git a/src/domain/analysisView/javascriptApplicationView.ts b/src/domain/analysisView/javascriptApplicationView.ts index 5216c72b44..d6e994fc94 100644 --- a/src/domain/analysisView/javascriptApplicationView.ts +++ b/src/domain/analysisView/javascriptApplicationView.ts @@ -243,6 +243,10 @@ export const projectJavaScriptApplicationView = ( ): Result => { const shared = parentFields(parent, analysis); const modules = moduleEntries(analysis.graph.nodes); + if (view.kind === "native") + return err( + jsIncompatible("Native views require analyze_function Evidence."), + ); if (view.kind === "summary") return ok({ kind: "summary", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts new file mode 100644 index 0000000000..d17f69b038 --- /dev/null +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -0,0 +1,202 @@ +import type { AnalysisError } from "../analysisErrorBase.js"; +import { uniqueSorted } from "../canonicalOrdering.js"; +import type { FunctionDossier } from "../hopperValues.js"; +import { jsonValueSchema } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + analysisViewInputError, + type AnalysisViewCoverage, + type AnalysisViewParent, + type AnalysisViewRequest, + type UnsignedAnalysisView, +} from "./analysisView.js"; +import { + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; + +type Artifact = { readonly path: string; readonly sha256: string }; + +const viewError = (message: string): AnalysisError => + analysisViewInputError([ + { path: ["view"], reason: "invalid_value", message }, + ]); + +const parentFields = ( + parent: AnalysisViewParent, + dossier: FunctionDossier, + artifact: Artifact, +) => { + const flow = dossier.native_value_flow; + const api = dossier.native_api; + const unknowns: string[] = []; + if (flow === null) unknowns.push("native_value_flow was not recorded"); + else if (!flow.available) + unknowns.push("native_value_flow unavailable: " + flow.reason); + else if (flow.truncated) + unknowns.push( + "native_value_flow is truncated; omitted observations remain unknown", + ); + if (api === null) unknowns.push("native_api was not recorded"); + else if (!api.available) + unknowns.push( + "native_api unavailable: " + api.reason, + ...api.residual_unknowns, + ); + return { + parent_evidence_id: parent.evidenceId, + parent_operation: "analyze_function" as const, + parent_digest: parent.evidenceId.slice(3), + procedure_address: dossier.procedure.address, + artifact, + limitations: uniqueSorted([ + ...parent.limitations, + ...dossier.limitations, + ...(flow?.limitations ?? []), + ...(api?.available ? api.limitations : []), + ]), + unknowns: uniqueSorted(unknowns), + }; +}; + +const pseudocodePage = ( + body: string, + offset: number, + limit: number, +): Result< + { + readonly item: { readonly text: string; readonly unit: "utf16-code-units" }; + readonly coverage: AnalysisViewCoverage; + }, + AnalysisError +> => { + const start = Math.min(offset, body.length); + const high = (n: number) => n >= 0xd800 && n <= 0xdbff; + const low = (n: number) => n >= 0xdc00 && n <= 0xdfff; + if ( + start > 0 && + start < body.length && + high(body.charCodeAt(start - 1)) && + low(body.charCodeAt(start)) + ) + return err(viewError("Pseudocode offset splits a UTF-16 surrogate pair.")); + let end = Math.min(start + limit, body.length); + if ( + end > start && + end < body.length && + high(body.charCodeAt(end - 1)) && + low(body.charCodeAt(end)) + ) + end--; + if (end === start && start < body.length) + return err( + viewError( + "Increase pseudocode limit to include the next complete Unicode character.", + ), + ); + return ok({ + item: { text: body.slice(start, end), unit: "utf16-code-units" }, + coverage: pageViewCoverage(start, end - start, body.length), + }); +}; + +const nativeRows = ( + dossier: FunctionDossier, + facet: Exclude< + Extract["facet"], + "procedure" | "pseudocode" | "value_flow_summary" + >, +) => { + const sourceFlow = dossier.native_value_flow; + const flow = sourceFlow?.available ? sourceFlow : undefined; + return { + assembly: dossier.assembly, + callers: dossier.callers, + callees: dossier.callees, + incoming_references: dossier.incoming_references, + outgoing_references: dossier.outgoing_references, + value_flow_operations: flow?.operations, + value_flow_def_use: flow?.def_use, + value_flow_effects: flow?.effects, + value_flow_parameters: flow?.parameters, + value_flow_parameter_uses: flow?.parameter_uses, + }[facet]; +}; + +/** Project bounded native function facets from complete retained Evidence without reanalysis. */ +export const projectNativeFunctionView = ( + parent: AnalysisViewParent, + dossier: FunctionDossier, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "native") + return err(viewError("analyze_function Evidence requires a native view.")); + const artifact = parent.artifact; + if (!artifact) + return err( + analysisViewInputError([ + { + path: ["source", "subject"], + reason: "invalid_value", + message: + "Native function Evidence requires an artifact-bound subject.", + }, + ]), + ); + const common = parentFields(parent, dossier, artifact); + const singleton = (item: unknown): UnsignedAnalysisView => ({ + kind: "native", + view, + item: jsonValueSchema.parse(item), + coverage: completeWithinViewCoverage(1, 1), + ...common, + }); + const flow = dossier.native_value_flow; + if (view.facet === "procedure" || view.facet === "value_flow_summary") { + if (view.offset !== 0) + return err(viewError("Singleton native facets require offset 0.")); + if (view.facet === "procedure") return ok(singleton(dossier.procedure)); + const status = + flow === null + ? { available: false, reason: "not recorded" } + : !flow.available + ? { available: false, reason: flow.reason } + : { + available: true, + provenance: flow.provenance, + truncated: flow.truncated, + omitted_operations_lower_bound: + flow.omitted_operations_lower_bound, + known_omitted_inputs: flow.known_omitted_inputs, + known_omitted_edges: flow.known_omitted_edges, + operations: flow.operations.length, + def_use: flow.def_use.length, + effects: flow.effects.length, + parameters: flow.parameters.length, + parameter_uses: flow.parameter_uses.length, + }; + return ok(singleton(status)); + } + if (view.facet === "pseudocode") { + const page = pseudocodePage(dossier.pseudocode, view.offset, view.limit); + return page.ok + ? ok({ kind: "native", view, ...page.value, ...common }) + : page; + } + const rows = nativeRows(dossier, view.facet); + if (rows === undefined) + return ok( + singleton({ + available: false, + reason: flow?.available === false ? flow.reason : "not recorded", + }), + ); + const items = rows.slice(view.offset, view.offset + view.limit); + return ok({ + kind: "native", + view, + item: jsonValueSchema.parse(items), + coverage: pageViewCoverage(view.offset, items.length, rows.length), + ...common, + }); +}; diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index 86e0d9b271..2abaf700ac 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -7,6 +7,8 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; import { analysisViewLayoutEvidence } from "../../fixtures/analysisView.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; cliTest( "projects one section from a JSON file of inline layout Evidence", async ({ cli }) => { @@ -43,6 +45,48 @@ cliTest( }, ); +cliTest( + "projects native function pseudocode without requiring Ghidra", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-native-view-cli-"); + const parent = createEvidence( + { path: "/fixtures/native.exe", format: "pe", sha256: "a".repeat(64) }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: { address: "0x401000" }, + result: ghidraFunctionDossier(), + }, + ); + const input = join(root, "native-view.json"); + await writeFile( + input, + JSON.stringify({ + source: { kind: "inline", evidence: parent }, + view: { kind: "native", facet: "pseudocode", offset: 0, limit: 12 }, + }), + ); + const result = await cli.run({ + arguments: ["inspect-analysis-view", input, "--json"], + environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root }, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + operation: "inspect_analysis_view", + normalized_result: { + kind: "native", + parent_evidence_id: parent.evidence_id, + procedure_address: "0x401000", + item: { text: "int fixture_", unit: "utf16-code-units" }, + }, + locations: [ + { kind: "artifact-path", path: "/fixtures/native.exe" }, + { kind: "address", address: "0x401000" }, + ], + }); + }, +); + cliTest("rejects malformed JSON before projection", async ({ cli }) => { const root = await createTestTempDirectory("rea-analysis-view-cli-bad-"); const result = await cli.run({ diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index 3edeffba78..3c15312aa6 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -6,7 +6,9 @@ import { z } from "zod"; import { BinaryLayoutService } from "../../../src/application/binaryDiagnostics/BinaryLayoutService.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; -import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; import { ok } from "../../../src/domain/result.js"; import { createJavaScriptApplicationGraph, @@ -109,6 +111,68 @@ it("advertises exact schemas and projects one layout section from retained Evide ); }); +it("delivers a bounded native dossier view over MCP without a Ghidra provider or lost connection", async () => { + const { client, session } = await connect(); + const original = functionDossierSchema.parse(ghidraFunctionDossier()); + const parent = createEvidence( + { + path: "/fixtures/oversized-native.exe", + format: "pe", + sha256: "b".repeat(64), + }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: { address: "0x401000" }, + result: { ...original, pseudocode: "X".repeat(11 * 1024 * 1024) }, + }, + ); + expect(session.recordEvidence(parent).ok).toBe(true); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_analysis_view", + ); + if (advertised?.outputSchema === undefined) + throw new Error("missing view schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + const arguments_ = { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "native", facet: "pseudocode", offset: 0, limit: 500 }, + }; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect( + ajv.validate(inputSchema, arguments_), + JSON.stringify(ajv.errors), + ).toBe(true); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: arguments_, + }); + expect(response.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_analysis_view").outputSchema.parse( + response.structuredContent, + ); + expect(parsed.normalized_result).toMatchObject({ + kind: "native", + parent_evidence_id: parent.evidence_id, + procedure_address: "0x401000", + coverage: { examined: 500, next_offset: 500, total: 11 * 1024 * 1024 }, + }); + expect(parsed.evidence_links).toEqual([parent.evidence_id]); + expect(parsed.locations).toEqual([ + { kind: "artifact-path", path: "/fixtures/oversized-native.exe" }, + { kind: "address", address: "0x401000" }, + ]); + expect(JSON.stringify(response.structuredContent).length).toBeLessThan(12000); + await client.ping(); +}); + it("retains an oversized selected observation and keeps subsequent views usable", async () => { const { client, session } = await connect(); const analysis = analysisViewJavaScriptAnalysisWithSource(); @@ -185,6 +249,71 @@ it("retains an oversized selected observation and keeps subsequent views usable" expect(summary.isError).not.toBe(true); }); +it("retains an oversized native assembly row and permits a subsequent smaller view", async () => { + const { client, session } = await connect(); + const row = "\0".repeat(1_000_000); + const parent = createEvidence( + { path: "/fixtures/large-row.exe", format: "pe", sha256: "c".repeat(64) }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: {}, + result: { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + assembly: [row], + }, + }, + ); + expect(session.recordEvidence(parent).ok).toBe(true); + const source = { kind: "retained-evidence", evidence_id: parent.evidence_id }; + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { kind: "native", facet: "assembly", offset: 0, limit: 1 }, + }, + }); + expect(response.isError).toBe(true); + const error = z + .object({ + error: z.object({ + details: z.object({ + reported_limits: z.object({ + evidence_reference: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }), + }), + }), + }) + .parse(parseMcpToolError(response)); + const retained = session.evidenceById( + error.error.details.reported_limits.evidence_reference.evidence_id, + ); + expect(retained?.normalized_result).toMatchObject({ + item: [row], + parent_evidence_id: parent.evidence_id, + }); + const next = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { kind: "native", facet: "pseudocode", offset: 0, limit: 12 }, + }, + }); + expect(next.isError).not.toBe(true); + expect( + toolContract("inspect_analysis_view").outputSchema.parse( + next.structuredContent, + ).normalized_result, + ).toMatchObject({ + item: { text: "int fixture_" }, + parent_evidence_id: parent.evidence_id, + }); + await client.ping(); +}); + it("returns a JavaScript summary without graph payloads from a retained reference", async () => { const { client, session } = await connect(); const parent = analysisViewJavaScriptEvidence(); From f4a0c1ce95375c1661ca95b414829113a10298b8 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:20:05 +0800 Subject: [PATCH 17/74] fix(javascript)!: avoid event payload copies in graph metadata (#1427) Share the canonical string identity rule with endpoint projection, scope event keys to each emitter, and keep complete event/listener names in properties. Verify real CLI and stdio MCP with large names, registration/removal, distinct emitters, exact name seeds, reserved-prefix values, and unresolved dynamic names. BREAKING CHANGE: New analyses use digest-based IDs for long or reserved-prefix event names and shorter event/listener display labels. Select IDs from the parent graph and read exact names from event_name properties. Stored graphs and their original IDs remain accepted for semantic tracing. --- docs/javascript-application-workflows.md | 10 ++ .../JavaScriptArtifactGraphFindings.ts | 11 +- .../JavaScriptSemanticGraphAsyncProjection.ts | 13 +- src/domain/compactStringIdentity.ts | 10 ++ .../javascriptEventPayload.test.ts | 164 ++++++++++++++++++ 5 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 src/domain/compactStringIdentity.ts create mode 100644 tests/acceptance/applications/javascriptEventPayload.test.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 592276e15e..f26c16d1f6 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -118,6 +118,16 @@ Use IDs from the selected graph and read the complete endpoint properties; previously stored graphs still support their original IDs. Literal endpoint search, semantic tracing, and version comparison use the preserved values. +Event and listener nodes likewise preserve the complete event name in +`properties.event_name`. Event role keys use the same reserved-prefix string +digest rule, scoped to the emitter; repeated registrations, removals, and +dispatches keep one event per exact emitter/name pair. Long event labels use +`event`, and listener labels use the shorter of `:` and +`:listener`. Dynamic names remain `null` with explicit uncertainty. New analyses +change hashed event IDs and longer event/listener labels; query the preserved +name or select IDs from the parent graph. Previously stored graphs keep their +original IDs and remain valid trace inputs. + ## Version comparison REA pairs entities only when a tier produces one unique candidate on each side. diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 580a4d0355..41f480daf3 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; +import { compactStringIdentityKey } from "../../domain/compactStringIdentity.js"; import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js"; import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -163,14 +163,7 @@ const addEndpoint = ( const source = sourceNodeFor(context, input.file.path, input.value.module_key) ?? input.asset; - const digestPrefix = "value-sha256:"; - const valueKey = `${digestPrefix}${digestCanonicalValue(input.value.value, "JavaScript endpoint")}`; - // Reserve the digest prefix so an exact caller value cannot alias a digest key. - const key = - input.value.value.length <= valueKey.length && - !input.value.value.startsWith(digestPrefix) - ? input.value.value - : valueKey; + const key = compactStringIdentityKey(input.value.value); const label = `${input.value.kind} endpoint`; const node = context.accumulator.addNode({ kind: "endpoint", diff --git a/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts b/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts index a8814281f3..f54c5c4f31 100644 --- a/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts @@ -1,3 +1,4 @@ +import { compactStringIdentityKey } from "../../domain/compactStringIdentity.js"; import { createJavaScriptSemanticGraphUnknown } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraphSchemas.js"; import type { @@ -21,7 +22,7 @@ export const projectSemanticEvents = ( const eventKey = operation.eventName === null ? operation.eventId - : `${operation.emitterKey}\u0000${operation.eventName}`; + : `${operation.emitterKey}\u0000${compactStringIdentityKey(operation.eventName)}`; const event = eventNodes.get(eventKey) ?? addEventNode(context, operation, eventKey); if (event === null) continue; @@ -59,7 +60,11 @@ const addEventNode = ( kind: "event", roleKey: `event:${eventKey}`, location: operation.eventName === null ? operation.location : null, - label: operation.eventName, + label: + operation.eventName === null || + operation.eventName.length <= "event".length + ? operation.eventName + : "event", functionNodeId: null, properties: { emitter_key: operation.emitterKey, @@ -78,7 +83,9 @@ const addListenerNode = ( label: operation.eventName === null ? operation.method - : `${operation.method}:${operation.eventName}`, + : operation.eventName.length <= "listener".length + ? `${operation.method}:${operation.eventName}` + : `${operation.method}:listener`, functionNodeId: operation.ownerCallableId === null ? null diff --git a/src/domain/compactStringIdentity.ts b/src/domain/compactStringIdentity.ts new file mode 100644 index 0000000000..421555a908 --- /dev/null +++ b/src/domain/compactStringIdentity.ts @@ -0,0 +1,10 @@ +import { digestCanonicalValue } from "./canonicalDigest.js"; + +/** Retain short exact string keys or commit their value in a reserved digest namespace. */ +export const compactStringIdentityKey = (value: string): string => { + const prefix = "value-sha256:"; + const digestKey = `${prefix}${digestCanonicalValue(value, "String identity")}`; + return value.length <= digestKey.length && !value.startsWith(prefix) + ? value + : digestKey; +}; diff --git a/tests/acceptance/applications/javascriptEventPayload.test.ts b/tests/acceptance/applications/javascriptEventPayload.test.ts new file mode 100644 index 0000000000..c4689250de --- /dev/null +++ b/tests/acceptance/applications/javascriptEventPayload.test.ts @@ -0,0 +1,164 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "retains large event names and emitter-scoped listener ownership through CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-event-payload-"); + const name = "payload/ğ".repeat(80_000); + const lookalike = `value-sha256:${createHash("sha256").update(JSON.stringify(name)).digest("hex")}`; + const source = [ + 'import {EventEmitter} from "node:events";', + "const bus = new EventEmitter(); const other = new EventEmitter(); const handler = () => {};", + `bus.on(${JSON.stringify(name)}, handler);`, + `bus.emit(${JSON.stringify(name)});`, + `bus.off(${JSON.stringify(name)}, handler);`, + `bus.on(${JSON.stringify(lookalike)}, handler); bus.emit(${JSON.stringify(lookalike)});`, + 'bus.on("ok", handler); bus.emit("ok"); other.on("ok", handler); other.emit("ok");', + "bus.on(getEventName(), handler);", + ].join("\n"); + await writeFile(join(root, "main.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + timeoutMs: 60_000, + }); + expect(cliResponse.exitCode).toBe(0); + const cliEvidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(cliResponse.json); + const client = new Client({ name: "event-payload-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + expect( + toolContract("analyze_javascript_application").outputSchema.parse( + JSON.parse(text.text), + ).evidence_id, + ).toBe(evidence.evidence_id); + const graph = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ).semantic_graph; + const events = graph.nodes.filter((node) => node.kind === "event"); + const selected = events.filter( + (node) => node.properties.event_name === name, + ); + expect(selected).toHaveLength(1); + const event = selected[0]; + if (event === undefined) throw new Error("Complete event was lost"); + const listeners = graph.nodes.filter( + (node) => + node.kind === "listener" && node.properties.event_name === name, + ); + expect(new Set(listeners.map((node) => node.properties.method))).toEqual( + new Set(["on", "off"]), + ); + const listenerIds = new Set(listeners.map((node) => node.node_id)); + expect( + graph.relations + .filter( + (relation) => + relation.source_node_id === event.node_id && + listenerIds.has(relation.target_node_id), + ) + .map((relation) => relation.relation), + ).toEqual( + expect.arrayContaining(["registers-listener", "removes-listener"]), + ); + const lookalikeEvent = events.find( + (node) => node.properties.event_name === lookalike, + ); + expect(lookalikeEvent).toBeDefined(); + expect(lookalikeEvent?.node_id).not.toBe(event.node_id); + const shortEvents = events.filter( + (node) => node.properties.event_name === "ok", + ); + expect(shortEvents).toHaveLength(2); + expect( + new Set(shortEvents.map((node) => node.properties.emitter_key)).size, + ).toBe(2); + expect( + shortEvents.every( + (node) => + node.label === "ok" && node.identity.role_key.endsWith("\0ok"), + ), + ).toBe(true); + const dynamic = events.find( + (node) => node.properties.event_name === null, + ); + expect(dynamic?.label).toBeNull(); + expect( + graph.unknowns.some( + (unknown) => + unknown.node_id === dynamic?.node_id && unknown.family === "event", + ), + ).toBe(true); + const traceResponse = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + query: { + seed: { kind: "event", name }, + direction: "forward-influence", + }, + }, + }); + expect(traceResponse.isError).not.toBe(true); + const traceEvidence = toolContract( + "trace_javascript_semantics", + ).outputSchema.parse(traceResponse.structuredContent); + const trace = javaScriptSemanticTraceResultSchema.parse( + traceEvidence.normalized_result, + ); + expect(new Set(trace.seed_node_ids)).toEqual( + new Set([event.node_id, ...listenerIds]), + ); + expect( + trace.nodes + .filter((node) => listenerIds.has(node.node_id)) + .map((node) => node.properties.event_name), + ).toEqual(listeners.map(() => name)); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } + }, + 120_000, +); From c29931ce81dc530d2623ec992714a70b1f3e50e4 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:25:00 -0400 Subject: [PATCH 18/74] fix(javascript): scan direct returns without recursive stack growth Use the shared AST child enumeration with an explicit stack, preserving return order and nested callable boundaries. Refs #1065. Co-authored-by: rudycelekli <47457359+rudycelekli@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- .../javascript/javascriptSemanticReturns.ts | 14 +++--- .../javascriptApplicationDeepReturns.test.ts | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts diff --git a/src/domain/javascript/javascriptSemanticReturns.ts b/src/domain/javascript/javascriptSemanticReturns.ts index bb931fba51..44cbbea5bb 100644 --- a/src/domain/javascript/javascriptSemanticReturns.ts +++ b/src/domain/javascript/javascriptSemanticReturns.ts @@ -131,15 +131,17 @@ const directReturnExpressions = (callable: t.Node): ReturnExpression[] => { return [{ node: callable.body, location: range(callable.body) }]; if (!t.isFunction(callable)) return []; const output: ReturnExpression[] = []; - const visit = (node: t.Node): void => { - if (t.isFunction(node) || t.isClass(node)) return; + const pending = childNodes(callable.body).reverse(); + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined || t.isFunction(node) || t.isClass(node)) continue; if (t.isReturnStatement(node)) { output.push({ node: node.argument ?? null, location: range(node) }); - return; + continue; } - for (const child of childNodes(node)) visit(child); - }; - for (const child of childNodes(callable.body)) visit(child); + // Reverse children so the explicit stack retains source-tree order. + for (const child of childNodes(node).reverse()) pending.push(child); + } return output; }; diff --git a/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts b/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts new file mode 100644 index 0000000000..32a2d99af7 --- /dev/null +++ b/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts @@ -0,0 +1,44 @@ +import { execFile } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("analyzes direct returns after deep source without exhausting the default process stack", async () => { + const root = await createTestTempDirectory("rea-deep-direct-returns-"); + await writeFile( + join(root, "app.js"), + `export function render() { object${".property".repeat(20_000)}; function nested() { return "nested"; } class Inner { method() { return "method"; } } return "ready"; }`, + ); + const service = new URL( + "../../../dist/application/javascript/JavaScriptApplicationService.js", + import.meta.url, + ).href; + const script = ` + import { analyzeJavaScriptApplication } from ${JSON.stringify(service)}; + const result = await analyzeJavaScriptApplication({ input_path: ${JSON.stringify(root)}, format: "directory" }); + if (!result.ok) throw result.error; + const nodes = result.value.normalized_result.semantic_graph.nodes; + console.log(JSON.stringify(nodes.filter(node => node.kind === "return-site").map(node => node.function_node_id))); + `; + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + ["--input-type=module", "-e", script], + { + timeout: 10_000, + // Bound the child heap while retaining Node's default stack size. + env: { + HOME: root, + TMPDIR: root, + NODE_OPTIONS: "--max-old-space-size=768", + }, + }, + ); + expect(stderr).toBe(""); + const owners: unknown = JSON.parse(stdout); + expect(owners).toHaveLength(3); + expect(new Set(Array.isArray(owners) ? owners : []).size).toBe(3); +}); From 6375ccaf04b4835cc67080ee1ecd91d1fc46645c Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:35:50 -0700 Subject: [PATCH 19/74] perf(native): record macOS command provenance once in its Evidence (#1430) inspect_macho, inspect_signature, inspect_plist, list_architectures and demangle_swift already return their command provenance inside the normalized result. The provider also copied the same array into raw_result, so every response carried it twice. raw_result now stays null, as docs/mcp-contracts.md describes for results without a distinct provider representation. Co-authored-by: Claude Opus 5.5 --- src/native/NativeMacOSProvider.ts | 6 --- src/native/NativeMachoInspection.ts | 2 - .../providers/native/nativeProvider.test.ts | 37 +++++++++++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 12c94814fc..d63db25061 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -233,7 +233,6 @@ class NativeMacOSClient implements AnalysisClient { return observation.ok ? ok( createAnalysisExecution(observation.value.result, IDENTITY, { - rawResult: { provenance: observation.value.provenance }, limitations: observation.value.limitations, locations: observation.value.locations, }), @@ -317,7 +316,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: [], locations: architectureLocations(result.architectures.items), }); @@ -351,7 +349,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: [], locations: [], }); @@ -448,7 +445,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: result.limitations, locations: [], }); @@ -611,7 +607,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: parsed.value.limitations, locations: [{ kind: "artifact-path", path: plist.value }], }); @@ -659,7 +654,6 @@ class NativeMacOSClient implements AnalysisClient { interface NativeObservation { readonly result: JsonValue; - readonly provenance: readonly NativeCommandInvocation[]; readonly limitations: readonly string[]; readonly locations: readonly EvidenceLocation[]; } diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index 417b43fef5..2ef2c95ae9 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -15,7 +15,6 @@ import { parseOtoolLoadCommands } from "./parsers/otool.js"; interface NativeMachoObservation { readonly result: JsonValue; - readonly provenance: readonly NativeCommandInvocation[]; readonly limitations: readonly string[]; readonly locations: readonly EvidenceLocation[]; } @@ -116,7 +115,6 @@ export const inspectNativeMacho = async ( : normalized; return ok({ result: jsonValueSchema.parse(result), - provenance: result.provenance, limitations: result.limitations, locations: fileOffsetLocations(result, segmentOffset), }); diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index d5be920ba1..70a3b06ad6 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -552,6 +552,43 @@ describe("native macOS provider inspection", () => { }); }); +describe("native macOS provider raw results", () => { + it("records command provenance once, in the normalized result", async () => { + directory = await createTestTempDirectory("rea-native-"); + const app = join(directory, "Fixture.app"); + const executable = join(app, "Contents/MacOS/Fixture"); + await mkdir(join(app, "Contents/MacOS"), { recursive: true }); + await writeFile(executable, "fixture"); + await writeFile(join(app, "Contents/Info.plist"), "fixture"); + const client = new NativeMacOSProvider( + {}, + new FixtureRunner(), + "darwin", + ).createClient(await nativeMachoTargetForFile(executable, app)); + const requests: readonly (readonly [ + Parameters[0], + Parameters[1], + ])[] = [ + ["inspect_macho", {}], + ["list_architectures", {}], + ["inspect_signature", {}], + ["inspect_plist", {}], + ["demangle_swift", { symbols: ["$s4Test3fooyyF", "plain_symbol"] }], + ]; + + for (const [operation, parameters] of requests) { + const execution = await client.execute(operation, parameters); + if (!execution.ok) throw execution.error; + expect(execution.value.rawResult, operation).toBeNull(); + expect(execution.value.result, operation).toMatchObject({ + provenance: expect.arrayContaining([ + expect.objectContaining({ tool: expect.any(String) }), + ]), + }); + } + }); +}); + describe("native plist defaults for iOS-style bundles", () => { it("defaults to the Info.plist the bundle program was resolved from", async () => { directory = await createTestTempDirectory("rea-native-flat-"); From 0b9815b6e7080081d8fd426ad9710aee36761afa Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:38:52 -0700 Subject: [PATCH 20/74] docs(skill): save CLI application Evidence and read a summary view (#1429) The skill's CLI fallback printed the complete application Evidence to stdout. A real ASAR produces hundreds of megabytes (334 MB for Obsidian), which an agent then reads into its context. Save the Evidence to a file and project a summary or module page with inspect-analysis-view instead. Co-authored-by: Claude Opus 5.5 --- docs/cli.md | 17 +++++++++++++++++ skill-src/reverse-engineer-anything/SKILL.md | 19 +++++++++++++------ src/generatedPackageMetadata.ts | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 2a9863c1e9..245251a286 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,6 +20,23 @@ files when neither `--provider` nor `--snapshot` is supplied. See [JavaScript artifact reconstruction](javascript-artifact-reconstruction.md) for results, integrity checks and coverage. +The complete application Evidence can be hundreds of megabytes for a real +application. When an agent or a person reads the result, save it to a file and +project a summary, a module page or one module from the saved Evidence: + +```bash +rea analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +rea inspect-analysis-view app-view.json +``` + +For example, a 334 MB Obsidian application Evidence file yields a summary view +of about 10 KB. Replace the view with +`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}` for a +module page. See +[JavaScript application workflows](javascript-application-workflows.md) for +views, feature traces and comparisons that take the same saved Evidence. + ## Native analysis Configure [Hopper or Ghidra](installation.md#hopper), or the diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/skill-src/reverse-engineer-anything/SKILL.md index d1650f26e8..3eaa239b8d 100644 --- a/skill-src/reverse-engineer-anything/SKILL.md +++ b/skill-src/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "34" + version: "35" --- # REA @@ -62,13 +62,20 @@ with a version-pinned `rea-agents` package or continue through the CLI. A concrete CLI fallback for an operator-supplied JavaScript tree or ASAR is: ```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +npx -y rea-agents@latest inspect-analysis-view app-view.json ``` -No MCP registration or native engine is required. Read the returned Evidence, -graph, limitations, and unknowns with the same care as an MCP result; the CLI -returns the Evidence record directly. This fallback does not establish that -MCP is configured. Native CLI tasks still require their selected engine. +No MCP registration or native engine is required. The CLI returns the complete +Evidence record directly, and a real application can produce hundreds of +megabytes, so save it to a file rather than reading it into context. Start from +the summary view, then request a module page +(`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}`) or one +module the same way, or pass the saved Evidence inline to +`trace-application-feature`. Read graph, limitations, and unknowns with the +same care as an MCP result. This fallback does not establish that MCP is +configured. Native CLI tasks still require their selected engine. ## Route the target first diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 29177bf02f..468117b055 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "34", + skillVersion: "35", } as const; From dc5af6d68f74155d32a0be957c88afb0ee4be699 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:49:09 +0800 Subject: [PATCH 21/74] perf(mcp): reuse output validators across catalog refreshes (#1433) Advertise content- and dialect-bound identities for self-contained output schemas so the SDK can reuse unchanged Ajv validators after complete tools/list refreshes. Preserve explicit IDs and relative reference bases. Verify changed schemas, availability transitions, reconnects, numeric and selector constraints, explicit IDs, relative references, and advertised dialects through the pinned SDK boundaries. Refs #1065, #723 --- docs/mcp-contracts.md | 7 ++ src/contracts/toolOutputReferences.test.ts | 58 ++++++++++ src/contracts/toolSchemaMetadata.ts | 44 ++++++- .../boundary/mcp/outputValidatorReuse.test.ts | 108 ++++++++++++++++++ 4 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 tests/boundary/mcp/outputValidatorReuse.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 7bf2fb76c0..70d843335b 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -36,6 +36,13 @@ after SDK conversion and check advertised validation and actual calls. Individual model APIs can impose additional nesting limits; complete producer captures can exceed ten structural levels. +Self-contained output schemas advertise a content-bound `$id`, including their +declared dialect. SDK validators can reuse compiled schemas across complete +catalog refreshes and equivalent tool outputs; changing the schema changes its +identity. Explicit schema IDs and relative external reference bases are +preserved. This keeps every tool and validation rule in discovery and does not +change the SDK's catalog invalidation or availability checks. + `compare_web_captures` accepts exactly one of two input shapes: - Passive: `before` and `after` each contain `inspection`, the complete diff --git a/src/contracts/toolOutputReferences.test.ts b/src/contracts/toolOutputReferences.test.ts index 70c1f5ea5e..cda49f1d5e 100644 --- a/src/contracts/toolOutputReferences.test.ts +++ b/src/contracts/toolOutputReferences.test.ts @@ -1,11 +1,69 @@ import { Ajv } from "ajv"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv"; import { expect, it } from "vitest"; import { z } from "zod"; import { toolContract } from "./toolContracts.js"; import { toolOutputSchemaWithMetadata } from "./toolSchemaMetadata.js"; +it("keeps dialect-specific validators distinct after JSON schema serialization", () => { + const contract = { + ...toolContract("binary_session"), + outputSchema: z.strictObject({ value: z.number().int().min(1).max(8) }), + }; + const output = toolOutputSchemaWithMetadata(contract); + const provider = new AjvJsonSchemaValidator(); + for (const target of ["draft-2020-12", "draft-07"] as const) { + const projected = output["~standard"].jsonSchema.output({ target }); + const parsed = z + .record(z.string(), z.unknown()) + .parse(JSON.parse(JSON.stringify(projected))); + const validate = provider.getValidator(parsed); + expect(validate({ value: 8 }).valid).toBe(true); + expect(validate({ value: 9 }).valid).toBe(false); + expect(validate({ value: 1.5 }).valid).toBe(false); + } +}); + +it("preserves explicit schema identity and relative reference resolution", () => { + const value = z.string(); + const canonical = z.strictObject({ value }); + const contract = { + ...toolContract("binary_session"), + outputSchema: canonical, + }; + const projected = toolOutputSchemaWithMetadata(contract)[ + "~standard" + ].jsonSchema.output({ + target: "draft-2020-12", + libraryOptions: { + override: (context: { + readonly zodSchema: unknown; + readonly jsonSchema: Record; + }) => { + if (context.zodSchema !== value) return; + delete context.jsonSchema.type; + context.jsonSchema.$ref = "child.json"; + }, + }, + }); + const ajv = new Ajv2020({ strict: false }); + ajv.addSchema({ $id: "child.json", type: "string", minLength: 3 }); + const validate = ajv.compile(projected); + expect(validate({ value: "abc" })).toBe(true); + expect(validate({ value: "ab" })).toBe(false); + + const identified = toolOutputSchemaWithMetadata({ + ...contract, + outputSchema: canonical.meta({ $id: "https://example.test/result" }), + })["~standard"].jsonSchema.output({ target: "draft-2020-12" }); + expect(identified.$id).toBe("https://example.test/result"); + const explicit = ajv.compile(identified); + expect(explicit({ value: "abc" })).toBe(true); + expect(explicit({ value: 1 })).toBe(false); +}); + it.each(["draft-2020-12", "draft-07"] as const)( "preserves refined numeric bounds and strict repeated objects through %s references", (target) => { diff --git a/src/contracts/toolSchemaMetadata.ts b/src/contracts/toolSchemaMetadata.ts index f2d395ece8..8f43f1708c 100644 --- a/src/contracts/toolSchemaMetadata.ts +++ b/src/contracts/toolSchemaMetadata.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { ToolContract } from "./toolContractTypes.js"; import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; +import { digestCanonicalValue } from "../domain/canonicalDigest.js"; const PROPERTY_DESCRIPTIONS: Readonly> = { addresses: "Ordered provider-normalized procedure addresses to analyze.", @@ -69,20 +70,55 @@ export const toolInputSchemaWithMetadata = ( }, ); -/** Share repeated output definitions without changing canonical validation. */ +/** Share output definitions and content-bound validator identities on the wire. */ export const toolOutputSchemaWithMetadata = ( contract: Contract, ): Contract["outputSchema"] => withAdvertisedJsonSchema( contract.outputSchema, "output", - (project) => (options) => - project({ + (project) => (options) => { + const shared = project({ ...options, libraryOptions: { reused: "ref", ...options.libraryOptions }, - }), + }); + // An explicit ID owns the reference base. Relative external references + // also depend on that base, so leave those projections unchanged. + if ("$id" in shared || hasRelativeSchemaReference(shared)) return shared; + return { + ...shared, + $id: `urn:rea:tool-output:sha256:${digestCanonicalValue( + { target: options.target, schema: shared }, + "Tool output schema", + )}`, + }; + }, ); +const hasRelativeSchemaReference = ( + schema: Record, +): boolean => { + const pending: unknown[] = [schema]; + const visited = new WeakSet(); + while (pending.length > 0) { + const current = pending.pop(); + if (current === null || typeof current !== "object" || visited.has(current)) + continue; + visited.add(current); + for (const [key, value] of Object.entries(current)) { + if ( + ["$id", "$ref", "$dynamicRef", "$recursiveRef"].includes(key) && + typeof value === "string" && + !value.startsWith("#") && + !URL.canParse(value) + ) + return true; + if (value !== null && typeof value === "object") pending.push(value); + } + } + return false; +}; + const withAdvertisedJsonSchema = ( canonical: Schema, io: "input" | "output", diff --git a/tests/boundary/mcp/outputValidatorReuse.test.ts b/tests/boundary/mcp/outputValidatorReuse.test.ts new file mode 100644 index 0000000000..256c29d588 --- /dev/null +++ b/tests/boundary/mcp/outputValidatorReuse.test.ts @@ -0,0 +1,108 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { McpServer } from "@modelcontextprotocol/server"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { toolOutputSchemaWithMetadata } from "../../../src/contracts/toolSchemaMetadata.js"; + +const advertised = < + Schema extends z.ZodType>>, +>( + schema: Schema, +): Schema => + toolOutputSchemaWithMetadata({ + ...toolContract("binary_session"), + outputSchema: schema, + }); + +const connect = async (server: McpServer, client: Client): Promise => { + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); +}; + +it("validates changed schemas after catalog refreshes and availability changes", async () => { + const client = new Client({ name: "output-reuse-test", version: "1" }); + const server = new McpServer({ name: "output-reuse-fixture", version: "1" }); + const schema = advertised(z.strictObject({ state: z.literal("ready") })); + let state = "ready"; + const selected = server.registerTool( + "selected", + { inputSchema: z.strictObject({}), outputSchema: schema }, + () => ({ content: [], structuredContent: { state } }), + ); + server.registerTool( + "equivalent", + { inputSchema: z.strictObject({}), outputSchema: schema }, + () => ({ content: [], structuredContent: { state: "ready" } }), + ); + try { + await connect(server, client); + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ + state: "ready", + }); + await client.listTools(); + await client.callTool({ name: "equivalent" }); + + selected.update({ + outputSchema: advertised(z.strictObject({ state: z.literal("updated") })), + }); + state = "updated"; + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ + state: "updated", + }); + selected.disable(); + const unavailable = await client.listTools(); + expect(unavailable.tools.some((tool) => tool.name === "selected")).toBe( + false, + ); + await expect(client.callTool({ name: "selected" })).rejects.toThrow( + /not found|disabled/iu, + ); + await client.callTool({ name: "equivalent" }); + selected.enable(); + await client.listTools(); + await client.callTool({ name: "selected" }); + } finally { + await Promise.allSettled([client.close(), server.close()]); + } +}); + +it("uses the new server's schema after reconnecting the client", async () => { + const client = new Client({ name: "output-reconnect-test", version: "1" }); + const servers: McpServer[] = []; + try { + for (const state of ["first", "second"]) { + const server = new McpServer({ name: `fixture-${state}`, version: "1" }); + servers.push(server); + server.registerTool( + "selected", + { + inputSchema: z.strictObject({}), + outputSchema: advertised(z.strictObject({ state: z.literal(state) })), + }, + () => ({ content: [], structuredContent: { state } }), + ); + await connect(server, client); + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ state }); + await client.close(); + await server.close(); + } + } finally { + await Promise.allSettled([ + client.close(), + ...servers.map((server) => server.close()), + ]); + } +}); From 0a14ca41b46e42d36d46ba07384ff13a7a97456d Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:00:11 +0800 Subject: [PATCH 22/74] fix(javascript): preserve effects through yielded array references (#1437) --- .../javascript/javascriptSemanticAnalysis.ts | 20 ++- .../javascriptSemanticIterationSources.ts | 12 +- .../javascriptSemanticMemberMutations.ts | 15 +- .../javascriptIterationAliases.test.ts | 145 ++++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 tests/acceptance/applications/javascriptIterationAliases.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 5a873429e1..1cf53fb313 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -565,9 +565,21 @@ const bindAssignment = ( else if (t.isForOfStatement(node) || t.isForInStatement(node)) for (const pattern of t.isVariableDeclaration(node.left) ? node.left.declarations.map(({ id }) => id) - : [node.left]) + : [node.left]) { for (const identifier of assignedPatternIdentifiers(pattern)) addAssignment(identifier, node, scope, state); + if (t.isForOfStatement(node)) + // Keep the loop value unknown, but preserve references yielded into + // each bound slot so writes/escapes can reach their iterable origins. + bindPattern({ + pattern, + initializer: node, + scope, + state, + mutable: true, + referenceOnly: true, + }); + } }; const assignedPatternIdentifiers = ( @@ -780,7 +792,11 @@ const bindPattern = (input: BindPatternInput): void => { } if (t.isIdentifier(pattern)) { if (referenceOnly) { - const binding = scope.bindings.get(pattern.name); + const binding = resolveSemanticBindingFromScope( + scope, + pattern.name, + state, + ); if (binding !== undefined && initializer !== null) binding.referenceInitializers.push({ node: initializer, diff --git a/src/domain/javascript/javascriptSemanticIterationSources.ts b/src/domain/javascript/javascriptSemanticIterationSources.ts index 1b9e7d7143..c83eb94ad9 100644 --- a/src/domain/javascript/javascriptSemanticIterationSources.ts +++ b/src/domain/javascript/javascriptSemanticIterationSources.ts @@ -8,8 +8,9 @@ interface IterationSourceValue { readonly projection: readonly (string | number | null)[]; } -/** An implicit iterator receiver, with guards for destructuring defaults. */ +/** An iterator receiver or yielded reference store, with default guards. */ export interface SemanticIterationSource extends IterationSourceValue { + readonly kind: "iteration" | "reference-store"; readonly mutation: t.Node; readonly fallbackSources?: readonly IterationSourceValue[]; readonly requiredSources?: readonly IterationSourceValue[]; @@ -30,8 +31,10 @@ const add = ( source: IterationSourceValue, mutation: t.Node, guards: SourceGuards, + kind: SemanticIterationSource["kind"] = "iteration", ): void => { sources.push({ + kind, node: source.node, projection: source.projection, mutation, @@ -66,6 +69,13 @@ const visit = ( guards: SourceGuards = {}, ): void => { const { sources, mutation } = context; + if (t.isForOfStatement(mutation) && t.isMemberExpression(pattern)) { + // Storing a yielded reference in a property has no lexical loop binding. + // Preserve its escape rather than assuming later writes cannot reach it. + if (source !== undefined) + add(sources, source, mutation, guards, "reference-store"); + return; + } if (t.isTSParameterProperty(pattern)) { visit(context, pattern.parameter, source, guards); return; diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 21a6aded04..4e4ea7b8b2 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -324,7 +324,7 @@ export const collectSemanticMemberMutations = ( bindings: new Set(), effect: "escape", originAt: source.mutation, - iterationOnly: true, + iterationOnly: source.kind === "iteration", }); if (t.isAssignmentExpression(node)) markTarget(node.left, node); else if (t.isUpdateExpression(node)) markTarget(node.argument, node); @@ -369,6 +369,19 @@ export const collectSemanticMemberMutations = ( const { initializer } = reference; if (referenceSourcesInactive(initializer, state)) pendingReferences.push(reference); + else if (t.isForOfStatement(initializer.node)) + deferIterable({ + node: initializer.node.right, + projection: [], + path: [null, ...reference.path], + fallbackPath: [], + bindings: reference.bindings, + effect: reference.effect, + originAt: initializer.node, + ...(reference.mutation === undefined + ? {} + : { mutation: reference.mutation }), + }); else if ( initializer.copyKind === "array-rest" && initializer.copyProjectionOffset !== undefined diff --git a/tests/acceptance/applications/javascriptIterationAliases.test.ts b/tests/acceptance/applications/javascriptIterationAliases.test.ts new file mode 100644 index 0000000000..1039954806 --- /dev/null +++ b/tests/acceptance/applications/javascriptIterationAliases.test.ts @@ -0,0 +1,145 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { projectedExportReturnShapesSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const source = [ + "export function inline() { const shared = { x: 1, keep: 7 }; for (const t of [shared]) t.x = 2; return { x: shared.x, keep: shared.keep }; }", + "export function named() { const shared = { x: 1 }; const arr = [shared]; for (const t of arr) t.x = 2; return shared.x; }", + "export function declaredVar() { const shared = { x: 1 }; for (var t of [shared]) t.x = 2; return shared.x; }", + "export function assigned() { const shared = { x: 1 }; let t; for (t of [shared]) t.x = 2; return shared.x; }", + "export function assignedDestructured() { const shared = { x: 1 }; let t; for ({child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function declaredVarDestructured() { const shared = { x: 1 }; for (var {child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function memberTarget() { const shared = { x: 1 }; const holder = {}; for (holder.item of [shared]) holder.item.x = 2; return shared.x; }", + "export function memberPattern() { const shared = { x: 1 }; const holder = {}; for ({child: holder.item} of [{child: shared}]) holder.item.x = 2; return shared.x; }", + "export function destructured() { const shared = { x: 1 }; for (const {child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function arrayDestructured() { const shared = { x: 1 }; for (const [t] of [[shared]]) t.x = 2; return shared.x; }", + "export function spread() { const shared = { x: 1 }; const arr = [shared]; for (const t of [...arr]) t.x = 2; return shared.x; }", + "export function objectRestChild() { const shared = { x: 1 }; for (const {...t} of [{child: shared}]) t.child.x = 2; return shared.x; }", + "export function arrayRestChild() { const shared = { x: 1 }; for (const [...t] of [[shared]]) t[0].x = 2; return shared.x; }", + "export function defaultChild() { const shared = { x: 1 }; for (const {child: t = shared} of [{}]) t.x = 2; return shared.x; }", + "export function updated() { const shared = { x: 1 }; for (const t of [shared]) t.x++; return shared.x; }", + "export function deleted() { const shared = { x: 1 }; for (const t of [shared]) delete t.x; return shared.x; }", + "export function escaped() { const shared = { x: 1 }; for (const t of [shared]) consume(t); return shared.x; }", + "export async function awaited() { const shared = { x: 1 }; for await (const t of [shared]) t.x = 2; return shared.x; }", + "export function indexed() { const shared = { x: 1 }; const arr = [shared]; for (let i = 0; i < arr.length; i++) { const t = arr[i]; t.x = 2; } return shared.x; }", + "export function copyWrite() { const shared = { x: 1 }; for (const {...t} of [shared]) t.x = 2; return shared.x; }", + "export function noWrite() { const shared = { x: 1 }; for (const t of [shared]) { const read = t.x; } return shared.x; }", + "export function empty() { const shared = { x: 1 }; for (const t of []) t.x = 2; return shared.x; }", + "export function keys() { const shared = { x: 1 }; for (const key in [shared]) { const read = key; } return shared.x; }", + "export function shadowed() { const t = { x: 1 }; const shared = { x: 3 }; for (const t of [shared]) t.x = 2; return t.x; }", +].join("\n"); + +const assertReturns = (value: unknown): void => { + const analysis = javascriptApplicationAnalysisResultSchema.parse(value); + const projections = analysis.graph.nodes + .flatMap(({ observations }) => observations) + .filter( + ({ properties }) => properties.semantic_role === "export-return-shapes", + ) + .map(({ properties }) => + projectedExportReturnShapesSchema.parse(properties), + ); + const fields = (name: string) => { + const projection = projections.find( + ({ exported_name }) => exported_name === name, + ); + if (projection === undefined) throw new Error(`Missing export ${name}`); + expect(projection.return_shape_coverage).toMatchObject({ + status: "complete", + projection_complete: true, + }); + return projection.static_return_shapes.flatMap(({ fields }) => fields); + }; + expect(fields("inline")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "/x", state: "unknown" }), + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ]), + ); + for (const name of [ + "named", + "declaredVar", + "assigned", + "assignedDestructured", + "declaredVarDestructured", + "memberTarget", + "memberPattern", + "destructured", + "arrayDestructured", + "spread", + "objectRestChild", + "arrayRestChild", + "defaultChild", + "updated", + "deleted", + "escaped", + "awaited", + "indexed", + ]) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "unknown" }), + ); + for (const name of ["copyWrite", "noWrite", "empty", "keys", "shadowed"]) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "literal", value: 1 }), + ); +}; + +cliTest( + "preserves yielded-object mutation uncertainty through CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-iteration-aliases-"); + await writeFile(join(root, "app.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(cliResponse.exitCode).toBe(0); + const contract = toolContract("analyze_javascript_application"); + const cliEvidence = contract.outputSchema.parse(cliResponse.json); + assertReturns(cliEvidence.normalized_result); + const client = new Client({ name: "iteration-aliases-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = contract.outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + assertReturns(evidence.normalized_result); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + assertReturns( + contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, + ); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } + }, + 120_000, +); From ce16f7bc951180b68707fc8cec7e20a0511bb5b3 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:05:12 +0800 Subject: [PATCH 23/74] test(native): verify macOS provenance through real CLI and MCP Verify five native operations through the public CLI and real SDK stdio MCP using a source-built macOS fixture. Retain normalized provenance, response parity and unchanged input bytes. Refs #1430, #1065, #723. --- scripts/verify/apple/macos-bundle.mjs | 3 + scripts/verify/apple/native-provenance.mjs | 71 ++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 scripts/verify/apple/native-provenance.mjs diff --git a/scripts/verify/apple/macos-bundle.mjs b/scripts/verify/apple/macos-bundle.mjs index c10abcb2e0..e2a1308421 100644 --- a/scripts/verify/apple/macos-bundle.mjs +++ b/scripts/verify/apple/macos-bundle.mjs @@ -17,6 +17,7 @@ import { import { verifyDyldEnvironment } from "./dyld-environment.mjs"; import { verifyDylibResolution } from "./macos-dylib-e2e.mjs"; import { verifyMachOOccurrences } from "./macho-occurrences.mjs"; +import { verifyNativeProvenance } from "./native-provenance.mjs"; const exec = promisify(execFile); @@ -69,6 +70,7 @@ const EXPECTED_BUNDLES = [ let report; try { const { app, executable } = await buildMacosBundleFixture(root); + const nativeProvenance = await verifyNativeProvenance(app, executable); const machoOccurrences = await verifyMachOOccurrences(app, executable); const zip = join(root, "MacFixture.zip"); await exec("/usr/bin/ditto", ["-c", "-k", "--keepParent", app, zip]); @@ -166,6 +168,7 @@ try { containers: containers.map(({ format }) => format), bundles: EXPECTED_BUNDLES.length, dmg_detached: true, + native_provenance: nativeProvenance, macho_occurrences: machoOccurrences, dylib_resolution: dylibs, dyld_environment: dyldEnvironment, diff --git a/scripts/verify/apple/native-provenance.mjs b/scripts/verify/apple/native-provenance.mjs new file mode 100644 index 0000000000..ad7bf6cc13 --- /dev/null +++ b/scripts/verify/apple/native-provenance.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +import { toolContract } from "../../../dist/contracts/toolContracts.js"; +import { + artifactCliEvidence, + withArtifactMcp, +} from "../../lib/artifact-e2e.mjs"; + +/** Inspect the compiled bundle with real native utilities through both adapters. */ +export async function verifyNativeProvenance(app, executable) { + const original = await readFile(executable); + const sha256 = createHash("sha256").update(original).digest("hex"); + const symbols = ["$s4main3FooV", "plain_symbol"]; + const operations = [ + ["inspect_macho", []], + ["inspect_signature", []], + ["inspect_plist", []], + ["list_architectures", []], + ["demangle_swift", symbols], + ]; + const observations = []; + await withArtifactMcp(app, async (client) => { + const definitions = new Map( + (await client.listTools()).tools.map((tool) => [tool.name, tool]), + ); + for (const [operation, arguments_] of operations) { + const cli = toolContract(operation).outputSchema.parse( + await artifactCliEvidence( + operation.replaceAll("_", "-"), + app, + arguments_, + ), + ); + const response = await client.callTool( + { + name: operation, + arguments: operation === "demangle_swift" ? { symbols } : {}, + }, + { toolDefinition: definitions.get(operation) }, + ); + assert.notEqual(response.isError, true, JSON.stringify(response)); + const mcp = toolContract(operation).outputSchema.parse( + response.structuredContent, + ); + assert.equal(cli.raw_result, null); + assert.equal(mcp.raw_result, null); + assert.ok(cli.normalized_result.provenance.length > 0); + assert.deepEqual(mcp.normalized_result, cli.normalized_result); + assert.deepEqual( + JSON.parse(response.content.find((item) => item.type === "text").text), + response.structuredContent, + ); + await client.ping(); + observations.push({ + operation, + commands: cli.normalized_result.provenance.length, + cli_bytes: Buffer.byteLength(JSON.stringify(cli)), + mcp_bytes: Buffer.byteLength(JSON.stringify(response)), + }); + } + }); + assert.deepEqual(await readFile(executable), original); + return { + real_native_utilities: true, + target_sha256: sha256, + bytes_unchanged: true, + observations, + }; +} From bb6fa1c6f500c6b6af42f1e3f72bd42c1dce91f6 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:13:05 +0800 Subject: [PATCH 24/74] docs(process): verify independent full-report hashing budgets (#1440) --- docs/process-capture.md | 46 +++++++- .../processFilesystemCoverageAdapters.test.ts | 104 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index 21f1f5343f..1d3d23e7fb 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -159,8 +159,11 @@ Trace assertions use coverage for the sources they select, so an assertion about complete raw terminal output need not fail because rendered snapshots were omitted. Captures missing `truncation_details` are rejected on import; preserve them as historical files and recapture for current comparisons. -These diagnostics use the existing scenario -budgets; they do not introduce a separate file-hashing budget. +`limits.file_bytes` is already the cumulative whole-file hashing budget, +independent of `limits.output_bytes`, which bounds retained terminal content. +Each filesystem checkpoint starts with a fresh `file_bytes` budget across all +selected roots; it is not a per-file allowance. File contents are read in +64 KiB chunks for hashing and are not embedded in capture Evidence. When the host withholds an unrelated process’s ownership token, REA leaves that process untouched and records its PID and reason in `cleanup.unverified_processes` @@ -176,6 +179,45 @@ as well. A newly started unreadable process can prevent verified cleanup even after the selected command exits. REA preserves this uncertainty and leaves that process untouched. +## Hash a report independently of terminal output + +Select the producer's report directory and give `file_bytes` enough room for +the complete files you need, including other files in those roots. For example, +save this scenario and run `rea capture-process ./scenario.json --json > capture.json`: + +```json +{ + "executable": "node", + "arguments": [ + "-e", + "const fs=require('node:fs');fs.mkdirSync('reports',{recursive:true});fs.writeFileSync('reports/final.json',JSON.stringify({status:'complete',data:'X'.repeat(131072)}));console.log('x'.repeat(8192));" + ], + "working_directory": ".", + "filesystem_observation_paths": ["./reports"], + "limits": { "output_bytes": 1024, "file_bytes": 262144 } +} +``` + +The report exceeds the terminal budget but fits the independent 256 KiB hash +budget. Its complete SHA-256 appears in `normalized_result.files_after` at +`root_0:final.json`; check `truncation_details.filesystem_after` for omissions +and actual bytes hashed. The verbose terminal output is truncated without +preventing the report digest. The same scenario object is the MCP +`capture_process_scenario` input. REA never returns a prefix hash as a full-file +digest: an insufficient remaining budget produces a null digest and an +explicit omission reason. + +A digest binds the complete bytes observed at that checkpoint, not the +producer's notion of a finished report. A stable periodic or running report +can also have a complete digest. Choose the final file and verify the +producer's documented completion marker, exit outcome and settlement +observations before calling it final. `after_settlement` names a filesystem +checkpoint, not a guarantee that an application finished successfully or +that an unobserved writer will never change the file. REA checks opened-file +identity/state around hashing and reports observed changes or unavailable +files; cancellation stops the hash. If an external sealer produces a manifest, +keep its completion claims and digest separate from REA's checkpoint facts. + ## Compare two captures Compare saved capture Evidence with: diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts index ec7ce5b4c0..ff3b118a73 100644 --- a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -7,6 +8,7 @@ import { expect, it, onTestFinished } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; +import { parseProcessScenario } from "../../../src/domain/process/processScenario.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -58,6 +60,108 @@ const captureViaMcp = async (scenario: Record) => { return evidence; }; +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "hashes the documented complete report despite omitted terminal output through %s", + async (adapter) => { + const document = await readFile("docs/process-capture.md", "utf8"); + const example = document + .split("## Hash a report independently of terminal output")[1] + ?.match(/```json\n([\s\S]*?)\n```/u)?.[1]; + if (example === undefined) + throw new Error("Missing report capture example"); + const scenario = parseProcessScenario(JSON.parse(example)); + const root = await createTestTempDirectory("rea-full-report-adapter-"); + const evidence = await (adapter === "cli" ? captureViaCli : captureViaMcp)({ + ...scenario, + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [join(root, "reports")], + }); + const capture = parseProcessCapture(evidence.normalized_result); + const bytes = await readFile(join(root, "reports", "final.json")); + expect(JSON.parse(bytes.toString())).toEqual({ + status: "complete", + data: "X".repeat(131072), + }); + expect(bytes.length).toBeGreaterThan(scenario.limits.output_bytes); + expect(capture.files_after).toContainEqual( + expect.objectContaining({ + path: "root_0:final.json", + size: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }), + ); + expect(capture.truncation_details.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hash_budget_bytes: scenario.limits.file_bytes, + hashed_bytes: bytes.length, + hash_omissions: [], + }); + expect(capture.truncation_details.raw_terminal.retained_bytes).toBeLessThan( + capture.truncation_details.raw_terminal.observed_bytes, + ); + }, + 20_000, +); + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "accounts for exact cumulative hash budgets independently at each %s checkpoint", + async (adapter) => { + const runCapture = adapter === "cli" ? captureViaCli : captureViaMcp; + for (const shortfall of [0, 1]) { + const root = await createTestTempDirectory("rea-report-cumulative-"); + const periodic = Buffer.from('{"status":"running"}'); + const final = Buffer.from('{"status":"complete"}'); + await writeFile(join(root, "periodic.json"), periodic); + const budget = periodic.length + final.length - shortfall; + const evidence = await runCapture({ + executable: process.execPath, + arguments: [ + "-e", + "require('node:fs').writeFileSync('final.json','{\"status\":\"complete\"}')", + ], + working_directory: root, + filesystem_observation_paths: [root], + limits: { output_bytes: 1, file_bytes: budget }, + }); + const capture = parseProcessCapture(evidence.normalized_result); + expect(capture.truncation_details.filesystem_before).toMatchObject({ + hashed_bytes: periodic.length, + hash_omissions: [], + }); + expect(capture.files_after).toContainEqual( + expect.objectContaining({ + path: "root_0:final.json", + sha256: createHash("sha256").update(final).digest("hex"), + }), + ); + expect(capture.truncation_details.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hashed_bytes: final.length + (shortfall === 0 ? periodic.length : 0), + hash_omissions: + shortfall === 0 + ? [] + : [ + { + path: "root_0:periodic.json", + reason: "file_bytes_budget", + remaining_budget_bytes: periodic.length - 1, + }, + ], + }); + expect( + capture.files_after.find(({ path }) => path === "root_0:periodic.json") + ?.sha256, + ).toBe( + shortfall === 0 + ? createHash("sha256").update(periodic).digest("hex") + : null, + ); + } + }, + 20_000, +); + itWithCaptureCapability.each(["cli", "mcp"] as const)( "reports unknown absence instead of deletion in truncated %s capture Evidence", async (adapter) => { From eeb83f65817819644af8e6fbf8a120227dfb88d4 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:16:44 -0700 Subject: [PATCH 25/74] perf(cli): defer Playwright and isomorphic-git until a command needs them Load browser composition after CLI scenario validation and load Git support only for local reference trees with Git metadata. Preserve default outputs and actual command behavior. Refs #1065. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- src/application/ReferenceSourceVcsAdapter.ts | 4 +- src/cliBrowserScenarioCommands.ts | 4 +- tests/boundary/cli/cliStartupImports.test.ts | 76 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 tests/boundary/cli/cliStartupImports.test.ts diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index 42ab4e7388..3ddad25d71 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -2,8 +2,6 @@ import fs from "node:fs"; import { lstat, readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; -import { resolveRef } from "isomorphic-git"; - export type ReferenceSourceVcsInfo = | { readonly kind: "git"; @@ -42,6 +40,8 @@ export const readReferenceSourceVcs = async ( }; const resolveSourceHead = async (root: string): Promise => { + // isomorphic-git is loaded on first use so CLI and MCP startup skip it. + const { resolveRef } = await import("isomorphic-git"); const dotgit = join(root, ".git"); if (!(await lstat(dotgit)).isFile()) return resolveRef({ fs, dir: root, ref: "HEAD" }); diff --git a/src/cliBrowserScenarioCommands.ts b/src/cliBrowserScenarioCommands.ts index f98e415cb2..a1a1f86e15 100644 --- a/src/cliBrowserScenarioCommands.ts +++ b/src/cliBrowserScenarioCommands.ts @@ -2,7 +2,6 @@ import { Cli } from "incur"; import { z } from "zod"; import { captureBrowserScenario } from "./application/BrowserScenarioCaptureService.js"; -import { createBrowserScenarioProvider } from "./composition/browserScenario.js"; import { CLI_COMMANDS } from "./cliCommandNames.js"; import { parseCliJsonInput, resolveCliJsonPaths } from "./cliJsonInput.js"; import { logCliCommand } from "./cliLogging.js"; @@ -44,6 +43,9 @@ export const registerBrowserScenarioCommands = ( projectInputIssues(scenario.error.issues, input.value), ), ); + // Load Playwright only for this command, not during CLI startup. + const { createBrowserScenarioProvider } = + await import("./composition/browserScenario.js"); const result = await captureBrowserScenario( createBrowserScenarioProvider(environment), scenario.data, diff --git a/tests/boundary/cli/cliStartupImports.test.ts b/tests/boundary/cli/cliStartupImports.test.ts new file mode 100644 index 0000000000..c084b62122 --- /dev/null +++ b/tests/boundary/cli/cliStartupImports.test.ts @@ -0,0 +1,76 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const MARKER = "REA_TEST_RESOLVED "; +// Dependencies that only one command uses; startup must not resolve them. +const DEFERRED_PACKAGES = ["playwright-core", "isomorphic-git"] as const; +// Report each resolved deferred package on stderr without changing resolution. +const RESOLUTION_HOOK = `data:text/javascript,${encodeURIComponent(` +import { registerHooks } from "node:module"; +const packages = ${JSON.stringify(DEFERRED_PACKAGES)}; +registerHooks({ + resolve(specifier, context, next) { + const resolved = next(specifier, context); + const name = packages.find((item) => + resolved.url.includes("/node_modules/" + item + "/"), + ); + if (name !== undefined) + process.stderr.write(${JSON.stringify(MARKER)} + name + "\\n"); + return resolved; + }, +}); +`)}`; + +const resolvedPackages = async ( + arguments_: readonly string[], +): Promise> => { + const { stderr } = await execute( + process.execPath, + [`--import=${RESOLUTION_HOOK}`, ...arguments_], + { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 }, + ); + return new Set( + stderr + .split("\n") + .filter((line) => line.startsWith(MARKER)) + .map((line) => line.slice(MARKER.length)), + ); +}; + +const evaluate = (source: string): readonly string[] => [ + "--input-type=module", + "--eval", + source, +]; + +describe("CLI startup imports", () => { + it("detects each deferred package when its command code loads", async () => { + await expect( + resolvedPackages( + evaluate('await import("./dist/composition/browserScenario.js");'), + ), + ).resolves.toContain("playwright-core"); + const source = await createTestTempDirectory("rea-startup-imports-"); + await mkdir(join(source, ".git")); + await expect( + resolvedPackages( + evaluate( + `const { readReferenceSourceVcs } = await import("./dist/application/ReferenceSourceVcsAdapter.js"); await readReferenceSourceVcs(${JSON.stringify(source)});`, + ), + ), + ).resolves.toContain("isomorphic-git"); + }); + + it("does not load command-specific packages to register commands", async () => { + await expect( + resolvedPackages(["scripts/rea.mjs", "--help"]), + ).resolves.toEqual(new Set()); + }); +}); From 33b9b31a6fb50cc3e6f4591d0395dbba15b51f13 Mon Sep 17 00:00:00 2001 From: keemsisi Date: Sat, 10 Oct 2026 06:21:13 +0100 Subject: [PATCH 26/74] perf(javascript): index change-graph nodes once per comparison Resolve each compared graph node through a per-side index while preserving right-hand evidence selection and complete comparison results. Refs #1065. Co-authored-by: keemsisi <7439375+keemsisi@users.noreply.github.com> --- .../javascriptApplicationChangeGraph.test.ts | 107 ++++++++++++++++++ .../javascriptApplicationChangeGraph.ts | 25 ++-- 2 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 src/domain/javascript/javascriptApplicationChangeGraph.test.ts diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts new file mode 100644 index 0000000000..2bbd653af5 --- /dev/null +++ b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; + +import { + createJavaScriptApplicationEdge, + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, + type JavaScriptApplicationGraph, +} from "./javascriptApplicationGraph.js"; +import { + artifactEvidence, + completeCoverage, +} from "./javascriptApplicationGraph.fixture.js"; +import { compareJavaScriptApplicationVersions } from "./javascriptApplicationVersionComparison.js"; + +const LEFT_SHA = "a".repeat(64); +const RIGHT_SHA = "b".repeat(64); + +const digest = (value: number): string => value.toString(16).padStart(64, "0"); + +const buildGraph = ( + artifactSha: string, + changed: number, +): JavaScriptApplicationGraph => { + const nodes = ["a", "b", "c", "d"].map((name, index) => + createJavaScriptApplicationNode({ + kind: "javascript-module", + identity: { + strategy: "canonical-path", + stability: "artifact-version", + artifact_sha256: artifactSha, + path: `${name}.js`, + }, + observations: [ + { + label: `${name}.js`, + properties: { + structural_fingerprint_sha256: digest(index + 1), + structural_fingerprint_algorithm: "fixture", + source_sha256: digest(index === 1 || index === 3 ? changed : 0), + }, + evidence: artifactEvidence(artifactSha, `${name}.js`), + }, + ], + }), + ); + const [a, b] = nodes; + if (a === undefined || b === undefined) + throw new Error("Missing fixture node"); + return createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [a.node_id], + nodes, + edges: [ + createJavaScriptApplicationEdge({ + source_node_id: a.node_id, + target_node_id: b.node_id, + relation: "imports", + properties: {}, + evidence: artifactEvidence(artifactSha, "a.js"), + }), + ], + coverage: completeCoverage, + limitations: [], + }); +}; + +describe("application change graph", () => { + it("anchors each changed_from edge to its own right-hand observation", () => { + const left = buildGraph(LEFT_SHA, 1); + const right = buildGraph(RIGHT_SHA, 2); + const result = compareJavaScriptApplicationVersions({ + left: { + evidenceId: `ev_${"1".repeat(64)}`, + rootArtifactSha256: LEFT_SHA, + graph: left, + }, + right: { + evidenceId: `ev_${"2".repeat(64)}`, + rootArtifactSha256: RIGHT_SHA, + graph: right, + }, + leftNativeEvidence: [], + rightNativeEvidence: [], + }); + const changedFrom = result.graph.edges.filter( + ({ relation }) => relation === "changed_from", + ); + expect(changedFrom).toHaveLength(2); + for (const edge of changedFrom) { + const source = right.nodes.find( + ({ node_id: id }) => id === edge.source_node_id, + ); + const target = left.nodes.find( + ({ node_id: id }) => id === edge.target_node_id, + ); + expect(source?.observations[0]?.label).toBe( + target?.observations[0]?.label, + ); + expect(edge.evidence.artifact).toEqual( + source?.observations[0]?.evidence.artifact, + ); + expect(edge.evidence.location).toEqual( + source?.observations[0]?.evidence.location, + ); + } + }); +}); diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.ts b/src/domain/javascript/javascriptApplicationChangeGraph.ts index 4c80b32045..134fff5200 100644 --- a/src/domain/javascript/javascriptApplicationChangeGraph.ts +++ b/src/domain/javascript/javascriptApplicationChangeGraph.ts @@ -53,8 +53,14 @@ export const buildJavaScriptApplicationChangeGraph = ( ({ source_node_id: source, target_node_id: target }) => retained.has(source) && retained.has(target), ); + const leftById = new Map( + input.left.nodes.map((node) => [node.node_id, node]), + ); + const rightById = new Map( + input.right.nodes.map((node) => [node.node_id, node]), + ); const comparisonEdges = input.items.flatMap((item) => - changedFromEdge(item, input, retained), + changedFromEdge(item, input, retained, leftById, rightById), ); const candidateEdges = uniqueEdges([...sourceEdges, ...comparisonEdges]); const rootNodeIds = preferredRoots.filter((nodeId) => retained.has(nodeId)); @@ -104,6 +110,8 @@ const changedFromEdge = ( item: ApplicationVersionComparisonItem, input: ChangeGraphInput, retained: ReadonlySet, + leftById: ReadonlyMap, + rightById: ReadonlyMap, ): ApplicationEdge[] => { const left = item.left_node_id; const right = item.right_node_id; @@ -117,7 +125,12 @@ const changedFromEdge = ( !retained.has(right) ) return []; - const evidence = comparisonEvidence(item, input); + const evidence = comparisonEvidence( + item, + input, + leftById.get(left), + rightById.get(right), + ); return [ createJavaScriptApplicationEdge({ source_node_id: right, @@ -138,13 +151,9 @@ const changedFromEdge = ( const comparisonEvidence = ( item: ApplicationVersionComparisonItem, input: ChangeGraphInput, + left: ApplicationNode | undefined, + right: ApplicationNode | undefined, ): ApplicationGraphEvidence => { - const right = input.right.nodes.find( - ({ node_id: id }) => id === item.right_node_id, - ); - const left = input.left.nodes.find( - ({ node_id: id }) => id === item.left_node_id, - ); const source = right?.observations[0]?.evidence ?? left?.observations[0]?.evidence; const complete = From 7633dd8d5dcb94eb5c2fca8b69858949c51b0eaf Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:01 +0800 Subject: [PATCH 27/74] fix(ghidra): publish Windows verification success after cleanup --- docs/native-investigation.md | 2 +- scripts/verify-packaged-ghidra-windows.mjs | 31 +++++++++++++--------- src/ghidra/GhidraProvider.behavior.test.ts | 26 ++++++++++++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 4dad23b3b1..b3051cc647 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -379,7 +379,7 @@ the tool fails with a `debugger-attach-denied` reason. Install a Ghidra 12.1.x release and the 64-bit full JDK it declares, then configure REA to use them. Ghidra analysis supports Linux x64/arm64 and macOS x64/arm64; each installation requires the matching native decompiler. Experimental Windows x64 P0 admits native -x86-64 PE applications on local NTFS using bundled Job Object ownership, +x86 and x86-64 PE applications and DLLs on local NTFS using bundled Job Object ownership, protected runtime DACLs, and handle-based path admission. See [Windows Ghidra P0](windows-ghidra-p0.md) and [issue #527](https://github.com/morluto/rea/issues/527). On Linux and macOS, `annotate_native_function` atomically edits a function name diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 268d949d9e..be21fde120 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -26,12 +26,21 @@ if (process.platform !== "win32" || process.arch !== "x64") ); const exec = promisify(execFile); const arguments_ = process.argv.slice(2); +const internalWorkspace = arguments_[0] === "--internal-workspace"; +const fixtureArguments = internalWorkspace ? arguments_.slice(2) : arguments_; +const x86Fixture = fixtureArguments.includes("--x86"); +const dllFixture = fixtureArguments.includes("--dll"); +const [packageRootArgument, targetArgument, reportArgument] = + fixtureArguments.filter( + (argument) => argument !== "--x86" && argument !== "--dll", + ); // Windows locks the loaded package-owned addon until the verifier exits. // Keep installation cleanup in a parent that never loads that addon. -if (arguments_[0] !== "--internal-workspace") { +if (!internalWorkspace) { const ownedWorkspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); + let result; try { - const result = await exec( + result = await exec( process.execPath, [ fileURLToPath(import.meta.url), @@ -41,8 +50,6 @@ if (arguments_[0] !== "--internal-workspace") { ], { timeout: 900_000, maxBuffer: 8 * 1024 * 1024 }, ); - process.stdout.write(result.stdout); - process.stderr.write(result.stderr); } catch (cause) { if (typeof cause.stdout === "string") process.stdout.write(cause.stdout); if (typeof cause.stderr === "string") process.stderr.write(cause.stderr); @@ -50,17 +57,17 @@ if (arguments_[0] !== "--internal-workspace") { } finally { await rm(ownedWorkspace, { recursive: true, force: true }); } + const report = JSON.parse(result.stdout); + report.workspaceCleanup = true; + report.ok = true; + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); + process.stdout.write(`${JSON.stringify(report)}\n`); + process.stderr.write(result.stderr); process.exit(0); } assert.ok(arguments_[1] !== undefined, "Missing verifier workspace."); const workspace = resolve(arguments_[1]); -const fixtureArguments = arguments_.slice(2); -const x86Fixture = fixtureArguments.includes("--x86"); -const dllFixture = fixtureArguments.includes("--dll"); -const [packageRootArgument, targetArgument, reportArgument] = - fixtureArguments.filter( - (argument) => argument !== "--x86" && argument !== "--dll", - ); let packageRoot = resolve(packageRootArgument ?? "."); // The default lane verifies the npm artifact in an isolated prefix. Explicit // package roots support an already-installed artifact without a second install. @@ -402,7 +409,7 @@ try { assert.deepEqual(await readdir(join(callerDirectory, name)), []); report.callerScriptCollisionsPreserved = true; report.runtimeCleanup = true; - report.ok = true; + // Only the parent can attest success after removing the installation. if (reportArgument !== undefined) await writeFile(reportArgument, `${JSON.stringify(report)}\n`); process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 88e1eb07fe..c607b99508 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -198,6 +198,32 @@ describe("Ghidra Mach-O slice support", () => { }); describe("Ghidra platform support", () => { + it("commits the PE role and managed classification to profile identity", async () => { + const ghidra = provider({ ...installationHost(), platform: "win32" }); + const application = peTarget("x86_64"); + const library = { + ...application, + executableRole: "shared-library", + } as const; + const digests = new Set(); + for (const target of [ + application, + library, + { ...library, managed: true }, + ]) { + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + const profile = resolved.value.profile; + if (profile === null) throw new Error("Expected a PE analysis profile"); + expect(profile.parameters).toMatchObject({ + executable_role: target.executableRole, + managed: target.managed, + }); + digests.add(profile.digest); + } + expect(digests.size).toBe(3); + }); + it("keeps Windows annotation mutation unavailable independently of native controls", () => { const ghidra = provider({ ...installationHost(), platform: "win32" }); expect( From ea1dbda1c747400fe2228c1802a3721995216277 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:47:06 +0800 Subject: [PATCH 28/74] test(ghidra): share result projection client setup --- src/ghidra/GhidraProvider.behavior.test.ts | 74 ++++++++-------------- 1 file changed, 26 insertions(+), 48 deletions(-) diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index c607b99508..c0518297b3 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -540,24 +540,23 @@ describe("Ghidra client projection", () => { }); describe("Ghidra result projection", () => { + const createElfClient = async (ghidra: GhidraProvider) => { + const target = executableTarget("elf", "x86_64"); + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + if (resolved.value.profile === null) + throw new Error("Expected a bound Ghidra profile"); + return ghidra.createClient(target, resolved.value.profile); + }; + it("rejects malformed inventory output before Evidence creation", async () => { const ghidra = provider(installationHost(), () => ({ start: () => Promise.resolve(ok(sessionInfo())), callTool: () => Promise.resolve(ok({ items: "not-an-inventory" })), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - - await expect( - ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("list_procedures", {}), - ).resolves.toMatchObject({ + const client = await createElfClient(ghidra); + await expect(client.execute("list_procedures", {})).resolves.toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" }, }); @@ -586,21 +585,13 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - + const client = await createElfClient(ghidra); await expect( - ghidra - .createClient( - executableTarget("elf", "x86_64"), - resolved.value.profile, - ) - .execute("list_procedures", {}), - ).resolves.toMatchObject({ ok: false, error: { _tag: tag } }); + client.execute("list_procedures", {}), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: tag }, + }); }, ); @@ -622,19 +613,12 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound profile"); + const client = await createElfClient(ghidra); await expect( - ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("annotate_native_function", { - procedure: "0x10100", - name: "bad name", - }), + client.execute("annotate_native_function", { + procedure: "0x10100", + name: "bad name", + }), ).resolves.toMatchObject({ ok: false, error: { @@ -662,16 +646,10 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - - const result = await ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("procedure_pseudo_code", { procedure: "main" }); + const client = await createElfClient(ghidra); + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); expect(result).toMatchObject({ ok: false, error: { _tag: tag }, From e09cfe9b5970d75da478c9dd741451cc2be5c91a Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:47:15 +0800 Subject: [PATCH 29/74] fix(javascript): invalidate references stored through property targets (#1445) * fix(javascript): invalidate references stored through property targets * refactor(javascript): share rest keys and return-shape fixtures --- docs/javascript-application-workflows.md | 6 +- src/domain/javascript/javascriptAstValues.ts | 10 ++ .../javascript/javascriptSemanticAnalysis.ts | 14 +-- .../javascriptSemanticIterationSources.ts | 69 ++++++++++--- .../javascriptIterationAliases.test.ts | 81 ++------------- .../applications/javascriptSlotStores.test.ts | 98 +++++++++++++++++++ tests/fixtures/javascriptReturnShapes.ts | 91 +++++++++++++++++ 7 files changed, 271 insertions(+), 98 deletions(-) create mode 100644 tests/acceptance/applications/javascriptSlotStores.test.ts create mode 100644 tests/fixtures/javascriptReturnShapes.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index f26c16d1f6..bdc0dcf214 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -183,8 +183,10 @@ Direct return expressions, including expression-bodied arrows, are evaluated through an execution-free value lattice. Literal object fields and direct return sites are represented in the result. Calls, dynamic spreads, computed keys, and parser recovery remain partial or unknown. -Objects and arrays passed to calls, constructors, or tagged templates, or used -as method receivers, are not assumed unchanged after the invocation. Aliases +Objects and arrays passed to calls, constructors, or tagged templates, used +as method receivers, or stored through property targets are not assumed +unchanged afterward. Property stores conservatively retain reference escape +uncertainty rather than pretending to execute later writes. Aliases and shared children in spread and rest copies retain that uncertainty; copied primitive slots and unrelated containing properties remain known. Object rest excludes consumed keys, array rest excludes the consumed prefix for known diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index f0a0528c06..5479c8d1f3 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -76,6 +76,16 @@ export const semanticStaticPropertyKey = ( return !computed && t.isIdentifier(property) ? property.name : null; }; +/** Read exact consumed object-pattern keys, leaving dynamic keys unknown. */ +export const semanticObjectPatternKeys = ( + pattern: t.ObjectPattern, +): readonly string[] => + pattern.properties.flatMap((property) => { + if (t.isRestElement(property)) return []; + const key = semanticStaticPropertyKey(property.key, property.computed); + return key === null ? [] : [key]; + }); + /** * Display an exact JavaScript string as nonempty label text. Graph labels and * artifact-local keys are nonempty, so the legal empty value is shown as `""`; diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 1cf53fb313..452d29074b 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -14,7 +14,10 @@ import { immutableSemanticBindings, immutableSemanticScopes, } from "./javascriptSemanticProjection.js"; -import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { + semanticObjectPatternKeys, + semanticStaticPropertyKey, +} from "./javascriptAstValues.js"; import type { JavaScriptSemanticAnalysisState, JavaScriptSemanticBindingState, @@ -880,14 +883,7 @@ const bindPattern = (input: BindPatternInput): void => { referenceOnly: true, copyKind: "object-rest", copyProjectionOffset: projection.length, - copyExcludedKeys: pattern.properties.flatMap((property) => { - if (t.isRestElement(property)) return []; - const key = semanticStaticPropertyKey( - property.key, - property.computed, - ); - return key === null ? [] : [key]; - }), + copyExcludedKeys: semanticObjectPatternKeys(pattern), }); } else { const name = semanticStaticPropertyKey(property.key, property.computed); diff --git a/src/domain/javascript/javascriptSemanticIterationSources.ts b/src/domain/javascript/javascriptSemanticIterationSources.ts index c83eb94ad9..ea3708e515 100644 --- a/src/domain/javascript/javascriptSemanticIterationSources.ts +++ b/src/domain/javascript/javascriptSemanticIterationSources.ts @@ -1,14 +1,21 @@ import * as t from "@babel/types"; -import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; -import { semanticArrayIndex } from "./javascriptSemanticPropertyPaths.js"; +import { + semanticObjectPatternKeys, + semanticStaticPropertyKey, +} from "./javascriptAstValues.js"; +import { + semanticArrayIndex, + type JavaScriptSemanticPropertyPath, +} from "./javascriptSemanticPropertyPaths.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; interface IterationSourceValue { readonly node: t.Node; - readonly projection: readonly (string | number | null)[]; + readonly projection: JavaScriptSemanticPropertyPath; } -/** An iterator receiver or yielded reference store, with default guards. */ +/** An iterator receiver or property reference store, with default guards. */ export interface SemanticIterationSource extends IterationSourceValue { readonly kind: "iteration" | "reference-store"; readonly mutation: t.Node; @@ -69,11 +76,25 @@ const visit = ( guards: SourceGuards = {}, ): void => { const { sources, mutation } = context; - if (t.isForOfStatement(mutation) && t.isMemberExpression(pattern)) { - // Storing a yielded reference in a property has no lexical loop binding. - // Preserve its escape rather than assuming later writes cannot reach it. + if (t.isMemberExpression(pattern)) { + // Property stores have no lexical alias binding. Preserve the selected + // reference's escape rather than claiming later writes cannot reach it. if (source !== undefined) - add(sources, source, mutation, guards, "reference-store"); + add( + sources, + source.restStartIndex === undefined + ? source + : { + node: source.node, + projection: [ + ...source.projection, + { excludedKeys: [], startIndex: source.restStartIndex }, + ], + }, + mutation, + guards, + "reference-store", + ); return; } if (t.isTSParameterProperty(pattern)) { @@ -104,8 +125,29 @@ const visit = ( } if (t.isObjectPattern(pattern)) { for (const property of pattern.properties) { - // Object rest creates a fresh object and has no iterable binding pattern. - if (t.isRestElement(property)) continue; + if (t.isRestElement(property)) { + // A rest copy owns its scalar slots; only selected child references + // escape when that fresh copy is stored through a member target. + visit( + context, + property.argument, + source === undefined + ? undefined + : { + node: source.node, + projection: [ + ...source.projection, + { + excludedKeys: [ + ...new Set(semanticObjectPatternKeys(pattern)), + ].sort(compareUnicodeCodePoints), + }, + ], + }, + guards, + ); + continue; + } visit( context, property.value, @@ -144,7 +186,7 @@ const visit = ( visit(context, pattern.argument, undefined, guards); }; -/** Describe implicit iteration sources without deciding their runtime values. */ +/** Describe iterator receivers and property reference stores without evaluation. */ export const semanticIterationSources = ( node: t.Node, parent: t.Node | null, @@ -153,7 +195,10 @@ export const semanticIterationSources = ( const context = { sources, mutation: node }; if (t.isVariableDeclarator(node) && node.init != null) visit(context, node.id, { node: node.init, projection: [] }); - else if (t.isAssignmentExpression(node) && node.operator === "=") + else if ( + t.isAssignmentExpression(node) && + ["=", "||=", "&&=", "??="].includes(node.operator) + ) visit(context, node.left, { node: node.right, projection: [] }); else if (t.isForOfStatement(node)) { add(sources, { node: node.right, projection: [] }, node, {}); diff --git a/tests/acceptance/applications/javascriptIterationAliases.test.ts b/tests/acceptance/applications/javascriptIterationAliases.test.ts index 1039954806..a7f039eb83 100644 --- a/tests/acceptance/applications/javascriptIterationAliases.test.ts +++ b/tests/acceptance/applications/javascriptIterationAliases.test.ts @@ -1,13 +1,9 @@ -import { Client } from "@modelcontextprotocol/client"; -import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; -import { writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; import { expect } from "vitest"; -import { toolContract } from "../../../src/contracts/toolContracts.js"; -import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; -import { projectedExportReturnShapesSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; -import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + verifyJavaScriptReturnShapes, + type JavaScriptReturnFields, +} from "../../fixtures/javascriptReturnShapes.js"; import { cliTest } from "../../support/cli/cliFixture.js"; const source = [ @@ -37,27 +33,7 @@ const source = [ "export function shadowed() { const t = { x: 1 }; const shared = { x: 3 }; for (const t of [shared]) t.x = 2; return t.x; }", ].join("\n"); -const assertReturns = (value: unknown): void => { - const analysis = javascriptApplicationAnalysisResultSchema.parse(value); - const projections = analysis.graph.nodes - .flatMap(({ observations }) => observations) - .filter( - ({ properties }) => properties.semantic_role === "export-return-shapes", - ) - .map(({ properties }) => - projectedExportReturnShapesSchema.parse(properties), - ); - const fields = (name: string) => { - const projection = projections.find( - ({ exported_name }) => exported_name === name, - ); - if (projection === undefined) throw new Error(`Missing export ${name}`); - expect(projection.return_shape_coverage).toMatchObject({ - status: "complete", - projection_complete: true, - }); - return projection.static_return_shapes.flatMap(({ fields }) => fields); - }; +const assertReturns = (fields: JavaScriptReturnFields): void => { expect(fields("inline")).toEqual( expect.arrayContaining([ expect.objectContaining({ path: "/x", state: "unknown" }), @@ -95,51 +71,6 @@ const assertReturns = (value: unknown): void => { cliTest( "preserves yielded-object mutation uncertainty through CLI and stdio MCP", - async ({ cli }) => { - const root = await createTestTempDirectory("rea-iteration-aliases-"); - await writeFile(join(root, "app.js"), source); - const cliResponse = await cli.run({ - arguments: [ - "analyze-javascript-application", - root, - "--artifact-format", - "directory", - "--json", - ], - }); - expect(cliResponse.exitCode).toBe(0); - const contract = toolContract("analyze_javascript_application"); - const cliEvidence = contract.outputSchema.parse(cliResponse.json); - assertReturns(cliEvidence.normalized_result); - const client = new Client({ name: "iteration-aliases-e2e", version: "1" }); - const transport = new StdioClientTransport({ - command: process.execPath, - args: [resolve("scripts/rea.mjs"), "mcp"], - cwd: process.cwd(), - env: { PATH: process.env.PATH ?? "" }, - stderr: "pipe", - }); - transport.stderr?.on("data", () => undefined); - try { - await client.connect(transport); - const response = await client.callTool({ - name: "analyze_javascript_application", - arguments: { input_path: root, format: "directory" }, - }); - expect(response.isError).not.toBe(true); - const evidence = contract.outputSchema.parse(response.structuredContent); - expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); - assertReturns(evidence.normalized_result); - const text = response.content.find((item) => item.type === "text"); - if (text?.type !== "text") throw new Error("Missing MCP text result"); - assertReturns( - contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, - ); - await client.ping(); - } finally { - await client.close(); - await transport.close(); - } - }, + ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns), 120_000, ); diff --git a/tests/acceptance/applications/javascriptSlotStores.test.ts b/tests/acceptance/applications/javascriptSlotStores.test.ts new file mode 100644 index 0000000000..3af7e3eeed --- /dev/null +++ b/tests/acceptance/applications/javascriptSlotStores.test.ts @@ -0,0 +1,98 @@ +import { expect } from "vitest"; + +import { + verifyJavaScriptReturnShapes, + type JavaScriptReturnFields, +} from "../../fixtures/javascriptReturnShapes.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const stores = { + object: "const o = {a: {x: 0}}; o.a = shared; o.a.x = 2;", + array: "const arr = [{x: 0}]; arr[0] = shared; arr[0].x = 2;", + captured: "const o = {a: {x: 0}}; o.a = shared; const t = o.a; t.x = 2;", + nested: "const o = {a: {b: {x: 0}}}; o.a.b = shared; o.a.b.x = 2;", + quoted: 'const o = {}; o["a/b~c"] = shared; o["a/b~c"].x = 2;', + dynamic: 'const o = {}; const key = "entry"; o[key] = shared; o[key].x = 2;', + receiverAlias: "const o = {}; const t = o; t.a = shared; o.a.x = 2;", + transitive: "const p = {}; const o = {}; o.a = p; p.b = shared; o.a.b.x = 2;", + overwritten: "const o = {}; o.a = {x: 0}; o.a = shared; o.a.x = 2;", + replacedReference: + "const o = {}; const other = {x: 0}; o.a = other; o.a = shared; o.a.x = 2;", + copiedDestination: + "const o = {}; o.a = shared; const copy = {...o}; copy.a.x = 2;", + iteration: "const arr = []; arr[0] = shared; for (const t of arr) t.x = 2;", + nestedArray: "const o = {}; o.a = [shared]; o.a[0].x = 2;", + orAssignment: "const o = {a: null}; o.a ||= shared; o.a.x = 2;", + nullishAssignment: "const o = {a: null}; o.a ??= shared; o.a.x = 2;", + andAssignment: "const o = {a: {x: 0}}; o.a &&= shared; o.a.x = 2;", + objectPattern: "const o = {}; ({child: o.a} = {child: shared}); o.a.x = 2;", + arrayPattern: "const o = {}; [o.a] = [shared]; o.a.x = 2;", + defaultPattern: "const o = {}; ({child: o.a = shared} = {}); o.a.x = 2;", + objectRestMember: + "const o = {}; ({...o.a} = {child: shared}); o.a.child.x = 2;", + arrayRestMember: "const o = {}; [...o.a] = [shared]; o.a[0].x = 2;", + iteratedObjectRest: + "const o = {}; for ({...o.a} of [{child: shared}]) o.a.child.x = 2;", + iteratedArrayRest: "const o = {}; for ([...o.a] of [[shared]]) o.a[0].x = 2;", + classField: + "class Holder {} const o = new Holder(); o.a = shared; o.a.x = 2;", +}; +const source = [ + ...Object.entries(stores).map( + ([name, body]) => + `export function ${name}() { const shared = {x: 1}; ${body} return shared.x; }`, + ), + "export function primitive() { const n = 1; const o = {}; o.a = n; o.a = 2; return n; }", + "export function nestedChild() { const shared = {inner: {x: 1}}; const o = {}; o.a = shared; o.a.inner.x = 2; return shared.inner.x; }", + "export function readOnlyStore() { const shared = {x: 1}; const o = {}; o.a = shared; const captured = o.a.x; return shared.x; }", + "export function copiedScalar() { const shared = {x: 1}; const o = {}; o.a = {...shared}; o.a.x = 2; return shared.x; }", + "export function copiedArrayScalar() { const shared = {x: 1}; const o = {}; o.a = [shared.x]; o.a[0] = 2; return shared.x; }", + "export function unrelated() { const shared = {x: 1}; const other = {x: 9}; const o = {}; o.a = shared; o.a.x = 2; return other.x; }", + "export function copiedChild() { const child = {x: 1}; const parent = {child, keep: 7}; const o = {}; o.a = {...parent}; o.a.child.x = 2; return {child: child.x, keep: parent.keep}; }", + "export function objectRestExclusion() { const child = {x: 1}; const skip = {x: 3}; const parent = {child, skip, keep: 7}; const o = {}; let ignored; ({skip: ignored, ...o.a} = parent); o.a.child.x = 2; return {child: child.x, skip: skip.x, keep: parent.keep}; }", + "export function arrayRestOffset() { const child = {x: 1}; const skip = {x: 3}; const items = [skip, child]; const o = {}; let ignored; [ignored, ...o.a] = items; o.a[0].x = 2; return {child: child.x, skip: items[0].x}; }", +].join("\n"); + +const assertReturns = (fields: JavaScriptReturnFields): void => { + for (const name of [...Object.keys(stores), "nestedChild"]) { + const field = fields(name).find(({ path }) => path === ""); + expect(field, name).toBeDefined(); + expect(field?.state, name).toMatch(/^(unknown|literal)$/); + if (field?.state === "literal") expect(field.value, name).toBe(2); + } + const readOnly = fields("readOnlyStore").find(({ path }) => path === ""); + expect(readOnly?.state).toMatch(/^(unknown|literal)$/); + if (readOnly?.state === "literal") expect(readOnly.value).toBe(1); + for (const [name, value] of Object.entries({ + primitive: 1, + copiedScalar: 1, + copiedArrayScalar: 1, + unrelated: 9, + })) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "literal", value }), + ); + const child = fields("copiedChild").find(({ path }) => path === "/child"); + expect(child?.state).toMatch(/^(unknown|literal)$/); + if (child?.state === "literal") expect(child.value).toBe(2); + expect(fields("copiedChild")).toContainEqual( + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ); + for (const name of ["objectRestExclusion", "arrayRestOffset"]) { + const child = fields(name).find(({ path }) => path === "/child"); + expect(child?.state, name).toMatch(/^(unknown|literal)$/); + if (child?.state === "literal") expect(child.value, name).toBe(2); + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "/skip", state: "literal", value: 3 }), + ); + } + expect(fields("objectRestExclusion")).toContainEqual( + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ); +}; + +cliTest( + "does not report stale literals after storing references in member slots", + ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns), + 120_000, +); diff --git a/tests/fixtures/javascriptReturnShapes.ts b/tests/fixtures/javascriptReturnShapes.ts new file mode 100644 index 0000000000..02c66bc84b --- /dev/null +++ b/tests/fixtures/javascriptReturnShapes.ts @@ -0,0 +1,91 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { projectedExportReturnShapesSchema } from "../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; +import type { TestCli } from "../support/cli/cliFixture.js"; + +const exportedReturnFields = (value: unknown) => { + const analysis = javascriptApplicationAnalysisResultSchema.parse(value); + const projections = analysis.graph.nodes + .flatMap(({ observations }) => observations) + .filter( + ({ properties }) => properties.semantic_role === "export-return-shapes", + ) + .map(({ properties }) => + projectedExportReturnShapesSchema.parse(properties), + ); + return (name: string) => { + const projection = projections.find( + ({ exported_name }) => exported_name === name, + ); + if (projection === undefined) throw new Error(`Missing export ${name}`); + expect(projection.return_shape_coverage).toMatchObject({ + status: "complete", + projection_complete: true, + }); + return projection.static_return_shapes.flatMap(({ fields }) => fields); + }; +}; + +/** Select complete return fields from a real JavaScript application result. */ +export type JavaScriptReturnFields = ReturnType; + +/** Exercise identical return assertions through the compiled CLI and stdio MCP. */ +export const verifyJavaScriptReturnShapes = async ( + cli: TestCli, + source: string, + check: (fields: JavaScriptReturnFields) => void, +): Promise => { + const root = await createTestTempDirectory("rea-return-shapes-"); + await writeFile(join(root, "app.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(cliResponse.exitCode).toBe(0); + const contract = toolContract("analyze_javascript_application"); + const cliEvidence = contract.outputSchema.parse(cliResponse.json); + check(exportedReturnFields(cliEvidence.normalized_result)); + const client = new Client({ name: "return-shapes-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = contract.outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + check(exportedReturnFields(evidence.normalized_result)); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + check( + exportedReturnFields( + contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, + ), + ); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } +}; From 0d2082d36b8bed3cda5daa22adee2163652fc511 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:56:40 +0800 Subject: [PATCH 30/74] fix(javascript): recognize unchanged var and let Electron aliases --- .../javascript/javascriptSemanticAnalysis.ts | 14 ++++- .../electronApplicationAnalysis.test.ts | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index b5c1c7f5df..d490adf659 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -352,9 +352,17 @@ const bindingOrigin = ( ? binding.directOrigins[0] : undefined; const [initializer] = binding.initializers; - if (binding.mutable || binding.initializers.length !== 1) return undefined; - const origin = semanticRequireOrigin(initializer?.node, state); - const projection = initializer?.projection ?? []; + // A var/let declaration permits writes without proving one occurred. + // Require one unconditional initializer and no actual binding writes. + if ( + initializer === undefined || + binding.initializers.length !== 1 || + binding.definitions.some(({ kind }) => kind === "assignment") || + state.conditionalInitializers.has(initializer.node) + ) + return undefined; + const origin = semanticRequireOrigin(initializer.node, state); + const projection = initializer.projection; return origin === undefined || !projection.every((key): key is string => typeof key === "string") ? undefined diff --git a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts index 4180e1818d..dd7bd8809e 100644 --- a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts +++ b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts @@ -116,6 +116,18 @@ describe("static Electron application analysis", () => { expect(handledChannels.sort()).toEqual(["rea:read", "rea:write"]); }); + it("maps unchanged var and let require aliases while excluding reassigned bindings", async () => { + const root = await requireAliasFixtureDirectory(); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + + expect(result.electron_summary.ipc).toMatchObject({ + main_handlers: 2, + paired_renderer_transmissions: 2, + unpaired_literal_renderer_transmissions: 0, + }); + }); + it("returns a tagged cancellation without executing application code", async () => { const root = await fixtureDirectory(); const controller = new AbortController(); @@ -326,6 +338,50 @@ const fixtureDirectory = async (): Promise => { return root; }; +const requireAliasFixtureDirectory = async (): Promise => { + const root = await createTestTempDirectory("rea-electron-require-aliases-"); + await Promise.all([ + writeFile( + join(root, "package.json"), + JSON.stringify({ + name: "rea-electron-require-aliases", + main: "main.cjs", + }), + ), + writeFile( + join(root, "main.cjs"), + String.raw` +var { ipcMain: main } = require("electron"); +let listener = require("electron/main").ipcMain; +main.handle("rea:read", async () => "value"); +listener.on("rea:write", () => undefined); + +let replaced = require("electron").ipcMain; +replaced = { handle() {} }; +replaced.handle("rea:reassigned", () => undefined); + +var assignedLater; +assignedLater.handle("rea:before-assignment", () => undefined); +assignedLater = require("electron").ipcMain; + +if (globalThis.registerIpc) { + var conditional = require("electron").ipcMain; +} +conditional.handle("rea:conditional", () => undefined); +`, + ), + writeFile( + join(root, "preload.cjs"), + String.raw` +const { ipcRenderer } = require("electron"); +ipcRenderer.invoke("rea:read"); +ipcRenderer.send("rea:write", "value"); +`, + ), + ]); + return root; +}; + const renamedBindingFixtureDirectory = async (): Promise => { const root = await createTestTempDirectory("rea-electron-renamed-"); await Promise.all([ From 66863b609fe5bdbb46b8634399ea08439a94a9f4 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:02:02 +0800 Subject: [PATCH 31/74] test(javascript): reuse comparison node fixtures Use the established typed graph fixture constructor while retaining right-hand comparison provenance assertions. Restore the unchanged duplicate-code gate on combined main. Closes #1446. Refs #1434, #1443, #1065. --- .../javascriptApplicationChangeGraph.test.ts | 28 ++++++++----------- .../javascriptApplicationGraph.fixture.ts | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts index 2bbd653af5..e3c8e13108 100644 --- a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts +++ b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts @@ -3,12 +3,12 @@ import { describe, expect, it } from "vitest"; import { createJavaScriptApplicationEdge, createJavaScriptApplicationGraph, - createJavaScriptApplicationNode, type JavaScriptApplicationGraph, } from "./javascriptApplicationGraph.js"; import { artifactEvidence, completeCoverage, + fixtureNode, } from "./javascriptApplicationGraph.fixture.js"; import { compareJavaScriptApplicationVersions } from "./javascriptApplicationVersionComparison.js"; @@ -22,26 +22,22 @@ const buildGraph = ( changed: number, ): JavaScriptApplicationGraph => { const nodes = ["a", "b", "c", "d"].map((name, index) => - createJavaScriptApplicationNode({ - kind: "javascript-module", - identity: { + fixtureNode( + "javascript-module", + { strategy: "canonical-path", stability: "artifact-version", artifact_sha256: artifactSha, path: `${name}.js`, }, - observations: [ - { - label: `${name}.js`, - properties: { - structural_fingerprint_sha256: digest(index + 1), - structural_fingerprint_algorithm: "fixture", - source_sha256: digest(index === 1 || index === 3 ? changed : 0), - }, - evidence: artifactEvidence(artifactSha, `${name}.js`), - }, - ], - }), + `${name}.js`, + { + structural_fingerprint_sha256: digest(index + 1), + structural_fingerprint_algorithm: "fixture", + source_sha256: digest(index === 1 || index === 3 ? changed : 0), + }, + artifactEvidence(artifactSha, `${name}.js`), + ), ); const [a, b] = nodes; if (a === undefined || b === undefined) diff --git a/src/domain/javascript/javascriptApplicationGraph.fixture.ts b/src/domain/javascript/javascriptApplicationGraph.fixture.ts index 00f7c229ad..3002b822f3 100644 --- a/src/domain/javascript/javascriptApplicationGraph.fixture.ts +++ b/src/domain/javascript/javascriptApplicationGraph.fixture.ts @@ -109,7 +109,7 @@ type FixtureNodeArguments = [ evidence: ApplicationGraphEvidence, ]; -const fixtureNode = ( +export const fixtureNode = ( ...[kind, identity, label, properties, evidence]: FixtureNodeArguments ): FixtureNode => createJavaScriptApplicationNode({ From 1ec490f36180cf6d3a8f353e9e43920dbd81ece0 Mon Sep 17 00:00:00 2001 From: Zeynel <137368801+zeynelacikgoez@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:04:31 +0200 Subject: [PATCH 32/74] fix(javascript): stop replaying earlier writes for each object property write Repeated property writes to one object made member-mutation collection cubic. Each write evaluated its target binding to rule out a primitive, and every evaluation folded all earlier mutation paths, copying every slot once per path. 2,000 writes to one object took 27 s and 4,000 took 203 s. - A recorded write always leaves a non-primitive binding value, so only the first write to a binding needs the primitive check. - Apply mutation paths per slot in one pass with the same ordering semantics instead of one full slot copy per path. Co-authored-by: Claude Opus 5.5 Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- ...tSemanticAnalysis.member_mutations.test.ts | 21 ++++ .../javascriptSemanticMemberMutations.ts | 5 +- .../javascriptSemanticMutationValues.ts | 118 ++++++++++++------ .../javascript/javascriptSemanticValues.ts | 10 +- 4 files changed, 106 insertions(+), 48 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts index f28436eb1d..3450060cbe 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts @@ -590,3 +590,24 @@ describe("property mutations through TypeScript satisfies aliases", () => { ).toEqual({ status: "literal", value: "TOKEN" }); }); }); + +describe("property mutation collection on minified bundles", () => { + it("records each write to one object without replaying earlier writes", () => { + const writes = Array.from( + { length: 2000 }, + (_, index) => `source.p${index} = ${index};`, + ).join(" "); + const start = performance.now(); + const value = resultValue( + `const source = { token: "TOKEN" }; ${writes} return [source.token, source.p1999];`, + ); + expect(performance.now() - start).toBeLessThan(2000); + expect(value).toMatchObject({ + status: "array", + items: [ + { value: { status: "literal", value: "TOKEN" } }, + { value: { status: "unknown" } }, + ], + }); + }, 30000); +}); diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 4e4ea7b8b2..1414d407f0 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -194,8 +194,11 @@ export const collectSemanticMemberMutations = ( current.originAt?.end, ]); if (recordedEffects.has(identity)) continue; + // A recorded write invalidates the binding to a non-primitive value, so + // only its first write can target a primitive. Re-evaluating it would + // replay every earlier write and make repeated writes superlinear. const value = - effect === "write" + effect === "write" && binding.mutatedPaths.length === 0 ? evaluateSemanticBinding(binding, state) : undefined; const primitiveWrite = diff --git a/src/domain/javascript/javascriptSemanticMutationValues.ts b/src/domain/javascript/javascriptSemanticMutationValues.ts index dbbc9d4572..bc2213da7b 100644 --- a/src/domain/javascript/javascriptSemanticMutationValues.ts +++ b/src/domain/javascript/javascriptSemanticMutationValues.ts @@ -33,49 +33,89 @@ export const invalidateSemanticEscapedPath = ( : { ...value, items: value.items.map(invalidateChild) }; }; -/** Invalidate only slots that an explicit mutation can affect. */ -export const invalidateSemanticMutationPath = ( +/** + * Invalidate only slots that explicit mutations can affect, as if applying each + * path in order. Paths are grouped per slot so that many writes to one object + * cost one pass over its slots rather than one pass per write. + */ +export const invalidateSemanticMutationPaths = ( value: JavaScriptSemanticValue, - path: JavaScriptSemanticPropertyPath, + paths: readonly JavaScriptSemanticPropertyPath[], ): JavaScriptSemanticValue => { - const [key, ...remaining] = path; - const unknown: JavaScriptSemanticValue = { - status: "unknown", - reason: "This property may have been mutated.", + if (paths.length === 0) return value; + // An unknown key or a replaced value leaves nothing that a later path can + // restore, wherever it occurs in the sequence. + if ( + (value.status !== "object" && value.status !== "array") || + paths.some(([key]) => key === undefined || key === null) + ) + return { + status: "unknown", + reason: "This property may have been mutated.", + }; + // Writing an array length discards every item known before it. + const reset = + value.status === "array" + ? paths.findLastIndex(([key]) => key === "length") + : -1; + const base: typeof value = + reset < 0 + ? value + : { status: "array", items: [], unknownItems: true, omittedItems: null }; + const slots = base.status === "object" ? base.properties : base.items; + const slotsByName = new Map(); + slots.forEach((property, index) => { + const indexes = slotsByName.get(property.name) ?? []; + indexes.push(index); + slotsByName.set(property.name, indexes); + }); + const slotPaths = new Map(); + const replacedSlots = new Set(); + const select = (index: number, remaining: JavaScriptSemanticPropertyPath) => { + const selected = slotPaths.get(index) ?? []; + selected.push(remaining); + slotPaths.set(index, selected); + if (remaining.length === 0) replacedSlots.add(index); }; - if (key === undefined || key === null) return unknown; - if (value.status === "object" || value.status === "array") { - if (value.status === "array" && key === "length") - return { - status: "array", - items: [], - unknownItems: true, - omittedItems: null, - }; - const slots = value.status === "object" ? value.properties : value.items; - const observed = slots.some((property) => - semanticPropertyPathKeyMatches(key, property.name), - ); - const properties = slots.map((property) => - semanticPropertyPathKeyMatches(key, property.name) - ? { + // An added slot is already unknown, so later paths that select it are inert. + const added = new Set(); + for (const [key, ...remaining] of paths.slice(reset + 1)) { + if (key === undefined || key === null) continue; + if (typeof key === "object") { + slots.forEach((property, index) => { + if (semanticPropertyPathKeyMatches(key, property.name)) + select(index, remaining); + }); + continue; + } + const indexes = slotsByName.get(String(key)); + if (indexes === undefined) added.add(String(key)); + else for (const index of indexes) select(index, remaining); + } + const properties: JavaScriptSemanticProperty[] = slots.map( + (property, index) => { + const selected = slotPaths.get(index); + return selected === undefined + ? property + : { ...property, - ...(remaining.length === 0 + ...(replacedSlots.has(index) ? { presence: "unknown-coverage" as const } : {}), - value: invalidateSemanticMutationPath(property.value, remaining), - } - : property, - ); - if (!observed && typeof key !== "object") - properties.push({ - name: String(key), - value: unknown, - presence: "unknown-coverage", - }); - return value.status === "object" - ? { ...value, properties } - : { ...value, items: properties }; - } - return unknown; + value: invalidateSemanticMutationPaths(property.value, selected), + }; + }, + ); + for (const name of added) + properties.push({ + name, + value: { + status: "unknown", + reason: "This property may have been mutated.", + }, + presence: "unknown-coverage", + }); + return base.status === "object" + ? { ...base, properties } + : { ...base, items: properties }; }; diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index 884915948c..87f6ad540e 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -10,7 +10,7 @@ import { semanticSlotAtPath } from "./javascriptSemanticSlots.js"; import { invalidateSemanticEscapedPath, - invalidateSemanticMutationPath, + invalidateSemanticMutationPaths, } from "./javascriptSemanticMutationValues.js"; import type { JavaScriptBindingProvenance } from "./javascriptSemanticIr.js"; @@ -211,13 +211,7 @@ const evaluateBinding = ( if (isPrimitive(captured)) value = captured; } } - let projected = value; - for (const path of binding.mutatedPaths) { - projected = invalidateSemanticMutationPath(projected, path); - // Every subsequent write returns this same root unknown. Replaying those - // paths at every newly collected effect otherwise becomes quadratic. - if (projected.status === "unknown") break; - } + let projected = invalidateSemanticMutationPaths(value, binding.mutatedPaths); for (const escape of binding.escapedPaths) { // Escapes only invalidate object references; other lattice values survive. if (projected.status !== "object" && projected.status !== "array") break; From c89c8f53187aac02e87f0ca11e3dd120b8c7efef Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:05:56 +0800 Subject: [PATCH 33/74] docs(mcp): record real Windows first-query recovery --- docs/mcp-contracts.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 70d843335b..fd801cc0a1 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -193,10 +193,27 @@ the current client, so it is not a fresh-start recovery. On the same connection: This close/reopen flow was exercised on one real Linux stdio connection after a controlled startup timeout, followed by a successful overview and function -analysis. It is not a Windows/macOS coverage claim. REA cleans only resources it -owns and never switches to another provider automatically. A provider timeout, -installation failure, or host permission denial needs its own reported recovery; -increasing a client deadline alone does not fix those failures. +analysis. A separate Windows x64 stdio run with Ghidra 12.1.4, JDK 21 and Node +22.19.0 exercised the same connection and caller-selected native PE: the default +60-second client request timed out, close succeeded, and reopening before a +240-second request completed the cold overview in about 163 seconds. The next +overview took about 19 ms. The cold and warm overviews reported two procedures +and two segments; the input digest was unchanged and no owned runtime root or +observed process-family member remained after final close. This was a small +1,024-byte fixture, so the longer request setting remains an example, not a +guarantee for larger targets. No macOS cold-start recovery was exercised. + +With a supplied progress token, the Windows run received an operation-start +notification before timeout and start/completion notifications for the +successful retry. It received no intermediate import or auto-analysis progress. +These operation markers do not measure analysis work completed or extend the +client's deadline. Each new provider session still imports the target into a +fresh temporary project. + +REA cleans only resources it owns and never switches to another provider +automatically. A provider timeout, installation failure, or host permission +denial needs its own reported recovery; increasing a client deadline alone does +not fix those failures. ## Tool results From 2781cad20125ff3a210b795f60620b048e1bbe9b Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:18:13 -0700 Subject: [PATCH 34/74] perf(process): stop padding rendered terminal lines to full width (#1450) Rendered frames padded every visible row with trailing spaces to the terminal width. Blank rows cost a full row each and every frame repeats them, so the padding made up 14-39% of capture Evidence (2 KB of sparse output produced 228 KB). The padding carried no information: columns is recorded and serialized_state keeps the exact terminal state. Lines are now trimmed after normalization, so line.padEnd(columns) restores the previous value. The comparison contract commits rendered_line_format so older padded captures do not compare as the same contract. Co-authored-by: Claude Opus 5.5 --- docs/process-capture.md | 6 +++ src/domain/process/processScenario.ts | 1 + src/process/capture/TerminalRenderer.test.ts | 47 +++++++++++++++++++ src/process/capture/TerminalRenderer.ts | 17 ++++--- .../processCaptureNormalization.test.ts | 1 + 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 src/process/capture/TerminalRenderer.test.ts diff --git a/docs/process-capture.md b/docs/process-capture.md index 1d3d23e7fb..c315e3e9ef 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -110,6 +110,12 @@ while comparisons and trace assertions use the normalized text. Retaining the original adds at most `limits.output_bytes` of text, because it comes from the same admitted PTY chunks rather than a second unbounded stream. +Rendered frame `lines` hold the visible rows after normalization, without +trailing spaces; pad a line to the frame's `columns` to restore the fixed-width +row. `serialized_state` keeps the exact terminal state. Captures commit this +line format in their comparison contract, so a capture whose lines were padded +to full width does not compare as the same contract. + Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and `listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with ports from 0 through 65535. A port may end a sentence, as in `port: 8080.`; diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index 8312e1d2a1..0ecf2c259e 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -108,6 +108,7 @@ export const processComparisonContract = ( ...(scenario.normalization.ports ? { port_normalization_version: "contextual-endpoints-v2" } : {}), + rendered_line_format: "trailing-spaces-trimmed-v1", }); /** diff --git a/src/process/capture/TerminalRenderer.test.ts b/src/process/capture/TerminalRenderer.test.ts new file mode 100644 index 0000000000..e5a19996b1 --- /dev/null +++ b/src/process/capture/TerminalRenderer.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from "vitest"; + +import { TerminalRenderer } from "./TerminalRenderer.js"; + +const renderFrames = async ( + data: string, + normalize: (value: string) => string = (value) => value, +) => { + const renderer = new TerminalRenderer({ + columns: 20, + rows: 4, + scrollback: 10, + maxBytes: 100_000, + normalize, + }); + renderer.write(data, 0); + const frames = await renderer.frames(); + const retention = renderer.retention(); + await renderer.dispose(); + return { frames, retention }; +}; + +it("omits trailing blank cells from visible lines without losing rows", async () => { + const { frames, retention } = await renderFrames( + "left right \r\n\r\n indented", + ); + const [frame] = frames; + expect(frame?.lines).toEqual(["left right", "", " indented", ""]); + // Fixed-width rows remain recoverable from the recorded column count. + expect(frame?.lines.map((line) => line.padEnd(frame.columns, " "))).toEqual([ + "left right ", + " ".repeat(20), + " indented ", + " ".repeat(20), + ]); + expect(retention.retained_bytes).toBe( + Buffer.byteLength(frame?.serialized_state ?? "") + + Buffer.byteLength("left right indented"), + ); +}); + +it("trims only after normalization sees the full-width row", async () => { + const { frames } = await renderFrames("pid 4242", (value) => + value.replace(/4242 +$/u, ""), + ); + expect(frames[0]?.lines[0]).toBe("pid "); +}); diff --git a/src/process/capture/TerminalRenderer.ts b/src/process/capture/TerminalRenderer.ts index e7f1a0fc8a..4322e2781a 100644 --- a/src/process/capture/TerminalRenderer.ts +++ b/src/process/capture/TerminalRenderer.ts @@ -14,6 +14,8 @@ const HeadlessPackage = const SerializePackage = require("@xterm/addon-serialize") as typeof import("@xterm/addon-serialize"); +const TRAILING_SPACES = / +$/u; + interface TerminalRendererOptions { readonly columns: number; readonly rows: number; @@ -106,13 +108,16 @@ export class TerminalRenderer { const lines: string[] = []; for (let row = 0; row < this.#terminal.rows; row += 1) { const line = buffer.getLine(buffer.viewportY + row); + // Trailing blank cells carry no information beyond `columns`; dropping + // them keeps sparse screens from costing a full row per blank line. lines.push( - this.options.normalize( - (line?.translateToString(false, 0, this.#terminal.cols) ?? "").padEnd( - this.#terminal.cols, - " ", - ), - ), + this.options + .normalize( + ( + line?.translateToString(false, 0, this.#terminal.cols) ?? "" + ).padEnd(this.#terminal.cols, " "), + ) + .replace(TRAILING_SPACES, ""), ); } const serializedState = this.options.normalize( diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 58cf22c0fd..8156aff97f 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -64,6 +64,7 @@ itWithCaptureCapability( ).toBe(true); expect(capture.manifest.comparison_contract).toMatchObject({ port_normalization_version: "contextual-endpoints-v2", + rendered_line_format: "trailing-spaces-trimmed-v1", }); expect(capture.exit).toMatchObject({ code: 0, reason: "exited" }); expect(capture.cleanup).toMatchObject({ From 696fb834b8873ac9f6df0c4d45377bdd7c3c5762 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:20:22 +0800 Subject: [PATCH 35/74] fix(skills): expose authored skill to repository discovery --- .../reverse-engineer-anything/SKILL.md | 0 .../references/android-applications.md | 0 .../references/evidence-workflows.md | 0 .../references/javascript-applications.md | 0 .../references/native-and-artifacts.md | 0 .../references/runtime-observation.md | 0 .github/pull_request_template.md | 2 +- AGENTS.md | 2 +- CONTRIBUTING.md | 8 +- docs/installation.md | 2 +- scripts/generate-package-metadata.mjs | 2 +- scripts/generate-skill-metadata.mjs | 4 +- scripts/verify-package-pack.mjs | 2 +- scripts/verify-package.mjs | 4 + scripts/verify-repository-skill.mjs | 97 +++++++++++++++++++ .../filesystem/dependencyInstall.test.ts | 2 +- .../filesystem/generatedSkill.test.ts | 2 +- turbo.json | 6 +- 18 files changed, 119 insertions(+), 14 deletions(-) rename {skill-src => .agents/skills}/reverse-engineer-anything/SKILL.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/android-applications.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/evidence-workflows.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/javascript-applications.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/native-and-artifacts.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/runtime-observation.md (100%) create mode 100644 scripts/verify-repository-skill.mjs diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md similarity index 100% rename from skill-src/reverse-engineer-anything/SKILL.md rename to .agents/skills/reverse-engineer-anything/SKILL.md diff --git a/skill-src/reverse-engineer-anything/references/android-applications.md b/.agents/skills/reverse-engineer-anything/references/android-applications.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/android-applications.md rename to .agents/skills/reverse-engineer-anything/references/android-applications.md diff --git a/skill-src/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/evidence-workflows.md rename to .agents/skills/reverse-engineer-anything/references/evidence-workflows.md diff --git a/skill-src/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/javascript-applications.md rename to .agents/skills/reverse-engineer-anything/references/javascript-applications.md diff --git a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/native-and-artifacts.md rename to .agents/skills/reverse-engineer-anything/references/native-and-artifacts.md diff --git a/skill-src/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/runtime-observation.md rename to .agents/skills/reverse-engineer-anything/references/runtime-observation.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ffbeb73ba..e33ce01f4f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -28,7 +28,7 @@ intentionally not included. --> - Provider, bridge, target-format, or platform compatibility: - Evidence, artifact, provenance, or reconstruction contract: - Process execution, authorization, cleanup, or containment impact: -- Generated metadata (`docs/public/product-catalog.json`), package, or installation impact (CI generates and validates ignored catalogs and `skills/`; edit authored instructions in `skill-src/` and never commit build outputs): +- Generated metadata (`docs/public/product-catalog.json`), package, or installation impact (CI generates and validates ignored catalogs and `skills/`; edit authored instructions in `.agents/skills/reverse-engineer-anything/` and never commit build outputs): ## Evidence and regression coverage diff --git a/AGENTS.md b/AGENTS.md index 15d3984ff3..6e5a2df446 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ Use [.nvmrc](.nvmrc) and `package.json#packageManager` for the pinned toolchain. Use relevant checks for the change. The complete local gate, `npm run check:pr`, is optional for broad changes or CI diagnosis. See [CONTRIBUTING.md](CONTRIBUTING.md) for development, generated-file ownership, and release conventions. -`docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/` are ignored outputs. Edit source contracts and `skill-src/`, then regenerate as needed. `.cache/mcp-tool-catalog.json` is test metadata; runtime builds and source checking do not consume it. Never commit binaries, provider project documents, credentials, `dist/`, `node_modules/`, or local planning artifacts. +`docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/` are ignored outputs. Edit source contracts and `.agents/skills/reverse-engineer-anything/`, then regenerate as needed. `.cache/mcp-tool-catalog.json` is test metadata; runtime builds and source checking do not consume it. Never commit binaries, provider project documents, credentials, `dist/`, `node_modules/`, or local planning artifacts. ## Tool and Test Changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07c71c4c53..9f4e6969b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,8 +95,12 @@ Pre-commit formats and lints staged files; pre-push runs `check:fast`. `evidence:generate` prepares only the managed commitment and its runtime/skill dependencies. The ignored outputs are `docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/`. Edit authored -skill instructions in `skill-src/`. CI validates generated outputs and retains -them as artifacts; do not commit them. Reviewed source metadata, including +skill instructions in `.agents/skills/reverse-engineer-anything/`, where Skills +can discover the public bundle directly from a clean checkout. `verify:package` +installs that bundle with Skills 1.7.2 using default repository-root discovery +and checks every installed file, alongside the packaged setup checks. +CI validates generated outputs and retains them as artifacts; do not commit +them. Reviewed source metadata, including `src/generatedPackageMetadata.ts` and `docs/error-contract.schema.json`, remains tracked and checked for freshness. diff --git a/docs/installation.md b/docs/installation.md index e54cc73ae9..38ea09f5a5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -95,7 +95,7 @@ npx skills add morluto/rea --skill reverse-engineer-anything This installs agent instructions and bundled references, not REA MCP registration or analysis engines. Follow the skill's -[conditional connection guide](https://github.com/morluto/rea/blob/main/skill-src/reverse-engineer-anything/SKILL.md#connect-only-when-needed). +[conditional connection guide](https://github.com/morluto/rea/blob/main/.agents/skills/reverse-engineer-anything/SKILL.md#connect-only-when-needed). Working tools can be used immediately. If tools are missing, inspect the current client's registration with `doctor --client codex --json` (substitute its client ID), then plan repairs with `setup --client codex --dry-run --json`. Show and diff --git a/scripts/generate-package-metadata.mjs b/scripts/generate-package-metadata.mjs index 8247dac1c2..0130401b17 100644 --- a/scripts/generate-package-metadata.mjs +++ b/scripts/generate-package-metadata.mjs @@ -17,7 +17,7 @@ const lock = JSON.parse( await readFile(join(root, "package-lock.json"), "utf8"), ); const skill = await readFile( - join(root, "skill-src/reverse-engineer-anything/SKILL.md"), + join(root, ".agents/skills/reverse-engineer-anything/SKILL.md"), "utf8", ); const skillVersion = /^\s{2}version:\s*"([^"]+)"\s*$/mu.exec(skill)?.[1]; diff --git a/scripts/generate-skill-metadata.mjs b/scripts/generate-skill-metadata.mjs index 5bb2e3ccff..3ca0a467db 100644 --- a/scripts/generate-skill-metadata.mjs +++ b/scripts/generate-skill-metadata.mjs @@ -10,7 +10,7 @@ for (const argument of arguments_) throw new Error(`Unknown skill metadata option: ${argument}`); const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const sourceRoot = join(root, "skill-src/reverse-engineer-anything"); +const sourceRoot = join(root, ".agents/skills/reverse-engineer-anything"); const outputRoot = join(root, "skills/reverse-engineer-anything"); const check = arguments_.has("--check"); const current = ( @@ -25,7 +25,7 @@ const { PRODUCT_IDENTITY } = await import( ); if (/^\s{2}(?:tool_count|catalog_digest):/mu.test(current)) throw new Error( - "Catalog metadata belongs in the generated skill, not skill-src", + "Catalog metadata belongs in the generated skill, not the authored source", ); const versionLine = /^ {2}version: "[^"\r\n]+"$/mu; if (!versionLine.test(current)) diff --git a/scripts/verify-package-pack.mjs b/scripts/verify-package-pack.mjs index 96f05417b9..fc9676745b 100644 --- a/scripts/verify-package-pack.mjs +++ b/scripts/verify-package-pack.mjs @@ -38,7 +38,7 @@ export async function verifyPackagePack({ root, workspace }) { throw new Error("package did not expose both rea command entry points"); if (packedManifest.mcpName !== "io.github.morluto/rea") throw new Error("package did not retain its MCP Registry ownership marker"); - if (packedFiles.some((path) => path.startsWith("package/skill-src/"))) + if (packedFiles.some((path) => path.startsWith("package/.agents/"))) throw new Error( "package included authored skill sources instead of only the generated bundle", ); diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 24923f7f6e..b70b06064b 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -17,6 +17,7 @@ import { verifyPackageMcp } from "./verify-package-mcp.mjs"; import { verifyPackagePack } from "./verify-package-pack.mjs"; import { verifyPackagePlatform } from "./verify-package-platform.mjs"; import { verifyPackageSetup } from "./verify-package-setup.mjs"; +import { verifyRepositorySkill } from "./verify-repository-skill.mjs"; import { verifyManaged } from "./verify-package-managed.mjs"; import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; @@ -34,9 +35,11 @@ let tarball; let mcpStartup; let mcpModuleLoading; let update; +let repositorySkill; let report; try { + repositorySkill = await verifyRepositorySkill({ root, workspace }); ({ tarball } = await verifyPackagePack({ root, workspace })); const environmentData = await verifyPackageEnvironment({ root, @@ -156,6 +159,7 @@ try { configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, + repositorySkill, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, diff --git a/scripts/verify-repository-skill.mjs b/scripts/verify-repository-skill.mjs new file mode 100644 index 0000000000..8340bce7b6 --- /dev/null +++ b/scripts/verify-repository-skill.mjs @@ -0,0 +1,97 @@ +import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import { dirname, join, relative } from "node:path"; + +import { exec, pathExists } from "./lib/verify-package-core.mjs"; + +/** Exercise Skills' default root discovery without ignored build outputs. */ +export async function verifyRepositorySkill({ root, workspace }) { + const snapshot = join(workspace, "repository"); + const consumer = join(workspace, "skill-consumer"); + const home = join(workspace, "skill-consumer-home"); + const tracked = ( + await exec( + "git", + ["ls-files", "--cached", "--format=%(objectmode) %(path)", "-z"], + { cwd: root }, + ) + ).stdout + .split("\0") + .filter(Boolean); + for (const entry of tracked) { + const mode = entry.slice(0, 6); + const path = entry.slice(7); + const destination = join(snapshot, path); + // A GitHub clone without submodule initialization has only these directories. + if (mode === "160000") { + await mkdir(destination, { recursive: true }); + continue; + } + await mkdir(dirname(destination), { recursive: true }); + await cp(join(root, path), destination, { verbatimSymlinks: true }); + } + if (await pathExists(join(snapshot, "skills"))) + throw new Error("repository skill check must run without generated skills"); + await mkdir(consumer, { recursive: true }); + await mkdir(home, { recursive: true }); + const skillsVersion = "1.7.2"; + await exec( + "npm", + [ + "exec", + "--yes", + "--package", + `skills@${skillsVersion}`, + "--", + "skills", + "add", + snapshot, + "--skill", + "reverse-engineer-anything", + "--agent", + "codex", + "--yes", + ], + { + cwd: consumer, + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, ".config"), + CODEX_HOME: join(home, ".codex"), + DO_NOT_TRACK: "1", + SKILLS_NO_TELEMETRY: "1", + CI: "1", + }, + timeout: 120_000, + }, + ); + const authoredRoot = join( + snapshot, + ".agents/skills/reverse-engineer-anything", + ); + const installedSkills = join(consumer, ".agents/skills"); + const installedRoot = join(installedSkills, "reverse-engineer-anything"); + const files = await filePaths(authoredRoot); + if ( + JSON.stringify(await readdir(installedSkills)) !== + JSON.stringify(["reverse-engineer-anything"]) || + JSON.stringify(await filePaths(installedRoot)) !== JSON.stringify(files) + ) + throw new Error( + "Skills did not install exactly the selected public bundle", + ); + for (const path of files) { + const authored = await readFile(join(authoredRoot, path)); + if (!authored.equals(await readFile(join(installedRoot, path)))) + throw new Error(`Skills changed authored skill content: ${path}`); + } + return { skillsVersion, files }; +} + +async function filePaths(root) { + return (await readdir(root, { recursive: true, withFileTypes: true })) + .filter((entry) => entry.isFile()) + .map((entry) => relative(root, join(entry.parentPath, entry.name))) + .sort(); +} diff --git a/tests/boundary/filesystem/dependencyInstall.test.ts b/tests/boundary/filesystem/dependencyInstall.test.ts index 66378360f4..806d91b321 100644 --- a/tests/boundary/filesystem/dependencyInstall.test.ts +++ b/tests/boundary/filesystem/dependencyInstall.test.ts @@ -169,7 +169,7 @@ const createBuildCacheFixture = async (): Promise => { "scripts/clean-build-output.mjs", "scripts/generate-package-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/reverse-engineer-anything/SKILL.md", + ".agents/skills/reverse-engineer-anything/SKILL.md", "src/generatedPackageMetadata.ts", ]; await Promise.all( diff --git a/tests/boundary/filesystem/generatedSkill.test.ts b/tests/boundary/filesystem/generatedSkill.test.ts index 5df6e3146a..2b7ebf754b 100644 --- a/tests/boundary/filesystem/generatedSkill.test.ts +++ b/tests/boundary/filesystem/generatedSkill.test.ts @@ -21,7 +21,7 @@ async function fixture(eol = "\n") { await mkdir(dirname(join(root, path)), { recursive: true }); await copyFile(join(repositoryRoot, path), join(root, path)); } - const sourceRoot = join(root, "skill-src/reverse-engineer-anything"); + const sourceRoot = join(root, ".agents/skills/reverse-engineer-anything"); const outputRoot = join(root, "skills/reverse-engineer-anything"); await mkdir(join(sourceRoot, "references"), { recursive: true }); await mkdir(join(root, "dist")); diff --git a/turbo.json b/turbo.json index 3925237083..5b93503d78 100644 --- a/turbo.json +++ b/turbo.json @@ -39,7 +39,7 @@ "package-lock.json", "scripts/generate-package-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/reverse-engineer-anything/SKILL.md", + ".agents/skills/reverse-engineer-anything/SKILL.md", "src/generatedPackageMetadata.ts" ] }, @@ -138,7 +138,7 @@ "docs/**", "!docs/public/product-catalog.json", "!docs/verification/managed-conformance-*.json", - "skill-src/**", + ".agents/skills/reverse-engineer-anything/**", "README.md", { "mode": "dependencyOutputs", @@ -170,7 +170,7 @@ "package-lock.json", "scripts/generate-skill-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/**" + ".agents/skills/reverse-engineer-anything/**" ], "outputs": ["skills/**"] }, From ef96657d6ba0c1f22ebdb1e6bd4d9a6ac9335002 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:07 -0700 Subject: [PATCH 36/74] perf(javascript): hash each semantic evidence context once (#1457) The semantic graph's evidence-context registry canonicalized the full provenance context three times for every node, relation, unknown and fingerprint: once to hash its ID and twice more to compare against the stored row. A file has only a few distinct contexts, so nearly all of that work repeated a known answer. Interning now keys contexts by their canonical text. Each fact canonicalizes once, and only a new context is hashed and checked for a digest collision. IDs and output are unchanged; Obsidian's 334 MB Evidence is byte-identical and analysis drops from about 47.0 s to 44.4 s. Co-authored-by: Claude Opus 5.5 --- .../javascript/javascriptSemanticGraph.ts | 23 +++++++---- .../javascriptSemanticGraphQuery.test.ts | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index e6cf1d5302..69b3bb9678 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -56,6 +56,9 @@ const evidenceContextId = (context: ApplicationGraphEvidenceContext): string => /** Graph-scoped owner for canonical provenance shared by semantic facts. */ export class JavaScriptSemanticEvidenceContextRegistry { readonly #contexts = new Map(); + // Facts share a few contexts, so key them by canonical text and hash each + // distinct context once instead of on every fact. + readonly #idsByCanonicalContext = new Map(); /** Intern normalized non-location provenance at fact construction. */ intern( @@ -63,18 +66,21 @@ export class JavaScriptSemanticEvidenceContextRegistry { ): JavaScriptSemanticEvidenceReference { const normalized = normalizeEvidence(evidence); const { location, ...context } = normalized; + const canonicalContext = canonicalJson( + context, + "JavaScript semantic graph", + ); + const internedId = this.#idsByCanonicalContext.get(canonicalContext); + if (internedId !== undefined) return { context_id: internedId, location }; const contextId = evidenceContextId(context); - const row = { ...context, context_id: contextId }; - const existing = this.#contexts.get(contextId); - if ( - existing !== undefined && - canonicalJson(existing, "JavaScript semantic graph") !== - canonicalJson(row, "JavaScript semantic graph") - ) + // This canonical context is new, so a row already stored under its + // digest holds different provenance. + if (this.#contexts.has(contextId)) throw new TypeError( "JavaScript semantic evidence context digest collision", ); - this.#contexts.set(contextId, row); + this.#contexts.set(contextId, { ...context, context_id: contextId }); + this.#idsByCanonicalContext.set(canonicalContext, contextId); return { context_id: contextId, location }; } @@ -101,6 +107,7 @@ export class JavaScriptSemanticEvidenceContextRegistry { /** Release interned contexts after the owning graph is finalized. */ clear(): void { this.#contexts.clear(); + this.#idsByCanonicalContext.clear(); } } diff --git a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts index ddf3015f08..363cf29ad2 100644 --- a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts +++ b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "vitest"; import { z } from "zod"; +import { digestCanonicalValue } from "../canonicalDigest.js"; import type { ApplicationGraphEvidence } from "./javascriptApplicationEvidenceSchemas.js"; import { JAVASCRIPT_SEMANTIC_NODE_KINDS, @@ -615,3 +616,42 @@ it.each(invalidGraphCases)( expect(issues(() => complete(steps))).toEqual(expected); }, ); + +it("interns each distinct evidence context once under its content digest", () => { + const registry = new JavaScriptSemanticEvidenceContextRegistry(); + const elsewhere: ApplicationGraphEvidence = { + ...evidence(), + location: { + available: true, + value: { + kind: "source-range", + source: "other.js", + start: { line: 7, column: 2 }, + end: { line: 7, column: 9 }, + }, + }, + }; + const first = registry.intern(evidence()); + const repeated = registry.intern(elsewhere); + const reordered = registry.intern({ + ...evidence("inferred"), + limitations: [ + "Static reachability does not prove runtime execution.", + "Static reachability does not prove runtime execution.", + ], + }); + const inferred = registry.intern(evidence("inferred")); + const { location: _location, ...context } = evidence(); + + expect(repeated.context_id).toBe(first.context_id); + expect(repeated.location).toEqual(elsewhere.location); + expect(reordered.context_id).toBe(inferred.context_id); + expect(inferred.context_id).not.toBe(first.context_id); + expect(first.context_id).toBe( + `jsrg_evidence_${digestCanonicalValue(context, "test")}`, + ); + expect(registry.contexts.map(({ context_id }) => context_id)).toEqual( + [first.context_id, inferred.context_id].toSorted(), + ); + expect(registry.resolve(repeated)).toEqual(elsewhere); +}); From b118f044926ae6b4880c4214621ae6c5b2a71f17 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:24 -0700 Subject: [PATCH 37/74] perf(apple): stop repeating the archive path in Interface Builder evidence (#1460) Every Interface Builder node and edge evidence record prefixed its description with the archive's relative path, which artifact_path in the same record already states. Bundles carry thousands of objects across localized nibs, so the repeated path made up 6-10% of decode_interface_builder output (XLD: 7.38 MB to 6.62 MB). Descriptions now name only the decoded item, such as "object 81" or "outlet o_quality". Co-authored-by: Claude Opus 5.5 --- .../apple/interfaceBuilderGraph.test.ts | 43 +++++++++++++++++++ src/domain/apple/interfaceBuilderGraph.ts | 4 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index 13d6f79fb9..fb1c0b9add 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -509,3 +509,46 @@ describe("AppKit keyed archive connectors", () => { expect(to).toEqual(expect.arrayContaining(["objc_selector", "unknown"])); }); }); + +it("names the archive once per evidence record", () => { + const relativePath = "Views/Main.storyboardc/scene.nib/objects.nib"; + const result = buildInterfaceBuilderAnalysis({ + targetSha256: hash, + toolVersion: "test", + documents: [ + { + relativePath, + archiveSha256: hash, + documentKind: "storyboard_scene", + raw: { + "com.apple.ibtool.document.objects": { + controller: { customClass: "StoreViewController" }, + next: { customClass: "BuildViewController" }, + }, + "com.apple.ibtool.document.connections": { + controller: [ + { type: "segue", identifier: "showBuild", destinationId: "next" }, + ], + }, + }, + }, + ], + limits: interfaceBuilderLimitsSchema.parse({}), + }); + const evidence = [...result.graph.nodes, ...result.graph.edges].flatMap( + (item) => item.evidence, + ); + expect(new Set(evidence.map(({ artifact_path }) => artifact_path))).toEqual( + new Set([relativePath]), + ); + expect(evidence.map(({ description }) => description)).toEqual( + expect.arrayContaining([ + "compiled document", + "object controller", + "segue showBuild", + ]), + ); + expect( + evidence.filter(({ description }) => description.includes(relativePath)), + ).toEqual([]); +}); diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index 17e94d584f..bd35130a85 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -72,7 +72,9 @@ export const buildInterfaceBuilderAnalysis = (input: { const evidenceFor = (description: string) => [ { kind: "interface_builder_resource" as const, - description: `${document.relativePath}: ${description}`, + // artifact_path names the archive; repeating it here cost a path per + // node and edge. + description, location: { address: null, file_offset: null }, artifact_path: document.relativePath, artifact_sha256: document.archiveSha256, From ea923287751043c7acfaa35a0acd10fe32f785d6 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:33:57 +0800 Subject: [PATCH 38/74] fix(mcp): validate doctor probe against its discovered output contract (#1461) Validate the target-free probe against the named definition discovered from the complete production catalog. Preserve initialized server and inventory observations when the SDK rejects the probe, and retain the schema failure reason, absolute deadline and owned-child cleanup. The negative stdio replay regression fails on original main and passes the fix; actual production-boundary fault injection confirms rejection. Nine doctor/help tests, serial full static checks and real small-target CLI/MCP parity pass. Successful doctor reports remain byte-identical. No performance or model-token reduction is claimed. Closes #1454. Refs #1065, #723, #1433. --- src/mcpDoctor.ts | 26 +++++++--- tests/boundary/mcp/mcpDoctor.test.ts | 77 +++++++++++++++++++++++++++- tests/fixtures/mcpDoctorReply.mjs | 48 +++++++++++++++++ 3 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/mcpDoctorReply.mjs diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index 0f5de39fce..62a5882807 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -6,6 +6,7 @@ import { resolve } from "node:path"; import { CATALOG_IDENTITY } from "./catalogIdentity.js"; import { PROMPT_CONTRACTS } from "./contracts/promptContracts.js"; import { TOOL_CONTRACTS } from "./contracts/toolContracts.js"; +import { err, ok } from "./domain/result.js"; import { PRODUCT_IDENTITY } from "./identity.js"; import { MCP_STARTUP_POLICY } from "./mcpStartupPolicy.js"; @@ -131,11 +132,22 @@ const inspectProductionMcpSession = async ( signal: AbortSignal, ) => { const request = requestOptions(deadline, signal); - const [tools, prompts, requestFlow] = await Promise.all([ + const [tools, prompts] = await Promise.all([ client.listTools(undefined, request), client.listPrompts(undefined, request), - client.callTool({ name: "binary_session", arguments: {} }, request), ]); + const sessionTool = tools.tools.find(({ name }) => name === "binary_session"); + const requestFlow = await client + .callTool( + { name: "binary_session", arguments: {} }, + { + ...requestOptions(deadline, signal), + ...(sessionTool === undefined ? {} : { toolDefinition: sessionTool }), + }, + ) + .then(ok, (cause: unknown) => + err({ kind: "request-flow" as const, detail: errorMessage(cause) }), + ); const inventory = { tools: compareInventory( TOOL_CONTRACTS.map(({ name }) => name), @@ -147,7 +159,7 @@ const inspectProductionMcpSession = async ( ), }; const observedIdentity = parseIdentityToolResult( - requestFlow.structuredContent, + requestFlow.ok ? requestFlow.value.structuredContent : undefined, ); const serverVersion = client.getServerVersion(); const protocolVersion = client.getNegotiatedProtocolVersion(); @@ -173,8 +185,10 @@ const inspectProductionMcpSession = async ( }, { name: "request-flow", - ok: requestFlow.isError !== true, - detail: "binary_session target-free request", + ok: requestFlow.ok && requestFlow.value.isError !== true, + detail: requestFlow.ok + ? "binary_session target-free request" + : requestFlow.error.detail, }, ]; return { @@ -194,7 +208,7 @@ const inspectProductionMcpSession = async ( }, request_flow: { tool: "binary_session", - ok: requestFlow.isError !== true, + ok: requestFlow.ok && requestFlow.value.isError !== true, }, checks, }; diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 9858e4ff2e..e9b68b36f5 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -1,4 +1,6 @@ -import { readFile } from "node:fs/promises"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -43,6 +45,79 @@ describe("production MCP doctor", () => { ); }, 30_000); + it("rejects a schema-invalid probe while retaining the discovered inventory", async () => { + const root = await createTestTempDirectory("rea-mcp-doctor-reply-"); + const snapshotPath = join(root, "reply.json"); + const environment = { PATH: process.env.PATH ?? "" }; + const client = new Client({ name: "doctor-reply-capture", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: environment, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const tools = await client.listTools(); + const prompts = await client.listPrompts(); + const toolDefinition = tools.tools.find( + ({ name }) => name === "binary_session", + ); + if (toolDefinition === undefined) + throw new Error("Production binary_session contract is missing"); + const reply = await client.callTool( + { name: "binary_session", arguments: {} }, + { toolDefinition }, + ); + await writeFile( + snapshotPath, + JSON.stringify({ + serverInfo: client.getServerVersion(), + protocolVersion: client.getNegotiatedProtocolVersion(), + tools, + prompts, + reply, + }), + ); + } finally { + await client.close(); + await transport.close(); + } + // Replay real production metadata through a synthetic stdio producer. + for (const mode of ["valid", "invalid"]) { + const result = await runProductionMcpDoctor({ + command: process.execPath, + args: [ + resolve("tests/fixtures/mcpDoctorReply.mjs"), + snapshotPath, + mode, + ], + cwd: process.cwd(), + environment, + }); + expect(result.healthy).toBe(mode === "valid"); + expect(result.inventory?.tools.observed).toBe( + CATALOG_IDENTITY.counts.mcp_tools, + ); + expect(result.inventory?.prompts.observed).toBe( + CATALOG_IDENTITY.counts.mcp_prompts, + ); + expect(result.checks).toContainEqual({ + name: "initialize", + ok: true, + detail: expect.any(String), + }); + if (mode === "invalid") + expect(result.checks).toContainEqual({ + name: "request-flow", + ok: false, + detail: expect.stringMatching(/output schema/iu), + }); + } + }, 30_000); + it("reports exact missing, unexpected, and duplicate inventory names", () => { expect(compareInventory(["a", "b"], ["b", "c", "c"])).toEqual({ expected: 2, diff --git a/tests/fixtures/mcpDoctorReply.mjs b/tests/fixtures/mcpDoctorReply.mjs new file mode 100644 index 0000000000..14dad1240e --- /dev/null +++ b/tests/fixtures/mcpDoctorReply.mjs @@ -0,0 +1,48 @@ +import { readFile } from "node:fs/promises"; +import { createInterface } from "node:readline"; + +const snapshot = JSON.parse(await readFile(process.argv[2], "utf8")); +const reply = structuredClone(snapshot.reply); +if (process.argv[3] === "invalid") { + reply.structuredContent.schema_violation_control = true; + reply.content = reply.content.map((item) => + item.type === "text" + ? { ...item, text: JSON.stringify(reply.structuredContent) } + : item, + ); +} + +const input = createInterface({ input: process.stdin }); +input.on("line", (line) => { + const request = JSON.parse(line); + if (request.id === undefined) return; + let result; + switch (request.method) { + case "initialize": + result = { + protocolVersion: snapshot.protocolVersion, + capabilities: { tools: {}, prompts: {} }, + serverInfo: snapshot.serverInfo, + }; + break; + case "tools/list": + result = snapshot.tools; + break; + case "prompts/list": + result = snapshot.prompts; + break; + case "tools/call": + if (request.params.name === "binary_session") result = reply; + break; + case "ping": + result = {}; + break; + } + const response = + result === undefined + ? { error: { code: -32601, message: "Unknown fixture method or tool" } } + : { result }; + process.stdout.write( + `${JSON.stringify({ jsonrpc: "2.0", id: request.id, ...response })}\n`, + ); +}); From 4e3d3354397374116577555200319975fc2ff86e Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:37:49 +0800 Subject: [PATCH 39/74] test(android): retain incomplete cleanup failure reasons (#1443) Include the full stop disposition in the fixture assertion message, so incomplete owned teardown retains its typed reason in Vitest failure reporters. Preserve the existing cleanup requirement without retries or relaxed ownership checks. Seven process-boundary tests, full serial typecheck/lint/duplication/format checks, and 19 final-head CI checks pass after integrating the combined-main fixture fix. A controlled failure-rendering probe verifies that the explicit message preserves the reason. The original CI cleanup failure remains unreproduced; this is diagnostic work with a synthetic JADX producer, not a root-cause or real-engine fix. Refs #1441, #1446, #1448, #1065. --- tests/fixtures/android/jadx.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/android/jadx.ts b/tests/fixtures/android/jadx.ts index 3488647a07..c824690393 100644 --- a/tests/fixtures/android/jadx.ts +++ b/tests/fixtures/android/jadx.ts @@ -74,7 +74,10 @@ export const createJadxProtocolFixture = async ( ownsProcessLifetime: true, cleanup: () => cleanupOwnedProcessGroup(spawned.ownership), }); - expect((await supervisor.stop()).status).not.toBe("incomplete"); + const stopped = await supervisor.stop(); + expect(stopped, JSON.stringify(stopped)).not.toMatchObject({ + status: "incomplete", + }); if (options.cwd !== undefined) await rm(options.cwd, { recursive: true, force: true }); }); From 78e1308f49492011be448c9b35ce16d98d60c4ad Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:58:15 +0800 Subject: [PATCH 40/74] docs(process): clarify normalized terminal line representation --- docs/process-capture.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index c315e3e9ef..5cc246ec23 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -110,11 +110,14 @@ while comparisons and trace assertions use the normalized text. Retaining the original adds at most `limits.output_bytes` of text, because it comes from the same admitted PTY chunks rather than a second unbounded stream. -Rendered frame `lines` hold the visible rows after normalization, without -trailing spaces; pad a line to the frame's `columns` to restore the fixed-width -row. `serialized_state` keeps the exact terminal state. Captures commit this -line format in their comparison contract, so a capture whose lines were padded -to full width does not compare as the same contract. +Rendered frame `lines` hold the visible rows after normalization, with trailing +U+0020 spaces removed. Blank rows and internal spaces remain. `columns` records +terminal cell width; normalization and Unicode can make string lengths differ +from that width. `serialized_state` holds terminal serialization after the +selected normalization. Original admitted PTY chunks are available through +`raw_data ?? data`, as described above. Captures commit this line format in their +comparison contract, so a capture whose lines were padded to full width does not +compare as the same contract. Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and `listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with From 79f6756239105164eac7aa5618844ffd9bdf41e5 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:00:09 -0700 Subject: [PATCH 41/74] docs(website): link the Discord community and REA on X (#1469) Add an X link beside the GitHub and Discord header icons on every content page, name the footer links "Join the community" and "Follow us on X", and close the homepage with a "Join the community" section. On narrow screens, pad the footer so the floating Top link no longer covers its last row of links. Co-authored-by: Claude Opus 5.5 --- website/README.md | 4 +- website/public/assets/styles.css | 12 ++++-- website/public/examples/dino-lab/index.html | 22 +++++++++- website/public/faq/index.html | 22 +++++++++- website/public/first-investigation/index.html | 22 +++++++++- website/public/get-started/index.html | 22 +++++++++- website/public/guides/browser/index.html | 22 +++++++++- website/public/guides/index.html | 22 +++++++++- website/public/guides/javascript/index.html | 22 +++++++++- website/public/guides/native/index.html | 22 +++++++++- website/public/index.html | 40 ++++++++++++++++++- website/public/showcase/aegis/index.html | 22 +++++++++- website/public/showcase/ctf/index.html | 22 +++++++++- website/public/showcase/dx-ball/index.html | 22 +++++++++- website/public/showcase/index.html | 22 +++++++++- website/public/showcase/notion/index.html | 22 +++++++++- website/public/showcase/th04/index.html | 22 +++++++++- 17 files changed, 345 insertions(+), 19 deletions(-) diff --git a/website/README.md b/website/README.md index 2005c633da..4262ccb246 100644 --- a/website/README.md +++ b/website/README.md @@ -181,7 +181,9 @@ the analysis guides. Case-study previews live on the Showcases page. The closing section offers copyable project prompts, from cloning `rea.tools` to reconstructing a game from its executable. The experienced-reader shortcut lands directly on the guide links below these prompts. -The page ends with FAQ, Discord and issue-report links under “Any questions?”. +“Any questions?” offers FAQ, Discord and issue-report links. The page ends +with “Join the community”, which links Discord and REA's X account. +Every content page's header and footer link GitHub, Discord and X. A right-side table of contents stays visible at widths of 1440px and above. On narrower screens it becomes a sticky, native disclosure; selecting a link closes the menu and focuses the destination. The homepage script follows diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index fb7093b9e7..2020fd5f86 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -178,7 +178,8 @@ section[id] { gap: 8px; } .nav-github, -.nav-discord { +.nav-discord, +.nav-x { display: inline-flex; align-items: center; justify-content: center; @@ -192,12 +193,14 @@ section[id] { text-decoration: none; } .nav-github:hover, -.nav-discord:hover { +.nav-discord:hover, +.nav-x:hover { background: var(--soft); color: var(--ink); } .nav-github svg, -.nav-discord svg { +.nav-discord svg, +.nav-x svg { display: block; flex-shrink: 0; } @@ -2117,6 +2120,9 @@ pre code { right: calc(1rem + env(safe-area-inset-right, 0px)); bottom: calc(1rem + env(safe-area-inset-bottom, 0px)); } + .site-footer { + padding-bottom: calc(2rem + 44px + env(safe-area-inset-bottom, 0px)); + } .container { width: calc(100% - 2.5rem); } diff --git a/website/public/examples/dino-lab/index.html b/website/public/examples/dino-lab/index.html index bfae794603..d0bc84c765 100644 --- a/website/public/examples/dino-lab/index.html +++ b/website/public/examples/dino-lab/index.html @@ -107,6 +107,25 @@ /> + + + @@ -383,7 +402,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/faq/index.html b/website/public/faq/index.html index 60fe1183db..fa4a64a83f 100644 --- a/website/public/faq/index.html +++ b/website/public/faq/index.html @@ -107,6 +107,25 @@ /> + + + @@ -482,7 +501,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/first-investigation/index.html b/website/public/first-investigation/index.html index 349ece913b..a642735c28 100644 --- a/website/public/first-investigation/index.html +++ b/website/public/first-investigation/index.html @@ -106,6 +106,25 @@ /> + + + @@ -420,7 +439,8 @@ JS SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/get-started/index.html b/website/public/get-started/index.html index b86987e9ed..d13ad67ba1 100644 --- a/website/public/get-started/index.html +++ b/website/public/get-started/index.html @@ -106,6 +106,25 @@ /> + + + @@ -479,7 +498,8 @@ npx rea-agents@latest setup SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/guides/browser/index.html b/website/public/guides/browser/index.html index 95a0aaaae9..b1567f9f1b 100644 --- a/website/public/guides/browser/index.html +++ b/website/public/guides/browser/index.html @@ -100,6 +100,25 @@ /> + + + @@ -403,7 +422,8 @@ link.click(); SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/guides/index.html b/website/public/guides/index.html index babd58bdf1..eeed0e7f13 100644 --- a/website/public/guides/index.html +++ b/website/public/guides/index.html @@ -100,6 +100,25 @@ /> + + + @@ -186,7 +205,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/guides/javascript/index.html b/website/public/guides/javascript/index.html index 5212b579d6..8e77022fe8 100644 --- a/website/public/guides/javascript/index.html +++ b/website/public/guides/javascript/index.html @@ -106,6 +106,25 @@ /> + + + @@ -405,7 +424,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/guides/native/index.html b/website/public/guides/native/index.html index a0e19caf5f..f3c4a658af 100644 --- a/website/public/guides/native/index.html +++ b/website/public/guides/native/index.html @@ -100,6 +100,25 @@ /> + + + @@ -335,7 +354,8 @@ return (DxBallInt)pan; SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/index.html b/website/public/index.html index 0282d64de2..11d70c1d97 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -109,6 +109,25 @@ /> + + + @@ -134,6 +153,7 @@
  • Try REA
  • Go deeper
  • Help & feedback
    • +
  • Community
    • @@ -1052,6 +1072,23 @@ SPEED: 6

    +
    +
    +

    Join the community

    +

    + Share investigations and get help from other REA users on Discord. + Follow REA on X for project updates. +

    +
    + Join the community + Follow @reatools on X +
    +
    +
    diff --git a/website/public/showcase/aegis/index.html b/website/public/showcase/aegis/index.html index 05a230501a..7949addfd3 100644 --- a/website/public/showcase/aegis/index.html +++ b/website/public/showcase/aegis/index.html @@ -108,6 +108,25 @@ /> + + + @@ -718,7 +737,8 @@ npx rea-agents@latest inspect-android-package \ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/showcase/ctf/index.html b/website/public/showcase/ctf/index.html index ed20029247..afe735fee2 100644 --- a/website/public/showcase/ctf/index.html +++ b/website/public/showcase/ctf/index.html @@ -107,6 +107,25 @@ /> + + + @@ -688,7 +707,8 @@ chmod +x ms_flag_checker SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/showcase/dx-ball/index.html b/website/public/showcase/dx-ball/index.html index d47449978c..d89d2bf5bc 100644 --- a/website/public/showcase/dx-ball/index.html +++ b/website/public/showcase/dx-ball/index.html @@ -107,6 +107,25 @@ /> + + + @@ -696,7 +715,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/showcase/index.html b/website/public/showcase/index.html index f0a537f696..168afcde91 100644 --- a/website/public/showcase/index.html +++ b/website/public/showcase/index.html @@ -107,6 +107,25 @@ /> + + + @@ -228,7 +247,8 @@ SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/showcase/notion/index.html b/website/public/showcase/notion/index.html index 5120d8ca61..c276d16c9b 100644 --- a/website/public/showcase/notion/index.html +++ b/website/public/showcase/notion/index.html @@ -107,6 +107,25 @@ /> + + + @@ -534,7 +553,8 @@ mimeTypes: SourceMIT licenseCommunity + >Join the communityFollow us on X diff --git a/website/public/showcase/th04/index.html b/website/public/showcase/th04/index.html index e08f8f2a3c..9a89af6399 100644 --- a/website/public/showcase/th04/index.html +++ b/website/public/showcase/th04/index.html @@ -107,6 +107,25 @@ /> + + + @@ -711,7 +730,8 @@ no_aim: SourceMIT licenseCommunity + >Join the communityFollow us on X From 934ffd28d15f4fbe6687ca8986a949e1b41a07d1 Mon Sep 17 00:00:00 2001 From: keemsisi Date: Sat, 10 Oct 2026 08:09:18 +0100 Subject: [PATCH 42/74] perf(javascript): derive the application graph ID once createJavaScriptApplicationGraph hashed the whole graph to derive its graph_id, then parsed it with the public schema, whose invariant check hashed the same content again to verify that ID. Validate the content with every other invariant first, then derive graph_id once from the validated copy. The public schema still verifies graph_id for stored and untrusted graphs, and created graphs are unchanged. --- .../javascriptApplicationGraph.test.ts | 15 +++++++++++++ .../javascript/javascriptApplicationGraph.ts | 22 ++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/domain/javascript/javascriptApplicationGraph.test.ts b/src/domain/javascript/javascriptApplicationGraph.test.ts index efedf78c06..58f1279284 100644 --- a/src/domain/javascript/javascriptApplicationGraph.test.ts +++ b/src/domain/javascript/javascriptApplicationGraph.test.ts @@ -547,6 +547,21 @@ describe("JavaScript Application Graph: source maps and observations", () => { /Edge identifier/u, ); }); + + it("creates exactly the graph the public schema accepts", () => { + const graph = buildSyntheticJavaScriptApplicationGraph(); + expect(JSON.stringify(graph)).toBe( + JSON.stringify( + javascriptApplicationGraphSchema.parse(structuredClone(graph)), + ), + ); + + const { graph_id: _graphId, ...semantic } = structuredClone(graph); + firstOf(semantic.edges, "edge").edge_id = `jag_edge_${"f".repeat(64)}`; + expect(() => createJavaScriptApplicationGraph(semantic)).toThrow( + /Edge identifier/u, + ); + }); }); describe("JavaScript Application Graph: duplicate and orphan rejection", () => { diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts index ba122e1f06..e16aded759 100644 --- a/src/domain/javascript/javascriptApplicationGraph.ts +++ b/src/domain/javascript/javascriptApplicationGraph.ts @@ -304,8 +304,8 @@ const checkNode = ( type GraphRecord = z.infer; -const checkGraphInvariants = ( - graph: GraphRecord, +const checkGraphContent = ( + graph: JavaScriptApplicationGraphInput, context: z.RefinementCtx, ): void => { sortedUniqueIssue( @@ -394,7 +394,13 @@ const checkGraphInvariants = ( context, ); } +}; +const checkGraphInvariants = ( + graph: GraphRecord, + context: z.RefinementCtx, +): void => { + checkGraphContent(graph, context); const { graph_id: identifier, ...semantic } = graph; if ( identifier !== @@ -411,6 +417,9 @@ const checkGraphInvariants = ( export const javascriptApplicationGraphSchema = javascriptApplicationGraphRecordSchema.superRefine(checkGraphInvariants); +const javascriptApplicationGraphContentSchema = + javascriptApplicationGraphInputSchema.superRefine(checkGraphContent); + /** Fully validated JavaScript Application Graph. */ export type JavaScriptApplicationGraph = z.infer< typeof javascriptApplicationGraphSchema @@ -482,10 +491,11 @@ export const createJavaScriptApplicationGraph = ( }, limitations: uniqueSorted(parsed.limitations), }; - return javascriptApplicationGraphSchema.parse({ - ...semantic, - graph_id: `jag_${digestCanonicalValue(semantic, "JavaScript Application Graph")}`, - }); + const content = javascriptApplicationGraphContentSchema.parse(semantic); + return { + ...content, + graph_id: `jag_${digestCanonicalValue(content, "JavaScript Application Graph")}`, + }; }; const validatedImmutableApplicationGraphs = new WeakSet(); From aa23ec883f6c7101db184767a1f48cddb636d597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Sat, 10 Oct 2026 15:12:09 +0800 Subject: [PATCH 43/74] feat(clients): register Qwen Code during setup (#1415) * feat(clients): register Qwen Code during setup * fix(clients): honor Qwen home and shared skill discovery --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/installation.md | 15 +- scripts/verify-package-environment.mjs | 1 + src/application/ClientRegistrationStatus.ts | 1 + src/application/SupportedClients.ts | 31 ++- .../filesystem/clientLifecycle.test.ts | 1 + .../qwenClientConfiguration.test.ts | 183 ++++++++++++++++++ tests/fixtures/clientEnvironment.ts | 1 + 7 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 tests/boundary/filesystem/qwenClientConfiguration.test.ts diff --git a/docs/installation.md b/docs/installation.md index 38ea09f5a5..acc285e694 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -135,10 +135,17 @@ listed after the table because its connector is not one of these files: | Antigravity | `antigravity` | | GitHub Copilot CLI | `copilot_cli` | | Command Code | `commandcode` | +| Qwen Code | `qwen_code` | | VS Code | `vscode` | | Grok Build | `grok_build` | | OMP | `omp` | +For Qwen Code, setup registers `rea` in the `mcpServers` table of the +user-level `~/.qwen/settings.json`, or `$QWEN_HOME/settings.json` when configured. +Setup and doctor follow Qwen Code's tilde and working-directory-relative home +overrides. Qwen Code also discovers personal skills in `~/.agents/skills`, +where setup installs the bundled workflow. + For OMP, setup writes a `type: "stdio"` entry to the user-level `~/.omp/agent/mcp.json`. It follows `PI_CONFIG_DIR`, an absolute `PI_CODING_AGENT_DIR`, and the profile selected by `OMP_PROFILE` or @@ -194,10 +201,10 @@ save verified paths for an existing Ghidra installation. The bundled skill is installed where each selected client discovers personal skills: Claude Code uses `~/.claude/skills` (or -`$CLAUDE_CONFIG_DIR/skills` when configured), while other supported clients -use the shared `~/.agents/skills` directory. A mixed selection plans both -paths. Selecting the skill without a client uses the shared directory. Setup -leaves existing skill copies in other locations untouched. +`$CLAUDE_CONFIG_DIR/skills` when configured), while other supported clients, +including Qwen Code, use the shared `~/.agents/skills` directory. A mixed selection +plans the relevant paths. Selecting the skill without a client uses the shared +directory. Setup leaves existing skill copies in other locations untouched. `doctor --skill --json` verifies the selected copies against the bundled instructions and references. Consumers should use `identity.skill.state`, diff --git a/scripts/verify-package-environment.mjs b/scripts/verify-package-environment.mjs index f9ee010af8..1a97cfbab1 100644 --- a/scripts/verify-package-environment.mjs +++ b/scripts/verify-package-environment.mjs @@ -85,6 +85,7 @@ export async function verifyPackageEnvironment({ PI_CODING_AGENT_DIR: undefined, PI_CONFIG_DIR: undefined, PI_PROFILE: undefined, + QWEN_HOME: undefined, OPENCODE_CONFIG: undefined, XDG_CONFIG_HOME: join(home, ".config"), PATH: `${fakeBin}:${process.env.PATH ?? ""}`, diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index a657f1df4f..1f5a124736 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -91,6 +91,7 @@ export const readClientRegistrationStatuses = async ( PI_CODING_AGENT_DIR: options.environment.PI_CODING_AGENT_DIR, PI_CONFIG_DIR: options.environment.PI_CONFIG_DIR, PI_PROFILE: options.environment.PI_PROFILE, + QWEN_HOME: options.environment.QWEN_HOME, SAND_DATA_ROOT: options.environment.SAND_DATA_ROOT, XDG_CONFIG_HOME: options.environment.XDG_CONFIG_HOME, }, diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 97467a227c..52f5577cac 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -1,4 +1,4 @@ -import { isAbsolute, join } from "node:path"; +import { isAbsolute, join, resolve } from "node:path"; import { lstatSync } from "node:fs"; /** One supported client configuration location. */ @@ -35,6 +35,7 @@ interface ClientPathContext { readonly PI_CODING_AGENT_DIR?: string | undefined; readonly PI_CONFIG_DIR?: string | undefined; readonly PI_PROFILE?: string | undefined; + readonly QWEN_HOME?: string | undefined; readonly SAND_DATA_ROOT?: string | undefined; readonly XDG_CONFIG_HOME?: string | undefined; }; @@ -85,6 +86,25 @@ const claudeCodeMarkerDirectory = ({ home, env }: ClientPathContext): string => const codexDirectory = ({ home, env }: ClientPathContext): string => env.CODEX_HOME ?? join(home, ".codex"); +/** Match Qwen Code's home override, including tilde and cwd-relative paths. */ +const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { + const override = env.QWEN_HOME; + if (!override) return join(home, ".qwen"); + const expanded = + override === "~" + ? home + : override.startsWith("~/") || override.startsWith("~\\") + ? join( + home, + ...override + .slice(2) + .split(/[/\\]+/) + .filter(Boolean), + ) + : override; + return isAbsolute(expanded) ? expanded : resolve(expanded); +}; + const grokDirectory = ({ home, env }: ClientPathContext): string => env.GROK_HOME ?? join(home, ".grok"); @@ -272,6 +292,14 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ markerPath: commandCodeDirectory, format: "commandcode", }, + { + name: "qwen_code", + displayName: "Qwen Code", + configPath: (context: ClientPathContext) => + join(qwenCodeDirectory(context), "settings.json"), + markerPath: qwenCodeDirectory, + format: "json", + }, { name: "vscode", displayName: "VS Code", @@ -323,6 +351,7 @@ export const supportedClients = ( PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR, PI_CONFIG_DIR: process.env.PI_CONFIG_DIR, PI_PROFILE: process.env.PI_PROFILE, + QWEN_HOME: process.env.QWEN_HOME, SAND_DATA_ROOT: process.env.SAND_DATA_ROOT, XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, }, diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 88eab99539..297787a10b 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -59,6 +59,7 @@ describe("client configuration filesystem lifecycle", () => { "antigravity", "copilot_cli", "commandcode", + "qwen_code", "vscode", "grok_build", "omp", diff --git a/tests/boundary/filesystem/qwenClientConfiguration.test.ts b/tests/boundary/filesystem/qwenClientConfiguration.test.ts new file mode 100644 index 0000000000..0fae29640f --- /dev/null +++ b/tests/boundary/filesystem/qwenClientConfiguration.test.ts @@ -0,0 +1,183 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { parse as parseJsonc } from "jsonc-parser"; + +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { systemSetupHost } from "../../../src/application/SetupHost.js"; +import { supportedClients } from "../../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { PRODUCT_IDENTITY } from "../../../src/identity.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + clearClientLocationEnvironment, + NPX_REGISTRATION_COMMAND, +} from "../../fixtures/clientEnvironment.js"; + +const command = NPX_REGISTRATION_COMMAND; + +beforeEach(clearClientLocationEnvironment); +afterEach(() => vi.unstubAllEnvs()); + +const qwenPath = ( + home: string, + platform: NodeJS.Platform, + env: Parameters[2] = {}, +) => + supportedClients(home, platform, env).find( + ({ name }) => name === "qwen_code", + ); + +describe("Qwen Code configuration paths", () => { + it("resolves the user settings file and marker directory", () => { + expect(qwenPath("/home/a", "linux")).toEqual({ + name: "qwen_code", + displayName: "Qwen Code", + configPath: join("/home/a", ".qwen", "settings.json"), + markerPath: join("/home/a", ".qwen"), + format: "json", + }); + expect(qwenPath("/Users/a", "darwin")?.configPath).toBe( + join("/Users/a", ".qwen", "settings.json"), + ); + }); + + it.each([ + ["", join("/home/a", ".qwen")], + ["~", "/home/a"], + ["~/profiles/work", join("/home/a", "profiles", "work")], + ["~\\profiles\\work", join("/home/a", "profiles", "work")], + ["profiles/work", resolve("profiles/work")], + [" profile ", resolve(" profile ")], + ["$PROFILE", resolve("$PROFILE")], + ])("matches Qwen's QWEN_HOME resolution for %j", (override, directory) => { + vi.stubEnv("QWEN_HOME", override); + const client = qwenPath("/home/a", process.platform, { + QWEN_HOME: override, + }); + expect( + supportedClients("/home/a").find(({ name }) => name === "qwen_code"), + ).toEqual(client); + expect(client).toMatchObject({ + configPath: join(directory, "settings.json"), + markerPath: directory, + }); + }); +}); + +describe("Qwen Code client lifecycle", () => { + const qwenClient = async () => { + const home = await createTestTempDirectory("rea-qwen-"); + const client = qwenPath(home, process.platform, {}); + if (client?.markerPath === undefined) + throw new Error("missing Qwen Code client"); + await mkdir(client.markerPath, { recursive: true }); + return { home, client }; + }; + + it("registers rea in mcpServers while preserving unrelated settings and comments", async () => { + const { home, client } = await qwenClient(); + await writeFile( + client.configPath, + `{ + // Keep this note. + "model": { "name": "qwen3-coder-plus" }, + "mcpServers": { "other": { "command": "other-server" } } +} +`, + ); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("// Keep this note."); + expect(parseJsonc(text)).toEqual({ + model: { name: "qwen3-coder-plus" }, + mcpServers: { + other: { command: "other-server" }, + rea: { command: "npx", args: command.slice(1) }, + }, + }); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const removedText = await readFile(client.configPath, "utf8"); + const removed = parseJsonc(removedText); + expect(removed).toEqual({ + model: { name: "qwen3-coder-plus" }, + mcpServers: { other: { command: "other-server" } }, + }); + expect(removedText).toContain("// Keep this note."); + }); + + it("reports a package-runner registration as aligned", async () => { + const { home, client } = await qwenClient(); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const entry = resolve("scripts/rea.mjs"); + const status = ( + await readClientRegistrationStatuses(home, entry, { environment: {} }) + ).find(({ client: name }) => name === "qwen_code"); + expect(status).toMatchObject({ state: "aligned" }); + }); +}); + +describe("Qwen Code profile setup", () => { + it.each([false, true])( + "installs and removes the shared skill with a custom home: %s", + async (customHome) => { + const home = await createTestTempDirectory("rea-qwen-profile-"); + const profile = join(home, customHome ? "qwen-profile" : ".qwen"); + const environment = { + HOME: home, + USERPROFILE: home, + ...(customHome ? { QWEN_HOME: profile } : {}), + }; + await mkdir(profile); + const setup = systemSetupHost(undefined, environment); + const client = (await setup.detectedClients()).find( + ({ name }) => name === "qwen_code", + ); + if (client === undefined) throw new Error("missing Qwen Code client"); + expect(client.configPath).toBe(join(profile, "settings.json")); + expect(await setup.configureClient(client, {}, command)).toEqual({ + status: "configured", + }); + expect(await setup.installSkill([client.name])).toBe("installed"); + const skill = join(home, ".agents", "skills", PRODUCT_IDENTITY.skillName); + expect(await readFile(join(skill, "SKILL.md"), "utf8")).toContain( + "reverse-engineer-anything", + ); + expect(await setup.clientNeedsConfigure(client, {}, command)).toBe(false); + expect(await setup.skillNeedsInstall([client.name])).toBe(false); + expect(await setup.installSkill([client.name])).toBe("unchanged"); + expect( + ( + await readClientRegistrationStatuses( + home, + resolve("scripts/rea.mjs"), + { environment }, + ) + ).find(({ client: name }) => name === "qwen_code"), + ).toMatchObject({ state: "aligned" }); + const uninstall = systemUninstallHost(home, undefined, environment); + const registered = (await uninstall.clients()).find( + ({ name }) => name === "qwen_code", + ); + expect(registered).toEqual(client); + expect((await uninstall.removeClient(client)).status).toBe("removed"); + expect((await uninstall.removeSkill()).status).toBe("removed"); + await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" }); + if (customHome) + await expect(access(join(home, ".qwen"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }, + ); +}); diff --git a/tests/fixtures/clientEnvironment.ts b/tests/fixtures/clientEnvironment.ts index 79961e49d4..0db5a69ec8 100644 --- a/tests/fixtures/clientEnvironment.ts +++ b/tests/fixtures/clientEnvironment.ts @@ -22,6 +22,7 @@ export const CLIENT_LOCATION_ENVIRONMENT = [ "PI_CODING_AGENT_DIR", "PI_CONFIG_DIR", "PI_PROFILE", + "QWEN_HOME", "SAND_DATA_ROOT", "XDG_CONFIG_HOME", ] as const; From 3c7b176a3df18b3971d9ea9ff40f7ebbd85fa40a Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:12:17 -0700 Subject: [PATCH 44/74] perf(artifacts): deliver the inspected artifact graph once (#1473) inspect_artifact nested the complete inventory Evidence and then restated every artifact node and occurrence as an observation and every edge as a derived relationship. No caller reads those copies; comparison and tests use the nested inventory Evidence. On Transmission.app the result shrinks from 2.46 MB to 1.10 MB. Observations keep the root manifest and integrity findings. Co-authored-by: Claude Opus 5.5 --- src/contracts/artifactToolContracts.ts | 2 +- src/domain/artifactInspection.ts | 59 +------------------ .../artifactProvider.archiveInventory.test.ts | 9 +++ 3 files changed, 13 insertions(+), 57 deletions(-) diff --git a/src/contracts/artifactToolContracts.ts b/src/contracts/artifactToolContracts.ts index a6926a19fb..744ea34c3f 100644 --- a/src/contracts/artifactToolContracts.ts +++ b/src/contracts/artifactToolContracts.ts @@ -63,7 +63,7 @@ const artifact = < export const ARTIFACT_TOOL_CONTRACTS = [ artifact( "inspect_artifact", - "Inspect an active archive or application package in one call. Returns the complete content-addressed artifact graph, source Evidence, observations, relationships, integrity contradictions, hypotheses, unexplored branches, limitations, and next probes inline. Read-only DMG mounting is used automatically on supported macOS hosts. Integrity mismatches fail by default; record-and-continue keeps mismatches explicitly untrusted. It does not extract files; use extract_artifact to materialize its regular contents.", + "Inspect an active archive or application package in one call. Returns the complete content-addressed artifact graph once, as the nested inventory Evidence, with root and integrity observations, integrity contradictions, hypotheses, unexplored branches, limitations, and next probes inline. Read-only DMG mounting is used automatically on supported macOS hosts. Integrity mismatches fail by default; record-and-continue keeps mismatches explicitly untrusted. It does not extract files; use extract_artifact to materialize its regular contents.", artifactInventoryInputSchema, ), artifact( diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index f4c197b9c5..19ccb9b8f9 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -16,30 +16,12 @@ const artifactIdSchema = prefixedDigestSchema("art"); const inspectionObservationSchema = z.strictObject({ observation_id: prefixedDigestSchema("aio"), - kind: z.enum(["root-manifest", "artifact", "occurrence", "integrity"]), + kind: z.enum(["root-manifest", "integrity"]), subject: textSchema, value: jsonValueSchema, evidence_id: evidenceIdSchema, }); -const inspectionRelationshipSchema = z.strictObject({ - relationship_id: prefixedDigestSchema("air"), - relation: z.enum([ - "contains", - "extracts", - "slice-of", - "embeds", - "loads", - "maps-source", - "derived-from", - ]), - source_artifact_id: artifactIdSchema, - target_artifact_id: artifactIdSchema, - occurrence_id: prefixedDigestSchema("occ"), - logical_path: textSchema.nullable(), - evidence_id: evidenceIdSchema, -}); - const inspectionHypothesisSchema = z.strictObject({ hypothesis_id: prefixedDigestSchema("aih"), statement: textSchema, @@ -97,7 +79,6 @@ export const artifactInspectionResultSchema = z.strictObject({ ) .min(1), observations: z.array(inspectionObservationSchema), - derived_relationships: z.array(inspectionRelationshipSchema), hypotheses: z.array(inspectionHypothesisSchema), contradictions: z.array(inspectionContradictionSchema), unexplored_branches: z.array(unexploredBranchSchema), @@ -107,8 +88,6 @@ export const artifactInspectionResultSchema = z.strictObject({ substeps_completed: z.literal(1), observations_retained: z.number().int().min(0), observations_omitted: z.number().int().min(0), - relationships_retained: z.number().int().min(0), - relationships_omitted: z.number().int().min(0), hypotheses_retained: z.number().int().min(0), hypotheses_omitted: z.number().int().min(0), unexplored_branches_retained: z.number().int().min(0), @@ -136,10 +115,6 @@ export const createArtifactInspection = ( inventory, inventoryEvidence.evidence_id, ); - const relationships = allRelationships( - inventory, - inventoryEvidence.evidence_id, - ); const hypotheses = allHypotheses(inventory, inventoryEvidence.evidence_id); const nextProbes = probesFor(inventory); const branches = allUnexploredBranches( @@ -149,14 +124,12 @@ export const createArtifactInspection = ( ); const retained = { observations, - relationships, hypotheses, branches, probes: nextProbes, }; const omissions = { observations: 0, - relationships: 0, hypotheses: 0, branches: 0, probes: 0, @@ -184,7 +157,6 @@ export const createArtifactInspection = ( }, ], observations: retained.observations, - derived_relationships: retained.relationships, hypotheses: retained.hypotheses, contradictions: contradictions(inventory, inventoryEvidence.evidence_id), unexplored_branches: retained.branches, @@ -194,8 +166,6 @@ export const createArtifactInspection = ( substeps_completed: 1 as const, observations_retained: retained.observations.length, observations_omitted: omissions.observations, - relationships_retained: retained.relationships.length, - relationships_omitted: omissions.relationships, hypotheses_retained: retained.hypotheses.length, hypotheses_omitted: omissions.hypotheses, unexplored_branches_retained: retained.branches.length, @@ -224,6 +194,8 @@ const parseInventoryEvidence = (input: unknown): Evidence => { return evidence; }; +// Artifact, occurrence, and edge records stay in the nested inventory Evidence; +// restating each one here would repeat the complete graph in every result. const allObservations = ( inventory: ArtifactInventoryResult, evidenceId: string, @@ -234,12 +206,6 @@ const allObservations = ( inventory.manifest, evidenceId, ), - ...inventory.nodes.map((node) => - observation("artifact", node.artifact_id, node, evidenceId), - ), - ...inventory.occurrences.map((occurrence) => - observation("occurrence", occurrence.occurrence_id, occurrence, evidenceId), - ), ...inventory.integrity_contradictions.map((value) => observation("integrity", value.contradiction_id, value, evidenceId), ), @@ -263,25 +229,6 @@ const observation = ( }; }; -const allRelationships = ( - inventory: ArtifactInventoryResult, - evidenceId: string, -): ArtifactInspectionResult["derived_relationships"] => - inventory.edges.map((edge) => { - const semantic = { - relation: edge.relation, - source_artifact_id: edge.parent_artifact_id, - target_artifact_id: edge.child_artifact_id, - occurrence_id: edge.occurrence_id, - logical_path: edge.logical_path, - evidence_id: evidenceId, - }; - return { - relationship_id: `air_${digestCanonicalValue(semantic, "Artifact inspection")}`, - ...semantic, - }; - }); - const allHypotheses = ( inventory: ArtifactInventoryResult, evidenceId: string, diff --git a/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts b/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts index 06fe705277..2da4a98f4f 100644 --- a/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts +++ b/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts @@ -60,6 +60,15 @@ describe("artifact archive inventory", () => { expect(inspectionEvidence.evidence_links).toEqual( inspection.evidence_links, ); + // The graph is delivered once, in the nested inventory Evidence. + expect( + artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ).occurrences, + ).toHaveLength(3); + expect(inspection.observations.map(({ kind }) => kind)).toEqual([ + "root-manifest", + ]); const cancellation = new AbortController(); cancellation.abort(); await expect( From 7296f2b5bfff921b2ca16f942a09616c8adf2b3f Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:13:34 -0700 Subject: [PATCH 45/74] perf(javascript)!: stop repeating the semantic identifier strategy per fact (#1474) Every semantic graph node and relation carried an identifier_strategy object whose schema admits exactly one value per record type. The constant made up about 9% of analyze_javascript_application output (Obsidian app.asar: 12.04 MB to 10.96 MB), and semantic traces repeated it on every returned fact. The node_id and relation_id schema descriptions now state how each identifier is derived. BREAKING CHANGE: Semantic graph nodes and relations no longer include identifier_strategy, and relation IDs no longer hash that constant. Evidence recorded by earlier versions does not validate against the new semantic graph schema. Co-authored-by: Claude Opus 5.5 --- src/domain/javascript/javascriptSemanticGraph.ts | 8 -------- .../javascript/javascriptSemanticGraphSchemas.ts | 16 ++++++---------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index 69b3bb9678..9ab2dd485c 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -127,10 +127,6 @@ export const createJavaScriptSemanticGraphNode = ( ...parsed, application_node_ids: uniqueSorted(parsed.application_node_ids), evidence: evidenceContexts.intern(parsed.evidence), - identifier_strategy: { - strategy: "semantic-content-sha256" as const, - stability: "artifact-version" as const, - }, }; return javaScriptSemanticNodeSchema.parse({ ...semantic, @@ -147,10 +143,6 @@ export const createJavaScriptSemanticGraphRelation = ( const semantic = { ...parsed, evidence: evidenceContexts.intern(parsed.evidence), - identifier_strategy: { - strategy: "semantic-content-sha256" as const, - stability: "relationship-exact" as const, - }, }; return javaScriptSemanticRelationSchema.parse({ ...semantic, diff --git a/src/domain/javascript/javascriptSemanticGraphSchemas.ts b/src/domain/javascript/javascriptSemanticGraphSchemas.ts index fb03f1a4d2..4048ae698e 100644 --- a/src/domain/javascript/javascriptSemanticGraphSchemas.ts +++ b/src/domain/javascript/javascriptSemanticGraphSchemas.ts @@ -233,11 +233,9 @@ export const javaScriptSemanticNodeInputSchema = z.strictObject({ export const javaScriptSemanticNodeSchema = javaScriptSemanticNodeInputSchema.extend({ evidence: javaScriptSemanticEvidenceReferenceSchema, - node_id: semanticNodeIdSchema, - identifier_strategy: z.strictObject({ - strategy: z.literal("semantic-content-sha256"), - stability: z.literal("artifact-version"), - }), + node_id: semanticNodeIdSchema.describe( + "SHA-256 of the node kind and identity; stable for one artifact version.", + ), }); /** One directed semantic relationship before its ID is derived. */ @@ -254,11 +252,9 @@ export const javaScriptSemanticRelationInputSchema = z.strictObject({ export const javaScriptSemanticRelationSchema = javaScriptSemanticRelationInputSchema.extend({ evidence: javaScriptSemanticEvidenceReferenceSchema, - relation_id: semanticRelationIdSchema, - identifier_strategy: z.strictObject({ - strategy: z.literal("semantic-content-sha256"), - stability: z.literal("relationship-exact"), - }), + relation_id: semanticRelationIdSchema.describe( + "SHA-256 of the exact relation content, including its evidence reference.", + ), }); /** Reasons semantic analysis can leave an explicit frontier. */ From 1f0775684b2cf8089bddfa7e2f0ee9aa6517c8a8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:21:16 +0800 Subject: [PATCH 46/74] fix(artifacts): keep case-distinct archive names in inventory (#1480) ZIP member names are case-sensitive. Inventory was folding them with a filesystem case rule, so shrunk APKs never became evidence for the Android application graph. Logical registration now keeps both spellings and records an English Unicode case-folding limitation. Extraction fails only when the destination directory already holds the other spelling. Co-authored-by: Cursor Agent --- docs/android-analysis.md | 9 +- src/artifacts/ArtifactPaths.ts | 168 ++++++++++++++++-- src/artifacts/SafeOutputTree.ts | 35 +++- src/artifacts/inventory/scanCanonical.ts | 23 ++- src/artifacts/inventory/scanReader.ts | 16 +- src/domain/analysisErrorPresentation.ts | 23 ++- src/domain/android/androidApplication.ts | 37 +--- src/domain/zipPackageFormat.ts | 3 +- .../artifactProvider.archiveSafety.test.ts | 70 ++++++-- .../artifactProvider.failureDetail.test.ts | 91 +++++++--- tests/composition/android/application.test.ts | 45 +++++ 11 files changed, 425 insertions(+), 95 deletions(-) diff --git a/docs/android-analysis.md b/docs/android-analysis.md index efcb05c645..849aeb2b11 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -126,8 +126,13 @@ also stops target hashing before provider admission. These operations do not execute the APK or provide split APK/AAB handling, signature validation, full resource-table semantics, native-library analysis or -Android runtime capture. Existing artifact inventory/extraction tools can supply -archive evidence. Project that Evidence with the execution-free inventory tool: +Android runtime capture. An `.aab` suffix is a ZIP archive, not an APK, so +Android tools and the application graph do not treat it as a standalone APK. +Existing artifact inventory/extraction tools can supply archive evidence. +Inventory keeps ZIP names that differ only in case and records that a +case-insensitive destination cannot store both spellings. Extraction fails +only when that destination directory cannot keep both names. Project that +Evidence with the execution-free inventory tool: ```sh rea project-android-application-graph '{"inventory_evidence":[]}' diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index d5ec35cb4b..ce8d6c5153 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -1,9 +1,24 @@ import { posix } from "node:path"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; const DRIVE_OR_UNC = /^(?:[A-Za-z]:|\\|\/\/)/u; +/** + * English Unicode case folding used only to describe logical names a + * case-insensitive destination might collapse. It is not a filesystem case table. + */ +const ENGLISH_UNICODE_CASE_NOTE = + "This comparison is not a filesystem observation."; + +/** Inventory limitation when logical names collide under English Unicode case folding. */ +export const ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION = `Logical archive names collide under English (en-US) Unicode case folding; a case-insensitive destination cannot store both spellings. ${ENGLISH_UNICODE_CASE_NOTE}`; + +/** User-facing prefix for a destination that cannot keep two logical spellings. */ +export const DESTINATION_CASE_COLLISION_PREFIX = + "Destination filesystem cannot store both "; + /** Normalize an untrusted logical archive path without touching filesystem. */ export const normalizeArtifactPath = (input: string): string => { if ( @@ -30,7 +45,7 @@ export const normalizeArtifactPath = (input: string): string => { return normalized; }; -/** Reject exact, Unicode-normalized, case-folded, and prefix collisions. */ +/** Reject exact and same-spelling prefix collisions. Case-distinct names stay distinct. */ export class ArtifactPathRegistry { readonly #root: PathTrieNode = { kind: undefined, children: new Map() }; @@ -43,16 +58,10 @@ export class ArtifactPathRegistry { "path", `Artifact prefix conflict: ${path}`, ); - const foldedPart = part.toLocaleLowerCase("en-US"); - let child = node.children.get(foldedPart); - if (child !== undefined && child.spelling !== part) - throw new ArtifactReaderFailure( - "path", - `Artifact path collision: ${path} differs only in case from ${[...parts.slice(0, index), child.spelling].join("/")}`, - ); + let child = node.children.get(part); if (child === undefined) { - child = { spelling: part, kind: undefined, children: new Map() }; - node.children.set(foldedPart, child); + child = { kind: undefined, children: new Map() }; + node.children.set(part, child); } node = child; if (index === parts.length - 1) { @@ -73,7 +82,144 @@ export class ArtifactPathRegistry { } interface PathTrieNode { - readonly spelling?: string; kind: "file" | "directory" | "symlink" | "slice" | undefined; readonly children: Map; } + +interface CaseNode { + readonly bySpelling: Map; + readonly spellingsByFold: Map; + readonly terminals: string[]; +} + +const emptyCaseNode = (): CaseNode => ({ + bySpelling: new Map(), + spellingsByFold: new Map(), + terminals: [], +}); + +const englishCaseFold = (part: string): string => + part.toLocaleLowerCase("en-US"); + +/** + * Record English (en-US) Unicode case collisions on each affected occurrence. + * Returns the inventory limitation when any logical names collide. + */ +export const annotateEnglishUnicodeCaseCollisions = ( + occurrences: readonly { + readonly logical_path: string; + limitations: string[]; + }[], +): string | undefined => { + const collisions = englishUnicodeCaseCollisions( + occurrences.map(({ logical_path }) => logical_path), + ); + if (collisions.size === 0) return undefined; + for (const occurrence of occurrences) { + const others = collisions.get(occurrence.logical_path); + if (others === undefined || others.length === 0) continue; + occurrence.limitations.push( + `Logical path ${occurrence.logical_path} collides under English (en-US) Unicode case folding with ${others.join(", ")}; a case-insensitive destination cannot store both. ${ENGLISH_UNICODE_CASE_NOTE}`, + ); + } + return ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION; +}; + +/** + * Name the directory-listing spelling that differs from the requested segment + * only by English (en-US) Unicode case. Exact matches are the same logical name. + */ +export const destinationCaseCollisionMessage = ( + requestedPath: string, + existingNames: readonly string[], +): string | undefined => { + const slash = requestedPath.lastIndexOf("/"); + const segment = slash < 0 ? requestedPath : requestedPath.slice(slash + 1); + if (existingNames.includes(segment)) return undefined; + const folded = englishCaseFold(segment); + const existing = existingNames + .filter((name) => englishCaseFold(name) === folded) + .sort(compareUnicodeCodePoints)[0]; + if (existing === undefined) return undefined; + const parent = slash < 0 ? "" : requestedPath.slice(0, slash + 1); + return `${DESTINATION_CASE_COLLISION_PREFIX}${requestedPath} and ${parent}${existing}; inventory retains both logical names.`; +}; + +const englishUnicodeCaseCollisions = ( + paths: readonly string[], +): ReadonlyMap => { + const root = emptyCaseNode(); + for (const path of paths) { + if (path.length === 0 || path === ".") continue; + let node = root; + for (const part of path.split("/")) { + const folded = englishCaseFold(part); + const spellings = node.spellingsByFold.get(folded); + if (spellings === undefined) node.spellingsByFold.set(folded, [part]); + else if (!spellings.includes(part)) spellings.push(part); + let child = node.bySpelling.get(part); + if (child === undefined) { + child = emptyCaseNode(); + node.bySpelling.set(part, child); + } + node = child; + } + node.terminals.push(path); + } + const collisions = new Map>(); + recordCaseCollisions(root, collisions); + return new Map( + [...collisions.entries()].map(([path, others]) => [ + path, + [...others].sort(compareUnicodeCodePoints), + ]), + ); +}; + +const recordCaseCollisions = ( + node: CaseNode, + collisions: Map>, +): void => { + for (const spellings of node.spellingsByFold.values()) + if (spellings.length > 1) noteFoldedSpellings(node, spellings, collisions); + for (const child of node.bySpelling.values()) + recordCaseCollisions(child, collisions); +}; + +const noteFoldedSpellings = ( + node: CaseNode, + spellings: readonly string[], + collisions: Map>, +): void => { + const groups = spellings.map((spelling) => { + const child = node.bySpelling.get(spelling); + return child === undefined ? [] : terminalsUnder(child); + }); + for (const [index, group] of groups.entries()) + noteCaseGroup( + group, + groups.flatMap((candidate, candidateIndex) => + candidateIndex === index ? [] : candidate, + ), + collisions, + ); +}; + +const noteCaseGroup = ( + group: readonly string[], + others: readonly string[], + collisions: Map>, +): void => { + for (const path of group) { + const set = collisions.get(path) ?? new Set(); + for (const other of others) if (other !== path) set.add(other); + collisions.set(path, set); + } +}; + +const terminalsUnder = (node: CaseNode): readonly string[] => { + const paths = [...node.terminals]; + for (const child of node.bySpelling.values()) + paths.push(...terminalsUnder(child)); + return paths; +}; diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index fafb6fee4a..05ce0ac680 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -4,6 +4,7 @@ import { lstat, mkdir, open, + readdir, realpath, rm, type FileHandle, @@ -14,6 +15,7 @@ import { streamChunkToBuffer } from "./StreamBytes.js"; import { ArtifactPathRegistry, + destinationCaseCollisionMessage, normalizeArtifactPath, } from "./ArtifactPaths.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; @@ -134,7 +136,8 @@ export class SafeOutputTree { constants.O_WRONLY | constants.O_NOFOLLOW, 0o600, - ).catch((cause: unknown) => { + ).catch(async (cause: unknown) => { + await throwIfDestinationCaseCollision(dirname(destination), path, cause); throw new ArtifactReaderFailure( "path", `Could not exclusively create extraction path: ${path}`, @@ -245,17 +248,28 @@ export class SafeOutputTree { if (fileName === undefined) throw new ArtifactReaderFailure("path", "Invalid extraction path"); let current = this.#outputRoot; + let logicalParent = ""; for (const part of parts) { + const parentDirectory = current; + const logicalPath = + logicalParent.length === 0 ? part : `${logicalParent}/${part}`; current = join(current, part); + let created = true; await mkdir(current, { mode: 0o700 }).catch((cause: unknown) => { if (!isAlreadyExists(cause)) throw cause; + created = false; }); + if (!created) + await throwIfDestinationCaseCollision(parentDirectory, logicalPath); const metadata = await lstat(current); - if (!metadata.isDirectory() || metadata.isSymbolicLink()) + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + await throwIfDestinationCaseCollision(parentDirectory, logicalPath); throw new ArtifactReaderFailure( "path", `Unsafe extraction parent: ${relativePath}`, ); + } + logicalParent = logicalPath; } return join(current, fileName); } @@ -330,6 +344,23 @@ const isNotFound = (cause: unknown): boolean => const isAlreadyExists = (cause: unknown): boolean => cause instanceof Error && "code" in cause && cause.code === "EEXIST"; +/** Fail when this directory already holds another spelling of the requested segment. */ +const throwIfDestinationCaseCollision = async ( + parentDirectory: string, + logicalPath: string, + cause?: unknown, +): Promise => { + const names = await readdir(parentDirectory).catch(() => undefined); + if (names === undefined) return; + const message = destinationCaseCollisionMessage(logicalPath, names); + if (message === undefined) return; + throw new ArtifactReaderFailure( + "path", + message, + cause === undefined ? undefined : { cause }, + ); +}; + const abortIfNeeded = (signal?: AbortSignal): void => { if (signal?.aborted === true) throw new ArtifactReaderFailure( diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.ts index abd5246934..0719e58f4c 100644 --- a/src/artifacts/inventory/scanCanonical.ts +++ b/src/artifacts/inventory/scanCanonical.ts @@ -73,12 +73,12 @@ export const scanCanonicalArtifactInventory = async ( try { if (reader instanceof AsarArtifactReader) await reader.prepareContainer(rootDigest?.sha256, options.signal); - const { nodes, occurrences, pendingContradictions } = await scanReader( - reader, - ownedReaders, - options.signal, - integrity, - ); + const { + nodes, + occurrences, + pendingContradictions, + caseCollisionLimitation, + } = await scanReader(reader, ownedReaders, options.signal, integrity); outcome = { kind: "completed", snapshot: await buildInventorySnapshot({ @@ -91,6 +91,7 @@ export const scanCanonicalArtifactInventory = async ( nodes, occurrences, pendingContradictions, + caseCollisionLimitation, }), }; } catch (cause: unknown) { @@ -130,6 +131,7 @@ interface SnapshotBuildInput { readonly nodes: Map; readonly occurrences: MutableOccurrence[]; readonly pendingContradictions: PendingIntegrityContradiction[]; + readonly caseCollisionLimitation: string | undefined; } const buildInventorySnapshot = async ( @@ -202,7 +204,12 @@ const buildInventorySnapshot = async ( edges: orderedEdges, provenance: reader?.provenance() ?? [], integrity_contradictions: integrityContradictions, - limitations: buildLimitations(rootFormat, reader, integrityContradictions), + limitations: buildLimitations( + rootFormat, + reader, + integrityContradictions, + input.caseCollisionLimitation, + ), }; }; @@ -310,8 +317,10 @@ const buildLimitations = ( rootFormat: ArtifactOccurrence["artifact_format"], reader: ArtifactReader | undefined, integrityContradictions: readonly IntegrityContradiction[], + caseCollisionLimitation: string | undefined, ): string[] => [ ...inventoryLimitations(rootFormat, reader), + ...(caseCollisionLimitation === undefined ? [] : [caseCollisionLimitation]), ...(integrityContradictions.length === 0 ? [] : [ diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 1ee691a74f..89b1b628e8 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -1,6 +1,7 @@ import { isAbsolute } from "node:path"; import { + annotateEnglishUnicodeCaseCollisions, ArtifactPathRegistry, normalizeArtifactPath, } from "../ArtifactPaths.js"; @@ -64,10 +65,16 @@ export const scanReader = async ( readonly nodes: Map; readonly occurrences: MutableOccurrence[]; readonly pendingContradictions: PendingIntegrityContradiction[]; + readonly caseCollisionLimitation: string | undefined; }> => { const { nodes, occurrences, pendingContradictions } = emptyScan(); if (reader === undefined) - return { nodes, occurrences, pendingContradictions }; + return { + nodes, + occurrences, + pendingContradictions, + caseCollisionLimitation: undefined, + }; const context: ScanContext = { reader, signal, @@ -90,7 +97,12 @@ export const scanReader = async ( context.occurrenceByPath, context.expandedContainerIds, )?.occurrence_id ?? null; - return { nodes, occurrences, pendingContradictions }; + return { + nodes, + occurrences, + pendingContradictions, + caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences), + }; }; const visitArtifactEntries = async ( diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 7c71a9b5dc..b4fe4ad64b 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -191,7 +191,7 @@ export const analysisErrorUserMessage = (error: AnalysisError): string => { const standardMessage = standardErrorMessage(error._tag); if (standardMessage !== undefined) return standardMessage; if (error instanceof ArtifactOperationError) - return artifactMessage(error.operation, error.reason); + return artifactMessage(error.operation, error.reason, error.detail); if (error instanceof EvidenceReferenceError) return error.reason === "missing" ? `Evidence ${error.evidenceId} is not retained in this session. Supply complete inline Evidence, re-run its producer, or import its bundle before using this reference.` @@ -329,13 +329,13 @@ const START_FAILURE_TAGS: ReadonlySet = new Set([ const artifactMessage = ( operation: ArtifactOperationError["operation"], reason: ArtifactOperationError["reason"], + detail?: string, ): string => { if (reason === "cancelled") return "Artifact operation was cancelled. Start it again when ready."; if (reason === "limit") return "Artifact is too large to process safely. Narrow the requested path or use a smaller artifact."; - if (reason === "path") - return "Artifact contains an unsafe or conflicting internal path. Inspect the reported path and correct the artifact before retrying."; + if (reason === "path") return artifactPathMessage(detail); if (reason === "unavailable") return "Artifact processing is unavailable for the current target or host. Check artifact support and required tools."; if (reason === "integrity" && operation === "analyze_javascript_application") @@ -345,6 +345,23 @@ const artifactMessage = ( return "Artifact could not be read or written. Check file access and try again."; }; +const artifactPathMessage = (detail: string | undefined): string => { + // Kept in step with DESTINATION_CASE_COLLISION_PREFIX. Domain does not import + // the artifact adapter that produces the detail. + if ( + detail !== undefined && + detail.startsWith("Destination filesystem cannot store both ") + ) + return detail; + if ( + detail !== undefined && + (detail.startsWith("Artifact path is absolute or unsafe:") || + detail.startsWith("Artifact path is not normalized:")) + ) + return "Artifact contains an unsafe internal path. Inspect the reported path and correct the artifact before retrying."; + return "Artifact contains a conflicting internal path. Inspect the reported path before retrying."; +}; + const evidenceFileMessage = ({ operation, reason, diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index 745b8e0c49..6744257490 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -125,6 +125,8 @@ export const projectAndroidApplication = ( "Manifest, resource, signing, and bytecode semantics require a dedicated Android provider; this projection reports exact inventory paths and hashes only.", "Runtime families are inferred from inventory formats and paths; filename suffixes do not establish valid DEX or JVM class bytes.", "Bridge candidates are path-based hypotheses, not decoded JNI declarations or observed runtime calls.", + "A bridge basis is inferred from the native path and is repeated for every managed component.", + "ZIP inventory cannot see the APK Signing Block, so an empty signing array does not mean the APK is unsigned.", ...(bridgeProjection.coverage.status === "partial" ? [ `Bridge candidate pairs exceeded the projection safety budget; ${bridgeProjection.coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, @@ -157,23 +159,12 @@ export const projectAndroidApplication = ( }; const classify = (all: readonly Component[]) => ({ - manifests: all.filter( - ({ path, format }) => - format === "android-manifest" || - /(?:^|\/)AndroidManifest\.xml$/u.test(path), - ), - resources: all.filter( - ({ path, format }) => - format === "android-resources" || /(?:^|\/)resources\.arsc$/u.test(path), - ), - dex: all.filter( - ({ path, format }) => - format === "dex" || /(?:^|\/)[^/]+\.dex$/iu.test(path), - ), - jvm_classes: all.filter( - ({ path, format }) => - format === "jvm-class" || /(?:^|\/)[^/]+\.class$/iu.test(path), + manifests: all.filter(({ path }) => + /(?:^|\/)AndroidManifest\.xml$/u.test(path), ), + resources: all.filter(({ path }) => /(?:^|\/)resources\.arsc$/u.test(path)), + dex: all.filter(({ path }) => /(?:^|\/)[^/]+\.dex$/iu.test(path)), + jvm_classes: all.filter(({ path }) => /(?:^|\/)[^/]+\.class$/iu.test(path)), native_libraries: all.filter( ({ path, format }) => format === "elf" || /(?:^|\/)lib\/[^/]+\/[^/]+\.so$/iu.test(path), @@ -189,19 +180,9 @@ const runtimeFamilies = (all: readonly Component[]) => { const families = new Set< AndroidApplicationProjectionResult["runtime_families"][number] >(); - if ( - all.some( - ({ path, format }) => - format === "dex" || path.toLowerCase().endsWith(".dex"), - ) - ) + if (all.some(({ path }) => path.toLowerCase().endsWith(".dex"))) families.add("dalvik-art"); - if ( - all.some( - ({ path, format }) => - format === "jvm-class" || path.toLowerCase().endsWith(".class"), - ) - ) + if (all.some(({ path }) => path.toLowerCase().endsWith(".class"))) families.add("java-kotlin"); if (all.some(({ format }) => format === "elf")) families.add("native"); if (all.some(({ format }) => format === "javascript-bundle")) diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts index c45109135a..39a8f86a58 100644 --- a/src/domain/zipPackageFormat.ts +++ b/src/domain/zipPackageFormat.ts @@ -22,7 +22,8 @@ export const zipPackageFormatForPath = ( const lower = path.toLowerCase(); if (lower.endsWith(".zip")) return "zip"; if (lower.endsWith(".ipa")) return "ipa"; - if (lower.endsWith(".apk") || lower.endsWith(".aab")) return "apk"; + // An Android App Bundle is a ZIP archive, not a standalone APK. + if (lower.endsWith(".apk")) return "apk"; if (lower.endsWith(".msix") || lower.endsWith(".msixbundle")) return "msix"; if (lower.endsWith(".appx") || lower.endsWith(".appxbundle")) return "appx"; return undefined; diff --git a/tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts b/tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts index 368d029dc3..92f88aa9da 100644 --- a/tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts +++ b/tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts @@ -9,6 +9,8 @@ import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js import { createDirectAnalysis } from "../../../../src/composition/directAnalysis.js"; import { ArtifactPathRegistry, + destinationCaseCollisionMessage, + ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, normalizeArtifactPath, } from "../../../../src/artifacts/ArtifactPaths.js"; import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; @@ -95,17 +97,8 @@ describe("artifact archive safety", () => { }); const registry = new ArtifactPathRegistry(); registry.add("A.js", "file"); - let collisionError: unknown; - try { - registry.add("a.js", "file"); - } catch (error: unknown) { - collisionError = error; - } - expect(collisionError).toBeInstanceOf(ArtifactReaderFailure); - expect(collisionError).toMatchObject({ - reason: "path", - message: "Artifact path collision: a.js differs only in case from A.js", - }); + expect(() => registry.add("a.js", "file")).not.toThrow(); + expect(() => registry.add("A.js", "file")).toThrow(ArtifactReaderFailure); for (const [filePath, childPath] of [ ["Foo", "foo/bar"], @@ -113,11 +106,15 @@ describe("artifact archive safety", () => { ] as const) { const casePrefixRegistry = new ArtifactPathRegistry(); casePrefixRegistry.add(filePath, "file"); - expect(() => casePrefixRegistry.add(childPath, "file")).toThrow( - ArtifactReaderFailure, - ); + expect(() => casePrefixRegistry.add(childPath, "file")).not.toThrow(); } + const exactPrefixRegistry = new ArtifactPathRegistry(); + exactPrefixRegistry.add("Foo", "file"); + expect(() => exactPrefixRegistry.add("Foo/bar", "file")).toThrow( + ArtifactReaderFailure, + ); + const sameDirectoryRegistry = new ArtifactPathRegistry(); sameDirectoryRegistry.add("Foo/one.js", "file"); expect(() => sameDirectoryRegistry.add("Foo/two.js", "file")).not.toThrow(); @@ -126,7 +123,50 @@ describe("artifact archive safety", () => { caseVariantDirectoryRegistry.add("Foo/one.js", "file"); expect(() => caseVariantDirectoryRegistry.add("foo/two.js", "file"), - ).toThrow(ArtifactReaderFailure); + ).not.toThrow(); + }); + + it("names a destination case collision from the directory listing", () => { + expect( + destinationCaseCollisionMessage("res/2f.xml", ["2F.xml", "other.xml"]), + ).toBe( + "Destination filesystem cannot store both res/2f.xml and res/2F.xml; inventory retains both logical names.", + ); + expect( + destinationCaseCollisionMessage("res/2f.xml", ["2f.xml", "2F.xml"]), + ).toBeUndefined(); + }); + + it("extracts logical names that differ only in case", async () => { + const root = await createTestTempDirectory("rea-case-distinct-zip-"); + const path = join(root, "fixture.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("Main.js", new TextReader("upper")); + await writer.add("main.js", new TextReader("lower")); + await writeFile(path, await writer.close()); + const parsed = await parseBinaryTarget(path); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + const result = await inventory(parsed.value); + expect(result.limitations).toContain( + ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, + ); + expect( + result.occurrences + .filter(({ logical_path }) => logical_path !== ".") + .map(({ logical_path }) => logical_path) + .sort(), + ).toEqual(["Main.js", "main.js"]); + const output = join(root, "output"); + const extracted = await new ArtifactProvider(process.env) + .createClient(parsed.value) + .execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + ); + expect(extracted.ok).toBe(true); + expect(await readFile(join(output, "Main.js"), "utf8")).toBe("upper"); + expect(await readFile(join(output, "main.js"), "utf8")).toBe("lower"); }); }); const inventory = async (targetValue: BinaryTarget) => { diff --git a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts index dcec03c93b..e2b2fd3859 100644 --- a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts +++ b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts @@ -6,30 +6,73 @@ import { writeOrderedZip } from "../../../fixtures/artifactEntryOrder.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; +import { + destinationCaseCollisionMessage, + ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, +} from "../../../../src/artifacts/ArtifactPaths.js"; import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; +import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; +import { ArtifactOperationError } from "../../../../src/domain/artifactOperationError.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; -it.each([ - [ - ["Main.js", "main.js"], - "Artifact path collision: main.js differs only in case from Main.js", - ], - [["a\\b.js"], 'Artifact path is absolute or unsafe: "a\\\\b.js"'], -] as const)( - "reports the failed path constraint for %j", - async (entries, detail) => { - const root = await createTestTempDirectory("rea-artifact-detail-"); - const path = join(root, "fixture.zip"); - await writeOrderedZip(path, entries); - const target = await parseBinaryTarget(path); - if (!target.ok) throw new Error("expected a ZIP target"); - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute("inventory_artifact", {}); - if (result.ok) throw new Error("expected the path constraint to fail"); - expect(projectAnalysisError(result.error)).toMatchObject({ - code: "artifact_operation_failed", - details: { operation: "inventory_artifact", reason: "path", detail }, - }); - }, -); +it("keeps logical names that differ only in case and records the limitation", async () => { + const root = await createTestTempDirectory("rea-artifact-detail-"); + const path = join(root, "fixture.zip"); + await writeOrderedZip(path, ["Main.js", "main.js"]); + const target = await parseBinaryTarget(path); + if (!target.ok) throw new Error("expected a ZIP target"); + const result = await new ArtifactProvider(process.env) + .createClient(target.value) + .execute("inventory_artifact", {}); + if (!result.ok) throw result.error; + const inventory = artifactInventoryResultSchema.parse(result.value.result); + expect(inventory.occurrences.map(({ logical_path }) => logical_path)).toEqual( + expect.arrayContaining(["Main.js", "main.js"]), + ); + expect(inventory.limitations).toContain( + ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, + ); + for (const logicalPath of ["Main.js", "main.js"]) { + expect( + inventory.occurrences + .find((occurrence) => occurrence.logical_path === logicalPath) + ?.limitations.join("\n"), + ).toContain("English (en-US) Unicode case folding"); + } +}); + +it("reports an unsafe path as a hostile artifact path", async () => { + const root = await createTestTempDirectory("rea-artifact-detail-"); + const path = join(root, "fixture.zip"); + await writeOrderedZip(path, ["a\\b.js"]); + const target = await parseBinaryTarget(path); + if (!target.ok) throw new Error("expected a ZIP target"); + const result = await new ArtifactProvider(process.env) + .createClient(target.value) + .execute("inventory_artifact", {}); + if (result.ok) throw new Error("expected the path constraint to fail"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "artifact_operation_failed", + message: + "Artifact contains an unsafe internal path. Inspect the reported path and correct the artifact before retrying.", + details: { + operation: "inventory_artifact", + reason: "path", + detail: 'Artifact path is absolute or unsafe: "a\\\\b.js"', + }, + }); +}); + +it("returns a destination case collision as the user-facing path message", () => { + const detail = destinationCaseCollisionMessage("res/2f.xml", ["2F.xml"]); + expect(detail).toEqual(expect.any(String)); + if (detail === undefined) return; + expect( + projectAnalysisError( + new ArtifactOperationError("extract_artifact", "path", undefined, detail), + ), + ).toMatchObject({ + message: detail, + details: { reason: "path", detail }, + }); +}); diff --git a/tests/composition/android/application.test.ts b/tests/composition/android/application.test.ts index 082a706b0b..3529dc4e85 100644 --- a/tests/composition/android/application.test.ts +++ b/tests/composition/android/application.test.ts @@ -86,6 +86,12 @@ describe("Android application projection", () => { expect(left.bridge_candidates).toEqual([ expect.objectContaining({ basis: "react-native-convention" }), ]); + expect(left.limitations).toEqual( + expect.arrayContaining([ + "A bridge basis is inferred from the native path and is repeated for every managed component.", + "ZIP inventory cannot see the APK Signing Block, so an empty signing array does not mean the APK is unsigned.", + ]), + ); expect(JSON.stringify(left)).not.toContain("opaque signing"); }); @@ -146,6 +152,45 @@ describe("Android application projection", () => { }); }); +describe("case-distinct APK resource names", () => { + it("inventories and projects an APK whose resource names differ only in case", async () => { + const root = await createTestTempDirectory("rea-android-case-"); + const path = join(root, "Shrunk.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add( + "AndroidManifest.xml", + new Uint8ArrayReader(Uint8Array.from([3, 0, 8, 0])), + ); + await writer.add( + "classes.dex", + new Uint8ArrayReader( + Uint8Array.from([0x64, 0x65, 0x78, 0x0a, 0x30, 0x33, 0x35, 0]), + ), + ); + await writer.add("res/2F.xml", new TextReader("upper")); + await writer.add("res/2f.xml", new TextReader("lower")); + await writeFile(path, await writer.close()); + + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + const result = projectAndroidApplicationEvidence({ + inventory_evidence: [inventory], + }); + const projection = androidApplicationProjectionResultSchema.parse( + requireSuccessfulProjection(result).normalized_result, + ); + expect(projection.root_format).toBe("apk"); + expect(projection.components.manifests).toHaveLength(1); + expect(projection.components.dex).toHaveLength(1); + expect(JSON.stringify(inventory)).toContain("res/2F.xml"); + expect(JSON.stringify(inventory)).toContain("res/2f.xml"); + expect(JSON.stringify(inventory)).toContain( + "English (en-US) Unicode case folding", + ); + }); +}); + describe("Android bridge candidate coverage", () => { it("returns every component and bridge candidate from the inventory", async () => { const root = await createTestTempDirectory("rea-android-complete-"); From b09d02d93cbadfa0ced21e7c489c7d47fb317228 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:55:18 +0800 Subject: [PATCH 47/74] feat(website): add a blog for methods and project notes --- website/public/blog/index.html | 154 ++++++++++++++++++ website/public/examples/dino-lab/index.html | 3 +- website/public/faq/index.html | 1 + website/public/first-investigation/index.html | 3 +- website/public/get-started/index.html | 3 +- website/public/guides/browser/index.html | 3 +- website/public/guides/index.html | 3 +- website/public/guides/javascript/index.html | 3 +- website/public/guides/native/index.html | 3 +- website/public/index.html | 1 + website/public/showcase/aegis/index.html | 1 + website/public/showcase/ctf/index.html | 1 + website/public/showcase/dx-ball/index.html | 1 + website/public/showcase/index.html | 1 + website/public/showcase/notion/index.html | 1 + website/public/showcase/th04/index.html | 1 + 16 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 website/public/blog/index.html diff --git a/website/public/blog/index.html b/website/public/blog/index.html new file mode 100644 index 0000000000..eec6da2a62 --- /dev/null +++ b/website/public/blog/index.html @@ -0,0 +1,154 @@ + + + + + + Reverse Engineering Methods and Project Notes — REA + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + REAReverse Engineer Anything + +
    + + + + + + + +
    +
    +
    +
    +
    +

    Home / Blog

    +

    Methods and project notes.

    +

    + How we approach reverse engineering, game reconstruction, ports and + mods. +

    +
    +
    +
    +

    Game reconstruction
    Article outline

    +
    +

    Reconstructing Touhou with AI agents

    +

    + A repeatable workflow, illustrated by TH08, TH095 and TH04: build + on existing work, carry lessons into a new game, and adapt the + method to an older platform. +

    +

    By N0zoM1z0 · Outline in progress

    +
    +
    +
    +
    + +

    + + Top + + + diff --git a/website/public/examples/dino-lab/index.html b/website/public/examples/dino-lab/index.html index d0bc84c765..02452bda9b 100644 --- a/website/public/examples/dino-lab/index.html +++ b/website/public/examples/dino-lab/index.html @@ -63,7 +63,8 @@ >Reverse Engineer Anything diff --git a/website/public/faq/index.html b/website/public/faq/index.html index fa4a64a83f..3ab51b9d39 100644 --- a/website/public/faq/index.html +++ b/website/public/faq/index.html @@ -63,6 +63,7 @@ >