Compare commits

..
Author SHA1 Message Date
279cca0828 experiment(re): validate guard slices and record token tradeoffs
Verify three repaired zlib mappings, a SQLite call guard and all ordinary 16-bit header values against retained native evidence. Record producer omissions, failed-run costs and the small-function context counterexample. Preserve existing capture outputs with exclusive writes.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-16 02:40:13 +00:00
bf86511477 experiment(re): compose native SSA uses and branch dependencies
Retain bounded native snapshots and public MCP evidence, then query explicit SSA relations without interpreting reconstructed C. Preserve unknown flow, cloned block identity and dependency frontiers.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-16 02:27:32 +00:00
500e5bb4be experiment(re): distinguish point facts from object selectors
Apply the interior-reference counterexample to the facade: label exact-address scope and emit location endpoints while preserving caller selectors separately. Carry canonical target-content provenance through experimental MCP replies and verify it on the real client. Keep original measurements pinned to the v1 response shape. Nine experiment checks and real Kuna lifecycle/context checks pass.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-15 17:21:05 +00:00
3e747586b4 experiment(re): record native sharing and composition counterexamples
Measure twelve equivalent MCP navigation workflows, native reference plans on SQLite/zlib, a compiled interior-reference oracle and a failed exact code-mapping composition. Preserve failed attempts and measurement limits, including unavailable live-agent effectiveness evidence. Add bounded native probes and deterministic mapping tests. All 191 Kuna cases pass serially; nine experiment tests and one-checker typecheck pass.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-15 17:17:02 +00:00
b4bdd61ace experiment(kuna): preserve typed reference facts at their owner
Expose source-owned reference kind, endpoints, function identities and instruction text for experiments. Keep the existing address-only public result and retained-cache shape; materialize richer facts only when explicitly requested. Record prior art, composition rules and the distinction between fact, transport and compute units. Verified 18 producer boundary cases and an independently checked native interior-reference example.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-15 16:46:28 +00:00
fa8d6a7a23 experiment(re): compare fact granularity through real MCP
Add a non-shipping MCP facade over the existing admitted analysis port, real-provider comparison runner and offline token accounting. Keep the full catalog, source Evidence and per-input failures. Separate verification runs from equivalent-work timings. Kuna zlib comparisons pass; broader comparisons remain in progress.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-15 16:35:23 +00:00
ce4fe65170 experiment(re): define composable facts and context boundaries
Start a non-integrating experiment around fact identity, per-input outcomes, paging and shared primitive composition. Include deterministic boundary checks; real-provider comparisons remain in progress.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-15 16:28:41 +00:00
478 changed files with 8726 additions and 33407 deletions

No files matched your search

-12
View File
@@ -60,7 +60,6 @@ jobs:
web_runtime: ${{ steps.plan.outputs.web_runtime }}
web_source_map: ${{ steps.plan.outputs.web_source_map }}
nativeaot: ${{ steps.plan.outputs.nativeaot }}
ghidra_bridge: ${{ steps.plan.outputs.ghidra_bridge }}
website: ${{ steps.plan.outputs.website }}
matrix: ${{ steps.plan.outputs.matrix }}
test_selection: ${{ steps.plan.outputs.test_selection }}
@@ -140,11 +139,6 @@ jobs:
- name: Validate workflow syntax and expressions
if: needs.changes.outputs.workflows == 'true'
run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes=
# Code tiers run this source-only policy test in their existing shards.
# Workflow-only changes need the guard, without preparing runtime artifacts.
- name: Check action pins for workflow-only changes
if: needs.changes.outputs.workflows == 'true' && needs.changes.outputs.code != 'true'
run: node scripts/run-exact-tests.mjs run tests/boundary/filesystem/workflowActionPinning.test.ts
- name: Record static task source and execution identity
continue-on-error: true
if: needs.changes.outputs.code == 'true'
@@ -905,11 +899,6 @@ jobs:
if: needs.changes.outputs.nativeaot == 'true'
uses: ./.github/workflows/nativeaot-fixtures.yml
ghidra_bridge:
needs: [changes, static]
if: needs.changes.outputs.ghidra_bridge == 'true'
uses: ./.github/workflows/ghidra-bridge-sources.yml
website:
needs: [changes, static]
if: needs.changes.outputs.website == 'true'
@@ -945,7 +934,6 @@ jobs:
web_runtime,
web_source_map,
nativeaot,
ghidra_bridge,
website,
]
runs-on: ubuntu-24.04
@@ -1,48 +0,0 @@
name: Ghidra bridge source compilation
on:
workflow_dispatch:
workflow_call:
permissions:
contents: read
env:
HUSKY: "0"
jobs:
ghidra-bridge-sources:
name: Compile packaged Ghidra bridge sources
runs-on: ubuntu-24.04
timeout-minutes: 15
env:
GHIDRA_RELEASE_URL: https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_12.1.4_build/ghidra_12.1.4_PUBLIC_20260921.zip
GHIDRA_RELEASE_SHA256: ddac49f903da9d5bac833e5cc79395098b9c33cfd3279be5f31bd00387d2d4db
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Validate the SDK pin against the provider's verified release
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
# Only the jars and application.properties are extracted: the check
# compiles against them and never launches Ghidra or imports a target.
- name: Acquire checksum-verified Ghidra jars in isolated runner storage
shell: bash
run: |
ghidra_root="${RUNNER_TEMP}/rea-ghidra"
mkdir -p "${ghidra_root}"
curl -fsSL --retry 3 --write-out 'ghidra_download_bytes=%{size_download} ghidra_download_seconds=%{time_total}\n' -o "${ghidra_root}/ghidra.zip" "${GHIDRA_RELEASE_URL}"
echo "${GHIDRA_RELEASE_SHA256} ${ghidra_root}/ghidra.zip" | sha256sum -c -
unzip -q "${ghidra_root}/ghidra.zip" '*/Ghidra/*.jar' '*/Ghidra/application.properties' -d "${ghidra_root}"
rm "${ghidra_root}/ghidra.zip"
properties="$(find "${ghidra_root}" -path '*/Ghidra/application.properties' -print -quit)"
test -n "${properties}"
echo "GHIDRA_INSTALL_DIR=$(dirname "$(dirname "${properties}")")" >> "${GITHUB_ENV}"
# JAVA_HOME_<major>_X64 comes from the runner image, not the env context.
- name: Compile bridge sources with the verified runner JDK
shell: bash
run: |
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
test -n "${!java_home_variable}"
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
+3 -3
View File
@@ -22,7 +22,7 @@ jobs:
sudo apt-get update
sudo apt-get install --no-install-recommends -y g++-mingw-w64-x86-64
- run: npm run build:windows-native
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
- uses: actions/upload-artifact@v4
with:
name: cutter-windows-native
path: native/windows/build/
@@ -53,7 +53,7 @@ jobs:
uses: ./.github/actions/npm-cache
- run: npm ci
- if: matrix.platform == 'Windows'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
uses: actions/download-artifact@v4
with:
name: cutter-windows-native
path: native/windows/build/
@@ -67,7 +67,7 @@ jobs:
env:
EXPECTED_PLATFORM: ${{ matrix.platform }}
run: npm run verify:cutter
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
- uses: actions/upload-artifact@v4
if: always()
with:
name: cutter-upstream-${{ matrix.platform }}
-2
View File
@@ -20,8 +20,6 @@ Give each interpretation and resource a clear owner. Consumers must use that own
Keep proof of ownership separate from validity, editability, and readiness. Malformed owned data may still be safely removable. Unresolved identity or applicability must remain unknown rather than be treated as absent, foreign, or aligned.
For lifecycle changes, follow [ownership and completion invariants](docs/tool-design.md#own-the-lifecycle) and select the applicable [lifecycle regressions](docs/testing.md#lifecycle-regressions). Include late handshakes, cancellation during final cleanup, mixed child outcomes, and cleanup retry where those stages exist; active-execution cancellation alone does not cover finalization.
Assess brittleness in touched code during implementation, debugging, review, and maintenance. Fix the producing representation or ownership rule before adding downstream repairs. Normalize once at the owning boundary using format-aware parsers. Remove superseded representations, aliases, wrappers, and redundant tests after verifying affected consumers. Introduce abstractions only when they remove observed complexity.
When fixing a bug, inspect the producing boundary and affected consumers for duplicated interpretation or unclear ownership. Complete any directly related refactor in the same change and remove superseded code. Passing regression tests alone does not establish that the brittleness is resolved. If related cleanup remains, explain what remains and why.
+1 -9
View File
@@ -1057,10 +1057,6 @@ def _serve_connection(connection):
):
raise PermissionError("Invalid bridge capability")
authenticated = True
# The connection grace period only bounds unauthenticated idle
# clients. Native analysis and a connected owner's idle time
# remain unbounded after capability validation.
connection.settimeout(None)
_write_message(file, {"id": request_id, "event": {
"type": "progress",
"phase": "hopper_bridge",
@@ -1152,17 +1148,13 @@ def _run():
server.bind(REA_SOCKET)
os.chmod(REA_SOCKET, 0o600)
server.listen(1)
# A bridge can start after its caller's startup/cleanup deadline. Bound
# the first accept as well as reconnects so an orphan cannot pin Hopper's
# dedicated Python thread and prevent later documents from starting REA.
server.settimeout(REA_RECONNECT_TIMEOUT_SECONDS)
try:
while True:
try:
connection, _ = server.accept()
except socket.timeout:
break
connection.settimeout(REA_RECONNECT_TIMEOUT_SECONDS)
connection.settimeout(None)
if _serve_connection(connection):
break
# Match REA's cleanup handshake budget without pinning Hopper's
+4 -48
View File
@@ -555,38 +555,6 @@ def _failed_launch_description(process):
return None
def _retire_failed_launch(process, config, listener, stdout, stderr):
"""Reconcile an SBProcess returned alongside a failed Launch error."""
if not process.IsValid():
stdout.close()
stderr.close()
return False
pid = process.GetProcessID()
if pid > 0:
# Even an already-exited process needs its PID published so the host
# can distinguish a completed launch failure from an unknown target.
with open(config["pid_path"], "w") as handle:
handle.write(str(pid))
if process.GetState() != lldb.eStateExited:
stdout.mark_launched()
stderr.mark_launched()
if pid > 0:
# Hold a surviving target until the host has captured its start
# identity. If that proof is unavailable, Kill still addresses
# this exact SBProcess created by the current launch request.
_wait_for_identity_ack(config)
process.Kill()
_wait_for_exit(listener, process, 5)
exit_observed = process.GetState() == lldb.eStateExited
expected_exit = exit_observed
stdout.close(expected_exit=expected_exit)
stderr.close(expected_exit=expected_exit)
return exit_observed
def trace(debugger, config):
global _TRACE_CONTEXT
debugger.SetAsync(True)
@@ -662,9 +630,8 @@ def trace(debugger, config):
)
process = target.Launch(_launch_info(target, config, listener), error)
if not error.Success() or not process.IsValid():
exit_observed = _retire_failed_launch(
process, config, listener, stdout, stderr
)
stdout.close()
stderr.close()
return {
"status": "launch-error",
"error": error.GetCString(),
@@ -673,26 +640,15 @@ def trace(debugger, config):
if process.IsValid()
else {}
),
**(
{"target_process_exit_observed": True}
if exit_observed
else {}
),
}
launch_failure = _failed_launch_description(process)
if launch_failure is not None:
exit_observed = _retire_failed_launch(
process, config, listener, stdout, stderr
)
stdout.close(expected_exit=True)
stderr.close(expected_exit=True)
return {
"status": "launch-error",
"error": launch_failure,
"target_process_created": True,
**(
{"target_process_exit_observed": True}
if exit_observed
else {}
),
}
stdout.mark_launched()
stderr.mark_launched()
-132
View File
@@ -15,8 +15,6 @@ fake_lldb = sys.modules.setdefault("lldb", types.ModuleType("lldb"))
fake_lldb.LLDB_INVALID_ADDRESS = -1
fake_lldb.eLaunchFlagDebug = 2
fake_lldb.eLaunchFlagStopAtEntry = 4
fake_lldb.eStateExited = 10
fake_lldb.SBEvent = lambda: object()
spec = importlib.util.spec_from_file_location(
"rea_lldb_tracer", os.path.join(os.path.dirname(__file__), "rea_lldb_tracer.py")
)
@@ -25,136 +23,6 @@ spec.loader.exec_module(tracer)
class NativeTraceBudgetTests(unittest.TestCase):
def test_failed_launch_does_not_infer_exit_from_a_failed_kill(self):
class Process:
def IsValid(self):
return True
def GetProcessID(self):
return 12345
def GetState(self):
return 6
def Kill(self):
return None
class Sink:
close_exit = None
def mark_launched(self):
pass
def close(self, expected_exit=False):
self.close_exit = expected_exit
with tempfile.TemporaryDirectory() as directory:
stdout, stderr = Sink(), Sink()
with patch.object(tracer, "_wait_for_identity_ack", return_value="unavailable"), patch.object(
tracer, "_wait_for_exit", return_value=None
):
exited = tracer._retire_failed_launch(
Process(), {"pid_path": os.path.join(directory, "pid")}, object(), stdout, stderr
)
self.assertFalse(exited)
self.assertFalse(stdout.close_exit)
self.assertFalse(stderr.close_exit)
def test_failed_launch_publishes_pid_and_retires_live_process(self):
class Process:
state = 6
killed = False
def IsValid(self):
return True
def GetProcessID(self):
return 12345
def GetState(self):
return self.state
def Kill(self):
self.killed = True
self.state = tracer.lldb.eStateExited
class Sink:
launched = False
close_exit = None
def mark_launched(self):
self.launched = True
def close(self, expected_exit=False):
self.close_exit = expected_exit
with tempfile.TemporaryDirectory() as directory:
pid_path = os.path.join(directory, "pid")
process = Process()
stdout, stderr = Sink(), Sink()
def acknowledge(config):
with open(config["pid_path"], "r") as pid_file:
self.assertEqual(pid_file.read(), "12345")
return "readable"
with patch.object(tracer, "_wait_for_identity_ack", acknowledge):
exited = tracer._retire_failed_launch(
process,
{"pid_path": pid_path},
object(),
stdout,
stderr,
)
self.assertTrue(exited)
self.assertTrue(process.killed)
self.assertTrue(stdout.launched)
self.assertTrue(stderr.launched)
self.assertTrue(stdout.close_exit)
self.assertTrue(stderr.close_exit)
def test_failed_launch_records_when_valid_process_is_already_exited(self):
class Process:
def IsValid(self):
return True
def GetProcessID(self):
return 12345
def GetState(self):
return tracer.lldb.eStateExited
def Kill(self):
raise AssertionError("an exited process must not be killed")
class Sink:
close_exit = None
def mark_launched(self):
raise AssertionError("an exited process has no output to drain")
def close(self, expected_exit=False):
self.close_exit = expected_exit
with tempfile.TemporaryDirectory() as directory:
pid_path = os.path.join(directory, "pid")
stdout, stderr = Sink(), Sink()
with patch.object(
tracer,
"_wait_for_identity_ack",
side_effect=AssertionError("exit is already observed"),
):
exited = tracer._retire_failed_launch(
Process(), {"pid_path": pid_path}, object(), stdout, stderr
)
with open(pid_path, "r") as pid_file:
self.assertEqual(pid_file.read(), "12345")
self.assertTrue(exited)
self.assertTrue(stdout.close_exit)
self.assertTrue(stderr.close_exit)
def test_pre_target_integrity_failure_reports_no_process_created(self):
original_debugger = getattr(tracer.lldb, "SBDebugger", None)
-10
View File
@@ -108,13 +108,6 @@ The `replaced` result is `null`: the successful install response does not say
whether the package was installed fresh or replaced, and REA does not query the
device first. The install verdict and host APK digest remain available.
Push and install consume a private snapshot of the selected local input. The
reported size and SHA-256 identify those consumed bytes; `local_path` and
`apk_path` retain the original input path, whose contents may change later.
Snapshot creation needs temporary disk space proportional to the input size.
Failed snapshot cleanup remains owned and is retried before later operations
and during shutdown.
## Interpret the result
- **Directory names retain whitespace and newlines.** Symlink targets do too,
@@ -175,9 +168,6 @@ and during shutdown.
- Cancellation during a package pull preserves caller-selected output. The
error includes partial observations identifying completed artifacts and the
interrupted local path; remaining APKs were not attempted.
- Screen capture checks cancellation before publishing its exclusive output.
If cancellation arrives after publication, the error retains the committed
file's path, digest and dimensions; the file remains available as evidence.
- Bugreports collect into isolated staging and publish with exclusive creation;
filename collisions select the next numeric suffix. If publication fails, the
completed archive remains in staging. The error's partial observation gives
-18
View File
@@ -147,24 +147,6 @@ received signal alone does not establish its exact cause. Dynamic tags require
a complete DT_NULL inside PT_DYNAMIC; interpreter names and ranges end at the
first NUL, with any padding still represented by the original segment range.
## Ghidra process failures
When output from the failing Ghidra startup or operation reports
`java.lang.OutOfMemoryError`, REA returns `resource_constraint`. Java heap or GC
overhead exhaustion includes heap-specific recovery advice. Metaspace and array
size failures do not recommend increasing the heap; native-thread exhaustion is
reported as a capacity constraint with unknown limits.
`configured_jvm_max_heap` records the explicit `-Xmx` value selected by REA's
launch builder. It is a configuration fact, not measured memory use or proof of
the effective JVM limit; JVM environment options may affect that limit.
When a Windows script wrapper selects the JVM, this value remains unknown.
Historical stdout and stderr remain available, but classification uses only the
failing operation's `failure_output`. Output draining waits at most one second,
subject to earlier cancellation or the startup deadline. `output_incomplete`
marks a snapshot taken before stream closure; cleanup ownership remains with
the session until release succeeds.
## Portable PE resources
`inspect_pe_resources` / `inspect-pe-resources` inspects an explicit PE32 or
+8 -19
View File
@@ -66,25 +66,14 @@ Electron `file://` pages use a separate provider and target boundary; see [elect
## Provider authorities
CDP page capture and V8 Inspector observation attach to an already-running
target without launching it, driving actions, or evaluating target code.
Playwright scenarios drive actions and capture step snapshots in either a
provider-owned browser or a caller-selected external browser. Separate providers
preserve these authority and lifecycle differences. They share one exclusion
vocabulary (`BrowserExclusionReason`): every denial reason maps onto it, even
where a wire schema keeps a narrower historical bucket.
CDP attachment is not a guarantee that the browser preserves all state.
Chromium can clear another client's `navigator.platform` emulation when REA's
page session disconnects, even if that session sent no emulation commands.
This affects page inspection, bundle capture, session observation, screenshots,
WebMCP discovery, and Electron page inspection; target listing alone does not
attach a page session. REA cannot read the other client's complete
user-agent/client-hint configuration and does not replay guessed settings.
Use a disposable browser when preserving externally configured emulation is
required, or have the owning client reapply its complete configuration afterward.
See [the reproduction and Chromium ownership limitation](https://github.com/morluto/rea/issues/4089).
CDP page capture, V8 Inspector observation, and Playwright scenarios stay
separate providers because their authorities differ: CDP and V8 attach to an
already-running target and never launch, drive, evaluate, or mutate it, while
Playwright providers own and drive the runtime (launch browsers, run actions,
capture step snapshots). Merging them would mix attach-only and owned-process
lifecycles in one contract. What they share instead is one exclusion
vocabulary (`BrowserExclusionReason`): every denial reason in every stack maps
onto it, even where a wire schema keeps a narrower historical bucket.
Existing static application Evidence and passive web/Electron captures can be
combined later through
[JavaScript static/runtime reconciliation](javascript-runtime-reconciliation.md).
-9
View File
@@ -144,15 +144,6 @@ distinction: it closes and deletes provider-owned profiles, but only disconnects
from an external CDP browser. Real-browser verification checks both outcomes and
confirms that an attached external browser remains alive.
Keeping that browser alive does not guarantee its emulation state survives.
Chromium can clear caller-owned `navigator.platform` emulation when a page CDP
session disconnects, including sessions that sent no emulation commands. REA
cannot snapshot the other client's complete user-agent/client-hint settings and
does not replay a partial reconstruction. Use launch mode to keep this effect
inside a provider-owned browser, or have the external owning client reapply its
complete configuration after capture. Connected captures report this limitation;
see [issue #4089](https://github.com/morluto/rea/issues/4089).
`environment.locale` sets Intl formatting in both modes. Launch also sets
`navigator.language`, `navigator.languages`, and the request `Accept-Language`.
Connect preserves the caller's page and request languages; every attached
-6
View File
@@ -168,12 +168,6 @@ be verified. See [MCP contracts](mcp-contracts.md) for session lifecycle and
[Ghidra first-query deadlines](mcp-contracts.md#ghidra-first-query-deadlines-and-recovery)
for import, client timeouts and recovery.
If one-shot analysis fails and closing its session also fails, the original
error details remain inline and `details.cleanup_error` describes the later
close failure. Each diagnostic retains its own reported resources; the later
failure does not establish whether an earlier resource was released. Captured
output and partial observations remain in the original details.
## Save and reuse analysis snapshots
A snapshot retains successful analysis results for later queries. REA reuses
-6
View File
@@ -2,12 +2,6 @@
REA can attach to an Electron/Chromium CDP endpoint and inspect existing `file://` renderer pages without evaluating JavaScript or invoking Electron APIs. The request supplies the endpoint directly; REA accepts only literal-port loopback HTTP endpoints. Selecting an endpoint exposes every Electron target it serves, including local file paths and page metadata.
Disconnecting the inspection's page session can clear caller-owned
`navigator.platform` emulation inside Chromium. REA cannot restore another
client's complete user-agent/client-hint configuration. Use a disposable target
or let the owning client reapply its configuration; see the shared
[CDP state limitation](browser-observation.md#provider-authorities).
This passive runtime surface is distinct from the target-free static
[`analyze_javascript_application`](javascript-artifact-reconstruction.md)
workflow. Static analysis reads a local ASAR or extracted directory without
-6
View File
@@ -139,12 +139,6 @@ overview merely because its source artifact is unchanged.
## Admitted function-analysis semantics
Concurrent callers share one startup while retaining independent cancellation.
Only the final departing caller abandons startup and waits for its cleanup.
New attempts wait for that cleanup; failed cleanup retains its original error
and owned directories until a successful close. Closing the provider reports a
provider-state failure to live callers rather than cancelling their signals.
| Concern | Ghidra contract |
| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Decompiler lifetime | One persistent `DecompInterface` is opened for the imported Program and disposed during bridge shutdown. Native decompilation has no fixed per-function deadline. External functions or functions without bodies return `null`; cancellation and native failure remain distinct. |
+2 -9
View File
@@ -39,8 +39,8 @@ container format, architecture, word size and byte order. A recognized
- `module.dependencies`, with the same fields and their original order;
- `module.settings`, retaining ordered key/value entries and duplicate keys;
- `module.unparsed_lines`, `module.unparsed_line_bytes_base64` and
`module.complete`, which says whether every present module-text line was
parsed. It does not establish that the dependency inventory is complete.
`module.complete`, so unfamiliar, malformed, conflicting or undecodable records
do not disappear behind a complete result.
Missing checksums are `null`; empty fields and development/local replacement
versions remain unchanged. Build settings such as `GOOS`, `GOARCH` or `vcs.revision`
@@ -50,13 +50,6 @@ base64 while other decodable module records are still reported; the reader does
not insert replacement characters or reject the entire artifact for those
strings.
When no main-module (`mod`) record was decoded, `module.main` is `null` and
dependency coverage is unknown. Successfully decoded dependency records remain
visible. GOPATH-mode builds may contain only a package `path` and build settings;
their empty dependency array does not establish the absence of dependencies.
The result's limitations state this explicitly. A module-mode build with a main
module record and no dependency records has an observed empty dependency list.
## Supported boundary
The built-in TypeScript reader supports ELF, PE and thin Mach-O images with Go's
+6 -24
View File
@@ -179,8 +179,6 @@ as well, so it is broader than this Claude-only migration.
history under `CLAUDE_CONFIG_DIR`. Setup follows the same
variable: it writes the registration to `$CLAUDE_CONFIG_DIR/.claude.json` and
the skill to `$CLAUDE_CONFIG_DIR/skills`.
For REA setup, an empty `CLAUDE_CONFIG_DIR` is treated as unset: registration
goes to `~/.claude.json` and the skill goes to `~/.claude/skills`.
```bash
CLAUDE_CONFIG_DIR=~/.claude-rea npx rea-agents setup --client claude_code
@@ -637,11 +635,6 @@ bridge readiness. For large targets, set `REA_HOPPER_STARTUP_TIMEOUT_MS` to a
decimal integer from 1 to 2,147,483,647 milliseconds in the environment that
starts REA. CLI and MCP use the same setting; invalid values fail configuration
validation. An MCP client's own request timeout must also allow enough time.
`open_binary` validates and selects a target without waiting for native provider
readiness. During Hopper startup, `binary_session.provider_operation_health`
reports `busy` at stage `launch`; `analysis_activity` describes bridge requests
and can remain idle until the socket is ready. Cleanup work reports `busy` at
stage `connection`.
If macOS denies Apple Events, allow the application that starts REA under
**System Settings → Privacy & Security → Automation**, then retry. REA releases
@@ -651,12 +644,9 @@ Arguments containing quotes, backslashes, or line breaks remain unchanged but
cannot establish that a failed handoff opened no document. Other failed handoffs
retain ownership until document closure is confirmed. If a timed-out bridge
becomes ready later, `close_binary` connects to that run, authenticates its
identity, and requests document shutdown. Once its listener is ready, the bridge
allows 30 seconds for the first connection and capability validation. After a
connection drops, it allows the same grace period for an authenticated cleanup
reconnect, then releases Hopper's Python execution thread. Initial analysis is
outside this listener grace period, and authenticated native work is not bounded
by it. Expiry does not establish document closure, so REA
identity, and requests document shutdown. After a connection drops, the bridge
allows 30 seconds for an authenticated cleanup reconnect, then releases Hopper's
Python execution thread. Expiry does not establish document closure, so REA
retains unconfirmed ownership. Retry close while the bridge remains available.
If a Linux launcher exists but cannot start, inspect its shared libraries:
@@ -746,17 +736,9 @@ Once document and process closure are confirmed, the lease can be released even
if temporary-file cleanup fails. Another client can then launch, but the owning
client must finish its retained cleanup before starting again.
An unconfirmed external document can be retried while its authenticated bridge
remains available. The same authentication and process-identity capture apply
when the first successful handshake occurs during cleanup after a startup
timeout. REA can confirm closure if that exact authenticated process exits,
including while a shutdown request is in flight. A helper exit alone is
insufficient, and an application that never authenticated has no verified
process identity. If no matching bridge becomes available, inspect and close
the reported document in Hopper before ending the owning REA connection.
Export the Evidence bundle before `close_binary` clears session records.
`binary_session.cleanup_pending` remains true while provider cleanup is retained,
and provider health remains visible even with `open: false`; that flag alone
does not establish successful cleanup.
remains connected. If that bridge has disconnected, another close cannot confirm
the document: inspect and close the reported document in Hopper before ending
the owning REA connection.
### Hopper in CI
-11
View File
@@ -68,17 +68,6 @@ The node vocabulary covers:
- runtime script instance;
- an explicit `unknown` kind when classification is not justified.
Static network endpoints can have `resolution: "partial"` when the analyzer
recognizes a request call but cannot resolve its target to one exact value.
The observation's `value` then preserves the source expression, and its
evidence points to the call site. Partial and exact endpoints have distinct
identities even when their displayed values match. Partial identities also retain
the source site: matching expression text at different sites does not establish
the same runtime target. This records observed source syntax; it does not
establish that the call ran or contacted that endpoint.
Graph coverage describes the analyzed artifact scope and resource limits, not
runtime execution. A partial target alone does not imply omitted source.
The directed relation vocabulary is `contains`, `loads`, `imports`, `maps_to`,
`exposes`, `sends`, `invokes`, `handles`, `calls`, `persists_to`, `observed_as`,
and `changed_from`. Provider names are deliberately absent from both
+2 -30
View File
@@ -95,14 +95,6 @@ matches exactly. Otherwise the result recommends provider-neutral follow-up
tools and reports `requires-provider-analysis`; it never starts or switches a
provider implicitly.
A handoff requires the add-on's exact content identity. Unavailable unpacked
companions remain visible in the trace graph with their canonical path,
availability and container-backed Evidence. They yield no actionable handoff
until their bytes are available and the application is reanalyzed; the archive
digest does not identify the missing add-on. An empty handoff list does not prove
that the trace contains no native components. Source partial coverage remains
visible independently of handoff eligibility.
## Semantic relation tracing
`trace_javascript_semantics` queries the authenticated companion graph without
@@ -125,17 +117,6 @@ registrations/removals and dispatch, Node timers and cancellation handles, async
defaults, request construction and response consumers, parse/coercion/
validation boundaries, and built-in resource acquisition/release.
Local callback candidates do not establish a complete target set when a
conditional write may leave a parameter, catch binding, or uninitialized
variable unchanged. These calls retain candidate edges and dynamic uncertainty;
default traces remain ambiguous with partial coverage. Opt into
`include_ambiguous_dynamic_edges` to traverse the candidates. Proven preceding
unconditional assignments, including writes dominating a read in the same
conditional block, can resolve a target; later writes do not replace an
earlier read or captured alias. Unproved ordering across branches or function
activations remains conservative. Returned-reference effects use the same
read-position facts as call resolution.
Mutation tracking covers explicit member assignments, updates, deletion, and
loop assignment targets, including the supported local aliases and shared
nested references. Calls conservatively invalidate exposed mutable references;
@@ -185,10 +166,8 @@ graph and read literal values from `properties.value`, rather than decoding
role keys or labels. Previously stored graphs and their original IDs remain
valid inputs.
Endpoint observations keep their value in `properties.value`. When
`properties.resolution` is `partial`, that value is the unresolved source
expression, with an identity distinct from an exact endpoint of the same text.
Semantic request and response nodes keep their endpoint in `properties.endpoint`. Endpoint
Endpoint nodes keep the exact endpoint in each observation's `properties.value`;
semantic request and response nodes keep it in `properties.endpoint`. Endpoint
lookup identities are preserved independently of the primitive-value lattice's
byte budget. An oversized normalized literal can become unknown while the same
string remains an exact endpoint for tracing and comparison. Endpoint
@@ -272,13 +251,6 @@ Direct return expressions, including expression-bodied arrows, are evaluated
through an execution-free value lattice. Literal object fields and direct
return sites are represented in the result. Calls, dynamic spreads, computed
keys, and parser recovery remain partial or unknown.
Rebound values are evaluated at their read position: saving an incoming parameter
before a later assignment preserves caller-dependent uncertainty, while a proven
preceding assignment can supply a constant. Unresolved ordering remains unknown.
Supported reads within object and array literals follow left-to-right evaluation;
an unproved relative order remains unknown. A later write in a separate conditional
block can still leave an earlier capture unknown because ordering across those
blocks is not proven.
Objects and arrays passed to calls, constructors, or tagged templates, used
as method receivers, or stored through property targets are not assumed
unchanged afterward. Property stores conservatively retain reference escape
+11 -38
View File
@@ -182,18 +182,6 @@ partial Evidence. Cancellation after source analysis begins can also retain
completed facts, including when interrupted during cleanup, graph construction,
or Evidence sealing. Unexamined relationships remain unknown.
Per-file gaps identify the applicable processing phase: JavaScript source analysis
uses AST authority, while package/JSON, HTML-reference and source-map processing
use artifact-byte authority. File-analysis interruption gaps carry `analysis_status`:
`interrupted` means the phase was entered (including source-worker startup),
and `not-started` means interruption prevented that phase from beginning.
Neither state establishes that parsing completed. Remaining parsers are not run
after interruption just to populate a result.
Native add-ons remain metadata-only and acquire no JavaScript parse or analysis
gap. This also applies when their paths are outside `source_paths`; their inventory
and caller selection remain available without implying applicable AST work.
MCP returns a `partial_observation` retained reference after recording succeeds;
open it with `inspect_analysis_view` on the same connection. If the SDK stopped
waiting before receiving the reply, follow [MCP cancellation recovery](mcp-contracts.md#progress-and-cancellation)
@@ -311,25 +299,13 @@ suffix. Utility calls such as `useUtils().invoice.list.fetch()`, `useQueries`,
and links whose callee was renamed by a minifier are not recognized.
Each recovered bundle module retains the exact factory-source digest. A complete
bounded AST also receives a `babel-ast-v3` structural fingerprint. It ignores
ordinary identifier spellings while retaining literal and directive values,
operators, fixed property names, and selected syntax flags such as computed
keys, declaration kind, function async/generator state, update-expression
prefix, method kind, and class static state. Local names used in computed keys
remain rename-resistant; private member names are also ignored while their
method kind and static state remain represented.
This fingerprint is structural evidence for candidate matching, not proof that
two modules behave equivalently. Its scope follows the AST facts listed above;
unrepresented syntax can still collide. The exact source digest remains
available to show byte-level change, and comparison reports content changes even
when structural matching succeeds.
Version comparison includes the algorithm in its structural matching key.
Retained `babel-ast-v1` results can still match other v1 results, and v2 results
can still match v2 results, but v1/v2 results cannot structurally match v3
results. Other matching tiers remain available across algorithm versions.
Re-analyze both snapshots to compare using the v3 rules.
bounded AST also receives a `babel-ast-v2` structural fingerprint that ignores
ordinary identifier names while retaining syntax, literals, operators, object
keys, and member properties. Computed member identifiers retain their access
syntax without treating a local variable's spelling as a fixed property; template
literal contents remain part of the fingerprint. Version comparison keeps the
algorithm in its matching key, so retained `babel-ast-v1` fingerprints cannot match
these new fingerprints. Re-analyze both snapshots to compare their new structure.
If fingerprint construction reaches its structural
bound, the fingerprint is unavailable and its status is `truncated`; a bounded
prefix is never presented as a complete stable fingerprint.
@@ -426,13 +402,10 @@ For `.node` bindings, member names mean “requested by JavaScript syntax.” A
resolved add-on path does not convert them into verified native exports. Native
symbol verification remains a separate deep-analysis claim.
Exact endpoint values preserve useful local diagnostics, including query values
and fragments, while removing URL username/password credentials when the value
parses as a URL. When a recognized request target cannot be resolved exactly,
the endpoint's partial resolution retains the original source expression with
its call-site provenance. This does not establish that the request executed.
Artifact paths, digests, parse locations, and analysis metadata remain
actionable because REA is local-only.
Endpoint observations preserve useful local diagnostics, including query values
and fragments, while removing only URL username/password credentials. Artifact
paths, digests, parse locations, and analysis metadata remain actionable
because REA is local-only.
## Safety boundary
-1
View File
@@ -48,7 +48,6 @@ The first text or assembly request in a session looks for this target's export i
- REA starts `kuna decompile-project` in the background on a private copy of the target.
- Meanwhile, pseudocode comes from one `kuna decompile --addr` run. Name and address lookups, and listings, use the inventory the export takes with `kuna functions --json` before it decompiles. The session runs `kuna functions` itself, under `REA_KUNA_TIMEOUT_MS`, only when no export is building (an inventory-only request does not start one) or the export ends without an inventory.
- Requests for the same function share one run; it stops when its last waiting request is cancelled. Each run analyzes the whole image again in its own process, so one REA process runs at most as many at once as the host has cores beyond `REA_KUNA_JOBS` (at least one). Further requests wait in order, and a waiting request is answered from the export if it finishes first. A run whose stop REA cannot prove keeps its slot until `close_binary` proves it stopped.
- The session keeps each completed single-function text, up to 16 MiB in all with the least recently used dropped first, and answers repeats from it until the export can. After a failed export that lasts for the session.
- Assembly and data names wait for the export; progress notifications report the wait every ten seconds. A session does not rebuild a failed export: requests that need it fail with `retryable: false`, the export's failure, its captured Kuna output and what to change before reopening the target. A setting such as `REA_KUNA_PROJECT_TIMEOUT_MS` or `REA_KUNA_JOBS` is read when REA starts, so changing it also needs a restart. Pseudocode keeps using single-function runs.
- A single-function answer says so in its limitations. It lacks the export's whole-program prototypes, so a later request can return different text for the same function.
+3 -32
View File
@@ -393,26 +393,15 @@ concrete version, selection source, and complete analysis profile through
choices return typed selection details. A selected provider is never replaced
automatically after a runtime failure.
Each provider client receives `binary_session.analysis_run.run_id` before any
provider startup, including clients whose startup fails. `process_lineage` is
`not_observed` until a dynamic provider starts, then becomes `snapshots` with
every started provider's identity
Every successful target transition allocates `binary_session.analysis_run.run_id`
before any provider startup. `process_lineage` is `not_observed` until a dynamic
provider starts, then becomes `snapshots` with every started provider's identity
and retained ownership observation. Each observation records `observed_at` and
is `unavailable` with a reason when ownership could not be revalidated, or
`verified` with launcher PID, parent PID, process group, and descendants observed
at that bounded check. These are historical snapshots, not live process
inventories, and do not claim that no short-lived descendant existed.
`binary_session.cleanup_pending` reports whether the session retains a provider
client, session file resource, or provider-candidate resource whose cleanup has
not succeeded. It can be true with `open: false`; target retirement does not
establish resource release. Health, activity, and lineage remain available
under a retained client's original `analysis_run.run_id` where the provider
supplies them. Successful client cleanup clears the retained run; a replacement
client receives a fresh run ID. Follow the cleanup error's reported resources
and recovery action before reopening; a missing health observation does not
establish an idle provider.
## Progress and cancellation
REA accepts ordinary `tools/call` progress tokens. The numeric `progress` counts
@@ -439,10 +428,6 @@ Native call tracing and process capture retain available observations in
`details.partial_observation` on failure, including when cleanup succeeds.
The observation reports its partial coverage; cleanup details describe host
state separately from the execution failure.
For these tools, cancellation observed during result finalization or owned
cleanup also returns typed cancellation with the available observations.
A normal target exit does not establish request completion. Incomplete cleanup
still takes precedence and retains its resource diagnostics.
A process capture that fails during a finalization interval keeps every signal
attempt and its delivery result in `details.partial_observation`, with
`elapsed_ms` and `exit_code` null when no exit was observed.
@@ -709,20 +694,6 @@ after SDK conversion and check advertised validation and actual calls.
Individual model APIs can impose additional nesting limits; complete producer
captures can exceed ten structural levels.
Boolean literal constraints in input schemas are relaxed only in this
advertisement. For example, a boolean `const` is presented as `type: boolean`;
an affected `not` is omitted, conditional predicates and their branches are
omitted together, pure boolean-literal `oneOf` branches collapse to a boolean,
and other affected `oneOf` branches are presented as `anyOf` to retain branch
guidance without making overlapping widened branches reject inputs.
This can make the advertised schema accept inputs the canonical contract
rejects. The MCP handler still validates every call with the canonical
contract. The Gemini CLI verifier checks the outgoing
`parametersJsonSchema` and canonical contract examples against that declaration
using a loopback model fixture; it does not establish live Google API or
OpenCode client behavior. Set `REA_VERIFY_GEMINI_SCHEMA_PROFILE=compact` to
verify the compact advertisement through the same client boundary.
### Compact input schema profile
Some function-calling providers reject a request while any single tool's
-22
View File
@@ -451,11 +451,6 @@ The run ends when the process exits, `max_events` entries are recorded,
Bounded termination reports partial coverage. REA checks process termination
using a launch-time OS start identity; if ownership or cleanup cannot be
verified, the result or error preserves that uncertainty. The result reports:
Launch errors keep the fact that LLDB created a process separate from whether
it remains alive. An exited `SBProcess` is reported as exited only after LLDB
confirms that state; unresolved ownership remains retained for cleanup retry.
Cancellation observed while output and Evidence are finalized returns partial
cancellation Evidence, even if the target has already exited.
- the outcome, exit status and captured stdout/stderr (1 MiB kept per stream
while excess bytes are drained and counted without growing capture files).
@@ -486,11 +481,6 @@ with partial coverage. Available output prefixes remain inline; their byte
counts are observed lower bounds when final drain counters are unavailable.
The original failure classification and cleanup uncertainty remain separate.
An interrupted journal row is ignored with an explicit limitation.
If the trace completed before cancellation or runtime-directory cleanup failed,
the retained observation also keeps its exit status, termination proof, launch
identity, breakpoint results and final stream counters. Those fields remain
absent when the interrupted producer did not report them; partial coverage alone
does not mean that every retained fact is incomplete.
Only entries are observed: return values, floating-point and stack arguments,
and inlined or `objc_direct` calls are not. The target runs with the current
@@ -587,18 +577,6 @@ Exact and ambiguous value counts describe emitted root/object value instances;
a root and an object may contain the same value. Archive digests identify the
original bytes.
Interface Builder's XML and binary plist archives use the same number metadata
and exact-decimal representation for emitted node attributes. Projection follows
object and connection classification, so archive IDs, UID links and graph counts
keep their meaning. Precision limitations name the source document. Unsafe
integer-literal counts cover the archive, including fields not emitted as node
attributes; exact and ambiguous projection counts cover emitted attribute values.
Integral reals stay numbers, and ambiguous or incompletely observed values retain
explicit precision limitations. Before plist decoding, the existing aggregate
decode budget reserves estimated room for number metadata and attribute
projection; these estimates are not a hard process-memory bound. Native
NIBArchive decoding is unchanged by this plist-specific representation.
Dates are reported as `{ "$plist_type": "date", "iso8601": "<timestamp>" }`. A
date the decoder cannot represent, such as an XML date with a year beyond 9999
or a negative year, is reported as `{ "$plist_type": "date", "iso8601": null }`
+1 -3
View File
@@ -425,9 +425,7 @@ results for unaffected dimensions and mark affected dimensions unknown; an
incomplete capture cannot locate the first divergence across all dimensions.
Trace assertions use coverage for the sources they select, so an assertion
about complete raw terminal output need not fail because rendered snapshots
were omitted. A trace difference applies to the dimensions identified by its
diagnostic; unrelated dimensions retain their own comparison, including unknown
coverage. Captures missing `truncation_details` are rejected on import;
were omitted. Captures missing `truncation_details` are rejected on import;
preserve them as historical files and recapture for current comparisons.
`limits.file_bytes` is already the cumulative whole-file hashing budget,
independent of `limits.output_bytes`, which bounds retained terminal content.
-14
View File
@@ -105,20 +105,6 @@ overall native heap and total resident memory are not capped. Row limits do not
override these byte and resource limits. Resource-limit failures do not return
an incomplete schema as successful complete evidence.
If the combined database and WAL exceed the snapshot budget, the refusal is a
`resource_constraint` and reports both the configured byte limit and the exact
observed combined size as a decimal string. The remediation directs callers to
create a smaller consistent offline snapshot containing only relevant data
through the application's supported export workflow. A generic backup or copy
does not guarantee a smaller result; REA does not truncate or modify the
selected source.
On hosts that cannot address the derived WAL or rollback-journal path for a
long selected database name, REA reports that the SQLite capability is
unavailable. It does not assume an inaccessible sidecar is absent. Use the
application's supported consistent offline export or copy workflow to place
the complete known file set at a shorter path.
MCP also applies its configured response budget to the complete Evidence
envelope. If delivery exceeds that budget, it reports a transport constraint
with guidance for exporting retained Evidence, requesting a matching larger
+1 -51
View File
@@ -199,44 +199,6 @@ Measure slow cases before pruning capacity coverage. Retain inputs beyond the
formerly failing size/depth and distinguish cold runs, warm builds, coverage,
and toolchain conditions when comparing timings.
## Lifecycle regressions
For changes to resource acquisition, readiness, cancellation, or cleanup, select
the affected owners and applicable scenarios below. Include temporary snapshots
and one-shot commands. Use public CLI/MCP workflows where practical and a
controlled production seam for interleavings that cannot be reproduced reliably.
| Scenario | Assertion and control |
| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Target admitted; startup still pending | Status preserves the readiness distinction and observed startup activity; readiness requires the provider's documented proof. |
| First handshake is late, absent, invalid, or recovered during close | Initial and recovery paths retain the same authenticated identity facts. Foreign run/token controls cannot authorize cleanup; authenticated idle/native work survives the unauthenticated listener deadline. |
| Launch returns an error with a valid process handle | Cover already-exited and still-live owned processes. Verify observed exit separately from creation; failed termination or unavailable identity remains unresolved. |
| Cancellation arrives after workload exit, during result reads or temporary-root removal | Return typed cancellation with available partial observations and actual cleanup status. The non-aborted control succeeds; existing primary/cleanup failures keep their precedence. |
| Children have mixed health or cleanup outcomes | Missing peer health does not erase an observed failure or activity. Idle plus unknown stays unknown. Retry a failed child without closing a successful peer again, including after a close-option change. |
| Cleanup fails after success or primary failure | Public output retains the completed observation or primary diagnostic, remaining resources, and owner recovery action. Restore the denied resource and verify the owning cleanup retry. |
| Target exits without an acknowledgement, or transport disappears while it stays alive | Positive owned exit proof resolves target lifetime; other resources still need cleanup. Lost transport alone does not prove exit. Verify unrelated/attached targets survive and same-connection recovery works when supported. |
Synchronize cancellation with the finalization boundary being tested; a fixed
delay after dispatch does not establish that phase. Use markers, barriers, or
owned cleanup seams while preserving real cleanup where feasible. Check server
settlement and retained resources even if the MCP SDK rejects the client promise
before the handler finishes. A cancellation during active execution does not
cover cancellation during final cleanup.
For atomic writes, exercise cancellation on both sides of the documented commit
point and preserve any publication receipt. A late interruption does not prove
that no output was published.
Representative regressions include
[process cleanup-time SIGINT](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/process/processCaptureCancellationAdapters.test.ts),
[late authenticated cleanup](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/providers/hopper/hopperClient.handshakeRecovery.test.ts),
[LLDB launch and finalization](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts),
[composed child ownership](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/src/application/binary/CompositeProvider.test.ts),
and [public CLI cleanup denial](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/cli/wasmCleanupCli.test.ts).
These controlled probes do not replace the matching real-provider lane. Record
the base failure, passing control, exact host/provider, verified cleanup, and
unavailable coverage; a missing engine is not a pass.
## Behavioral depths
| Depth | Location | What it proves |
@@ -1027,19 +989,7 @@ the Apple or Windows curated lanes invoke explicitly, which also select that
lane. Edits to verifier entrypoints run only by manual, licensed or self-hosted
workflows run static checks and the Linux suite. Website changes run website
checks. Provider workflow changes validate the workflow and run that provider's
verifier. Changes under `bridge/ghidra/` also select the `ghidra_bridge` lane,
which compiles the packaged bridge sources against the pinned Ghidra 12.1.4
jars with `npm run verify:ghidra:bridge-sources`. It never launches Ghidra or
imports a target, so a missing import fails the PR instead of every Ghidra
session. The launcher, compiler and package checks share named required source
roles. The manifest must match the complete Java cohort, so deleting a helper
or adding an unregistered dependency cannot silently change the shipped set.
The preflight reuses the provider's Java Properties parser with the pinned Node
toolchain's native type stripping, without an npm install or runtime build.
The shared release policy owns the verified version and JDK major. CI checks
the archive pin before downloading and verifies the extracted SDK version;
changes to these source owners also select the compile lane.
Manual-provider/Android workflow changes run workflow validation only.
verifier. Manual-provider/Android workflow changes run workflow validation only.
The four-version SQLite compatibility matrix has its own `sqlite` scope for
SQLite implementation, contracts, CLI/MCP registration, fixtures, and shared
runtime boundaries. SQLite and server-lifecycle test edits and the SQLite npm
-48
View File
@@ -68,54 +68,6 @@ satisfy. A plan without execution does not establish runtime behavior.
constraints. Report pagination, truncation, cancellation, and partial failure.
An assumed agent budget is insufficient justification for a limit.
## Own the lifecycle
For tools that acquire resources, identify the owner and the observations that
establish each transition. Include temporary files and one-shot commands, as
well as persistent engine sessions. Use the existing provider contracts;
providers need not expose identical state machines.
- Keep target admission, provider readiness, active work, observed process exit,
and resource release separate. A valid process handle, successful launcher,
or selected target establishes only its own fact. Publish available identity
and exit observations even when launch fails; retain unknown ownership.
- Authenticate and capture identity through the same boundary on initial and
recovery handshakes. Bound abandoned unauthenticated listeners where the
protocol permits it; that deadline must not expire authenticated idle sessions
or long-running native work. Listener expiry does not prove target closure.
- Shared startup has one producer and independently cancellable caller waits.
Keep it running while any caller still needs it. The final departing caller
waits for owned cleanup and retains its failure. Provider closure is distinct
from caller cancellation. Retained cleanup capabilities block replacement;
derive readiness from the current startup's completion and live connection.
- For cancellable observations, carry cancellation through result consumption
and awaited cleanup. Before returning success, check the signal after the
last awaited finalization step.
Retain collected observations under typed cancellation. Preserve an existing
primary failure and let incomplete cleanup retain its own classification.
Client cancellation alone does not prove native work stopped.
For atomic publication, document the cancellation cutoff and preserve any
committed output's publication receipt.
- Release only owned resources. Attaching owns a connection or session unless
the provider contract explicitly establishes target ownership. A helper exit,
missing reply, or missing PID does not establish target exit.
- Retain successful child cleanup and retry only unresolved children, including
when close options change. Preserve observed child failure or activity when
another child's health is unavailable; idle plus missing coverage stays
unknown.
Bind cleanup failures to their exact resource owner. Resolve current cleanup
uncertainty only after that owner confirms release, while preserving the
original failure and completed observations. Later failure composition must
report only resources whose cleanup remains unresolved.
- Await one-shot cleanup without discarding the completed result or primary
diagnostic. Preserve partial observations, remaining resources, and the cleanup
owner's recovery instructions. CLI adapters should reuse
[`finishAnalysisCommand`](https://github.com/morluto/rea/blob/main/src/cli/finishAnalysisCommand.ts) where its contract
applies rather than duplicate close/finally handling.
Exercise the applicable [lifecycle regression scenarios](testing.md#lifecycle-regressions)
through the producing owner and affected CLI/MCP consumers.
## Implement and evaluate
Put provider protocols/parsing in adapters and shared CLI/MCP workflows in the
File diff suppressed because it is too large. Load diff
@@ -1,245 +0,0 @@
# Interrupted application processing: applicability and Evidence
Tracks #3948 under #4027. Measured baseline:
`c7f6b8bad1a34d5efb30ff2a017672dea1fcdabb`.
The question is which processing phases remain unavailable after a source
interruption, while keeping native inventory and already collected facts.
Native identity and provider linkage must stay independent of application-wide
partial coverage. This is static artifact analysis; no target application or
native provider runs.
## Observed failure and correction
The interruption producer previously attached one generic failure to every
remaining relevant file. Graph projection interpreted it as JavaScript AST work.
Real Joplin reproduced an `analyze-javascript-source` / `ast-static-analysis`
unavailable scope on keytar's native binary and on unvisited package, HTML and
source-map files. Public CLI/MCP controls reproduced the same applicability
problem. Deterministic cancellation before native metadata additionally produced
a phantom JavaScript semantic module for the binary.
The producer now records the applicable operation/authority and whether the
phase was entered. `analysis_status: interrupted` includes worker startup and
does not imply parsing completed; `not-started` identifies unvisited phases.
Native add-ons need no post-inventory source-processing phase, so neither interrupted
nor caller-excluded native files acquire AST scopes or semantic modules.
Actual JavaScript failures, structured-file unknowns, caller selection and byte
identity remain available. The graph consumes these produced facts.
The AST-only unavailable Evidence helper is replaced by an explicitly typed
static helper. Normal package/JSON/map gaps also retain their artifact-byte
authority. Per-file progress uses the same applicability owner: JavaScript keeps
its source phase, and metadata uses `process_javascript_artifact_metadata`.
Remaining parsers do not run after interruption. Failure sharing is limited to
five applicable phases during one interruption; no persistent cache is added.
A second review found the alternate full-analysis semantic builder still
created modules for every excluded file kind. An actual filesystem-producer
regression on revision `323fae8cbf3e36677bcb7b22ba31fa1a7e3c5cd4` reproduced
phantom native/JSON/HTML/package/source-map modules, while selected and excluded
JavaScript were positive controls. Semantic applicability now belongs to the
shared `recordUnavailableFile` projection entry point, before retaining limits,
nodes, roots or unknowns. Three duplicate caller checks are removed. Both full
and incremental construction use this boundary; actual excluded JavaScript
continues to carry source-bound unknowns and partial semantic coverage.
## Real input and reproduction
Use the [official Joplin v3.7.21 acquisition and extraction instructions](https://github.com/morluto/rea/blob/c7f6b8bad1a34d5efb30ff2a017672dea1fcdabb/docs/verification/javascript-native-handoff-identity-2026-10-15.md).
The AppImage SHA-256 is
`5d202dcaca31e14206826aa3f04125e7c218dbc3dc3d98479d811efbd6b856c1`;
ASAR SHA-256 is
`767df409b18effe60421fbefcca873de237c7e813eac744ecdeebbc5e825a33a`.
The 6,324-file inventory includes 46 unpacked entries. Both cohorts stage all
three native companions; the other 43 unpacked entries remain unavailable.
Build with pinned Node 24.18.0/npm 11.16.0 and locked dependencies. The extended
existing verifier also runs against the baseline runtime; copy the verifier and
`scripts/lib/javascript-file-scope-oracle.mjs` from the fixed revision into a
built baseline checkout when reproducing the failure.
```bash
artifact_path="/absolute/path/to/cases/available/app.asar"
common=(
--artifact "$artifact_path"
--addon-path node_modules/keytar/build/Release/keytar.node
--native-sha256 0700016bcd761cbe02d275f2046c7cc42f39f730a50d66e3d3307aa9d413373a
--source-path ElectronAppWrapper.js
--source-path ElectronAppWrapper.js.map
--source-path index.html
--source-path package.json
--source-path node_modules/keytar/build/Release/keytar.node
--structured-path ElectronAppWrapper.js.map
--structured-path index.html
--structured-path package.json
--structured-expectations '{"ElectronAppWrapper.js.map":{"operation":"parse-local-source-map","analysis_status":"not-started"},"index.html":{"operation":"parse-html-script-references","analysis_status":"not-started"},"package.json":{"operation":"parse-package-json","analysis_status":"not-started"}}'
--rounds 3 --cli-parity
--tokenizer-python /absolute/path/to/python-with-tiktoken-0.12.0
)
node scripts/verify/javascript/native-handoffs.mjs "${common[@]}" \
--analysis-timeout-ms 1 --expect-timeout --require-safe
node scripts/verify/javascript/native-handoffs.mjs "${common[@]}" \
--validate-structured-from-export --require-safe
```
Omit `--require-safe` on the baseline timeout run to record its incorrect
applicability. Its diagnostic exit 0 is not a passing analyst task. A supported
1 ms deadline includes worker startup; the verifier requires a typed timeout
and retained partial reference for that cohort. Each structured path supplies an
independent expected operation and phase-entry status; the safety oracle matches
all gaps exactly, including unavailable state and unknown confidence. Missing or
conflicting expectations do not pass.
The native path is explicitly selected. An earlier structured-only run excluded
it and therefore did not reproduce the native interruption. Input selections
and that preliminary result remain recorded. Both final cohorts preserve
partial coverage for unselected/unavailable scopes.
The supplied native Evidence is simulated solely to check exact subject linkage.
The verifier checks physical companion bytes and the exported parent's root
digest, validates public CLI/MCP native-trace parity, and reports only after
owned transport/command/private-file cleanup. Applications and large Evidence
are regenerable and are not archived.
## Two distinct task projections
The **timeout task** includes the expected typed-timeout preparation, one inline
native trace and three inline structured-path traces. It checks that native has
no AST failure and that each unvisited structured phase remains unavailable with
the appropriate artifact-byte authority and `not-started` status.
The **default-deadline control** models the summary plus one inline native trace.
Its named structured paths are additionally validated from the already exported
complete parent, explicitly labelled `exported-parent-validation`. Those checks
are diagnostic overhead outside the modeled native-task tokens and inside total
verifier resources. They are not presented as inline metadata answers.
A control that also requested the complete outgoing `package.json` feature trace
hit stdio admission twice: at least 10,587,373 bytes against a 10,484,736-byte
result budget, with identical retained parent/trace identities. It is not a
completed equivalent task or an efficiency baseline. [#4190](https://github.com/morluto/rea/issues/4190)
records the separate need for exact retained metadata/native fact views. Complete
export remains available. Do not compare costs across these different tasks.
## Measurements
| Cohort | Revision | Wall s | CPU s | Peak cgroup bytes | Applicability |
| ------------------------------ | -------- | ------: | ---------: | ----------------: | ------------- |
| Timeout, four inline traces | baseline | 60.738 | 60.289464 | 3,092,045,824 | wrong scope |
| Timeout, four inline traces | final | 57.515 | 57.089883 | 2,763,276,288 | pass |
| Default control, native inline | baseline | 61.542 | 61.135522 | 2,808,336,384 | pass |
| Default control, native inline | final | 186.085 | 142.371100 | 2,959,585,280 | pass |
| Defined task | Selected catalog + parsed/structured replies BPE | Selected catalog + complete tool replies BPE |
| -------------- | -----------------------------------------------: | -------------------------------------------: |
| Timeout | 21,136 → 20,161 | 36,468 → 34,471 |
| Native control | 11,776 → 11,720 | 17,738 → 17,632 |
The full-catalog alternatives and per-query samples are preserved in JSON.
Native content identity, observed byte provenance, source selection and handoff
provider Evidence remain equal. Corrected global provenance changes derived
parent/trace identities; null partial-Evidence subjects remain null. Both final
runs have exact CLI/MCP native-trace parity and match the earlier post-fix
controls in every recorded non-timing field.
The whole verifier includes input hashing, analysis, three native trace rounds,
applicable structured queries, export, CLI parity and tokenization. Every heavy
run uses the preserved [resource observer](https://github.com/morluto/rea/blob/b3bdabab694af1f9801eff7caf75001b92314a58/experiments/analysis-efficiency/resource-run.py):
one CPU, 8 GiB hard cap, 6 GiB early stop and no swap. Cgroup peaks include charged
file cache, not summed RSS. Counters are observed at payload exit while the
observer remains alive, not final whole-scope counters. Completed runs verify
owned cleanup and report no OOM.
Each MCP process is fresh, but input hashing warms its file cache. Build/OS caches
and shared-host scheduling are uncontrolled. First and repeated retained-query
wall samples remain separate; isolated query CPU/memory are unmeasured. An
earlier fixed timeout took 150.173 s wall / 119.468013 s CPU; its exact unchanged
runtime repeat took 58.827 s / 58.482996 s, with identical non-timing facts,
Evidence and BPE. Both receipts are retained. The final default control also took 186.085 s wall / 142.371100 s CPU; its
unchanged-runtime repeat took 62.651 s / 62.240964 s with identical non-timing
results. The slow final row remains reported. Attribution remains unresolved;
these runs establish no speedup or performance-equivalence bound.
The stricter post-review verifier reruns also passed without changing production
code: timeout 162.645 s wall / 125.876114 s CPU / 2,809,643,008 B peak; default
control 61.367 s / 60.970077 s / 2,526,949,376 B. Both preserve the original final
analytical fields, identities, coverage and BPE exactly; only timings and the new
explicit oracle-input field differ. These additional receipts do not replace the
slower rows above.
After centralizing semantic applicability, both real public controls passed again:
timeout 60.443 s wall / 59.970344 s CPU / 3,078,295,552 B peak; default control
63.574 s / 63.177884 s / 3,189,719,040 B. Analytical results, identities, coverage,
BPE and exact CLI/MCP parity remain unchanged. JSON identifies the runtime and
verifier source manifests for each cohort and keeps all earlier receipts.
Static `cl100k_base`/tiktoken 0.12.0 projections include the selected or complete
name/description/inputSchema catalog, requests and structured Evidence or parsed
text errors. Complete-tool-reply alternatives include MCP text/structured
duplication. The selected two-tool catalog is 5,570 BPE; the complete projection
is 195,522. Output schemas, host rendering/replay, notifications, model reasoning,
final answers and billing are unmeasured. Repeated rounds, export, CLI parity,
tokenization and exported-parent diagnostics are validation overhead outside the
defined task tokens but inside resource receipts. Removing unsupported claims
from the incorrect timeout baseline earns no efficiency credit.
## Verification and limitations
The initial production fix had 49 distinct passing cases across eight files. The initial
pre-progress-cleanup seven-file selection passed 44/44. After progress cleanup,
the same selection passed 43 and hit the existing 30-second test deadline in one
native identity case; an unchanged two-file rerun passed both native cases and
five public CLI cancellation cases. The original deadline was not increased.
The runtime verifier collects interruption-bearing scopes; separate regressions
cover native caller exclusion and malformed structured data. Source selection,
malformed structures, actual worker resource constraints,
retained large partial observations, cancellation/publication/cleanup, native
identity and both public adapters remain covered.
The first review found that the real verifier previously accepted any non-JavaScript
operation with artifact-byte authority and ignored phase-entry status. The
scenario now supplies exact operation/status expectations independently of the
production phase map. Six additional oracle cases use the actual retained
Joplin snapshots and reject wrong metadata operations, entered or missing
statuses, and absent or conflicting gaps. They passed, bringing the distinct
coverage to 55 cases across nine files. The production implementation did not
change during this review correction.
After the shared semantic-boundary correction from the second review, 78
distinct cases passed across ten files: the nine-file selection passed 65/65,
and the original root reconstruction file passed 13/13. This includes the
alternate-builder owning test file's 23 cases. The new producer-boundary
regression checks both construction paths, all five excluded metadata/native
kinds, and selected/excluded JavaScript controls. Real Joplin controls exercise
the public incremental workflow; the alternate full-builder path has filesystem
boundary verification rather than a separately measured real-app workflow.
Recorded attempts also include the verifier's initial non-null-subject assumption
and an invalid top-level semantic-module-path absence check. Both were corrected
before their assertions were used as proof. Corrected cancellation reproduction
shows the actual native AST scope and phantom module on the baseline. Final
source and execution-driver versions, source pins, input digests, prior attempts
and resource receipts are in the [compact JSON record](javascript-interruption-applicability-2026-10-15.json).
Before review, `npm run check && npm run docs:check` passed, including five consistency
tests: 165.553 s wall / 150.247257 s CPU /
4,232,491,008 B peak cgroup memory, clean owned exit and no OOM.
The first full check stopped on a 172-line test callback exceeding the 150-line
rule. A named assertion helper now shares scope lookup across all four public
results; the strengthened six-case file passed again before the final gates.
After the review correction, the full static/docs gate passed again, including
the five consistency tests: 111.395 s wall / 110.979710 s CPU /
4,347,125,760 B peak, with clean owned exit and no OOM.
After centralizing semantic applicability, the final full static/docs gate also
passed, including five consistency tests: 117.636 s wall / 117.194118 s CPU /
4,263,153,664 B peak, clean owned exit and no OOM.
At the initial PR revision, 36 CI checks passed; macOS `hdiutil create` failed
while creating its DMG fixture before REA verification, also failing the aggregate
check. No Apple fixture/workflow code changed. That attempt provides no passing
Apple runtime evidence.
Live Electron/application execution, real native-provider behavior, other-platform
runtime coverage and hosted-agent costs are **NOT_ENOUGH_INFO**. The verification
does not claim them.
File diff suppressed because it is too large. Load diff
@@ -1,179 +0,0 @@
# Native handoff byte-identity verification
Tracks #3926 under #4027. Baseline: `9f8659e0536e4e50a438f5b9d5661f3c9624f6ac`.
Final integration base: `62bf09809ef7bb224028916424f8f07652a20f4f`.
The analyst already knows an ASAR-relative native path and supplies prior native
Evidence. The question is which Evidence can be linked to that component's exact
bytes. Discovery and native-provider execution are outside this task.
## Observed behavior and correction
The native-node producer distinguishes a content digest from a canonical path
whose bytes are unavailable. Its unavailable observation legitimately cites the
containing archive. The handoff consumer previously treated that supporting
artifact digest as the native component's digest.
Public CLI/stdio MCP regression: the missing-companion case failed and the
available-byte control passed. Real Joplin reproduced the same result: missing
keytar bytes produced an `evidence-linked` handoff with the ASAR digest and the
simulated container observation. CLI/MCP agreed on that incorrect answer.
The consumer now uses only the node owner's content-digest identity. The fallback
observation scan is removed. Unavailable native nodes retain their canonical
path, availability, container-backed provenance and partial coverage in the
inline graph; they yield no false actionable handoff or native Evidence link.
An empty handoff list describes eligibility, not absence of native components.
The per-query node index is allocated only after an eligible target exists; no
persistent cache, schema, tool or target authority is added.
Final real checks passed both cases with exact CLI/MCP trace parity. The available
control preserved the complete parent/trace Evidence IDs, native facts, links,
coverage and all token projections. The missing control preserved its parent,
native node, source provenance and coverage, while removing the wrong handoff
and container Evidence link.
## Real input and scope
[Official Joplin v3.7.21 Linux AppImage](https://github.com/laurent22/joplin/releases/download/v3.7.21/Joplin-3.7.21.AppImage),
published 2026-09-25, 201,462,481 bytes, SHA-256
`5d202dcaca31e14206826aa3f04125e7c218dbc3dc3d98479d811efbd6b856c1`.
Read-only 7z 26.02 inspection found 6,324 ASAR files, 46 unpacked entries and
three `.node` components. Extract only the ASAR and those three binaries:
```sh
input_root="$(mktemp -d -t rea-native-handoff-XXXXXX)"
curl --fail --location --output "$input_root/Joplin-3.7.21.AppImage" \
https://github.com/laurent22/joplin/releases/download/v3.7.21/Joplin-3.7.21.AppImage
sha256sum "$input_root/Joplin-3.7.21.AppImage"
7z x -mmt=1 "-o$input_root/extracted" "$input_root/Joplin-3.7.21.AppImage" \
resources/app.asar \
resources/app.asar.unpacked/node_modules/keytar/build/Release/keytar.node \
resources/app.asar.unpacked/node_modules/sqlite3/lib/binding/napi-v6-linux-glibc-x64/node_sqlite3.node \
resources/app.asar.unpacked/node_modules/onnxruntime-node/bin/napi-v6/linux/x64/onnxruntime_binding.node
python3 - "$input_root" <<'PY'
import os, sys
from pathlib import Path
root = Path(sys.argv[1])
source = root / 'extracted/resources'
for mode in ('available', 'missing'):
case = root / 'cases' / mode
case.mkdir(parents=True)
os.link(source / 'app.asar', case / 'app.asar')
if mode == 'available':
for native in (source / 'app.asar.unpacked').rglob('*.node'):
target = case / native.relative_to(source)
target.parent.mkdir(parents=True, exist_ok=True)
os.link(native, target)
PY
```
Verify the asset digest above before extraction. The ASAR is 233,575,014 bytes,
SHA-256 `767df409b18effe60421fbefcca873de237c7e813eac744ecdeebbc5e825a33a`.
Keytar is 76,960 bytes, SHA-256
`0700016bcd761cbe02d275f2046c7cc42f39f730a50d66e3d3307aa9d413373a`.
All extracted component hashes are in the JSON record. 7z reported one
trailing-data warning for the AppImage's embedded SquashFS and exited 0.
Both cases select only `package.json` for static source analysis while retaining
the inventory. The available case stages all three native companions; the other
43 unpacked entries are not staged. The missing case stages none. Whole-application
coverage is therefore partial in both cases, independently of the selected
native component's known or unknown byte identity. Neither the application nor
a native analysis provider is executed. Native Evidence is explicitly simulated
to test exact subject linkage. Live Electron, real Ghidra/Hopper/IDA behavior and
Windows/macOS verification are **NOT_ENOUGH_INFO**.
## Reproduce the public workflow
Use pinned Node 24.18.0/npm 11.16.0, `npm ci`, then `npm run build:cached`.
The verifier returns compact records and deletes its private exported parent and
CLI input after owned cleanup. No downloadable application or large Evidence is
archived.
```sh
for mode in available missing; do
node scripts/verify/javascript/native-handoffs.mjs \
--artifact "$input_root/cases/$mode/app.asar" \
--addon-path node_modules/keytar/build/Release/keytar.node \
--native-sha256 0700016bcd761cbe02d275f2046c7cc42f39f730a50d66e3d3307aa9d413373a \
--source-path package.json --rounds 3 --cli-parity --require-safe
done
```
Add `--tokenizer-python /absolute/path/to/python-with-tiktoken-0.12.0` for the
reported BPE projections. The npm alias `verify:javascript:native-handoffs`
builds first; recorded costs use the direct Node invocation above after building.
For a pre-fix observation, run the saved verifier on the baseline build without
`--require-safe`; `safe_identity: false` is an incorrect answer, not a passing
analyst task. Paths affect serialized requests and some Evidence identities.
The exact resource runs additionally used the preserved
[8 GiB observer](https://github.com/morluto/rea/blob/b3bdabab694af1f9801eff7caf75001b92314a58/experiments/analysis-efficiency/resource-run.py)
and adjacent `resource_payload.py`; both hashes are pinned in the JSON.
## Completed workflow costs and limits
| Case | Revision | Wall s | CPU s | Peak cgroup bytes | Identity oracle |
| ---------------------- | -------- | ------: | ---------: | ----------------: | -------------------------- |
| Native bytes available | Before | 60.369 | 59.806670 | 2,662,629,376 | pass |
| Native bytes available | After | 258.885 | 106.221003 | 2,678,456,320 | pass, exact baseline facts |
| Native bytes missing | Before | 57.930 | 57.479109 | 2,719,457,280 | wrong identity/linkage |
| Native bytes missing | After | 59.738 | 59.254776 | 2,571,235,328 | pass, unknown retained |
Whole-verifier resources include input hashing, analysis, three retained trace
queries, export, CLI parity and tokenization. One CPU, 8 GiB hard cap, 6 GiB early
stop, no swap; cleanup verified and no OOM. Memory includes charged file cache,
not summed RSS. Counters are at payload exit while the observer remains alive.
The available control's final wall and CPU times are substantially higher than
baseline. Earlier post-fix controls on the original base took 61.657/74.956 s wall
and 61.135962/60.352428 s CPU for available/missing cases; their complete receipts
remain separately recorded. The final missing case took 59.738 s wall. This
variability is unresolved and cannot be attributed to the native consumer change.
These runs establish neither speedup nor performance equivalence. Input hashing warms the ASAR
cache, build/OS caches and shared-host scheduling are uncontrolled. Fresh MCP
sessions do not establish OS-cold conditions. First post-analysis and repeated
warm-query samples are preserved separately; query CPU/memory are unmeasured.
One defined task is one analysis summary plus one retained trace, with the path
and prior Evidence already supplied. Static `cl100k_base`/tiktoken 0.12.0 counts
include both requests, replies and name/description/inputSchema contracts:
| Available-byte task, unchanged before/after | BPE tokens |
| ----------------------------------------------- | ---------: |
| Selected catalog + structured replies | 12,340 |
| Selected catalog + complete tool replies | 18,906 |
| Full catalog projection + structured replies | 202,292 |
| Full catalog projection + complete tool replies | 208,858 |
These are alternative reply/catalog projections. The selected catalog alone is
5,570 BPE; the full projection is 195,522. Output schemas, annotations, host
rendering/replay, reasoning, final answers and model billing are unmeasured.
Repeated queries/export/CLI/tokenization are validation overhead outside this
one-task token count, but inside resource receipts. Missing-case structured-task
counts change 11,372 → 11,103 by removing unsupported claims; the incorrect
baseline earns no efficiency credit.
## Regression and verification record
The initial four-file focused selection passed 24 tests. The two public cases
were rerun after adding named summary-result parsing and correcting an optional
architecture factory input. Omitting that field instead of passing null produced
byte-identical normalized simulated Evidence in all four real-case inputs.
The initial CLI-selector harness error and first typecheck failure are retained
as harness/validation attempts, not product reproductions.
After syncing the reviewed upstream changes, the updated application workflow
CLI acceptance file passed 34 tests, bringing the focused selection to 58 distinct
cases across five files. Both Joplin controls were then repeated on the final
integration base. Canonical byte identity is also used by the related version
matching/content comparison consumers; no equivalent observation-digest fallback
was found there.
Final `npm run check && npm run docs:check` passed on the integration base,
including five documentation consistency tests: 128.732 s wall, 115.518920 s CPU,
4,301,684,736 B peak cgroup memory, clean owned exit and no OOM.
Final public and static/docs outcomes are recorded in the
[compact measurements and source pins](javascript-native-handoff-identity-2026-10-15.json).
+154
View File
@@ -0,0 +1,154 @@
# Atomic RE tools: an experiment
Experimental branch only: no PR, merge, production registration, or Code Mode.
Started from REA main `5b16f2431062eea6ffd245a9e05512790007cb13`.
Read [measured results](results.md) and [mechanism / prior-art notes](research.md).
The first round includes an object-reference counterexample and a failed
code-mapping composition; it does not claim every hypothesis succeeded.
The [control-flow / SSA follow-up](relations.md) repairs three zlib mapping
paths, tests a SQLite call guard, and records where structured slices cost more.
The question is whether composable facts improve **completed reverse-engineering
tasks**, especially on complex software. Three outcomes matter together:
1. Time, native work, CPU and process-family memory, including startup and retries.
2. Whole-task estimated tokens: catalog, arguments, results and necessary follow-up.
3. Correct answers with traceable evidence, meaningful coverage and explicit unknowns.
Smaller replies, fewer tool names and fewer calls are intermediate measurements,
not proof of a better investigation. Negative results are useful experiment results.
## Working definition
An atom is a coherent fact or relationship about an identified object, with its
producing evidence and uncertainty. It need not be a single field, MCP call,
instruction, native command or engine pass. A string's encoding and the status of
that encoding belong together; separating them can change the meaning.
Candidate reusable building blocks:
| Object / fact | Relationships and questions |
| ------------------------------------ | --------------------------------------------------------------------------------------- |
| Artifact and analysis context | Which bytes, architecture, address space and analysis profile produced this? |
| Location and byte span | What is mapped here? Where are the source bytes? Is the span initialized? |
| Instruction | What bytes decoded here, and with which mode and provenance? |
| Function | Which entry is identified? Which locations does the engine actually assign to its body? |
| String, symbol, type | What was observed or recovered, and what remains unknown? |
| Reference / call / control-flow edge | What are both endpoints, the reference site, kind and coverage? |
| Decompiled view | Which engine reconstruction and machine locations support this selected text? |
An inventory extent is an upper bound, not proof that an interior address belongs
to the preceding function. A reference to a string's start address is not proof
of a runtime string use, and misses references into its interior. Different
providers' recovered functions and types are not automatically the same entities.
Identity needs artifact digest + analysis profile + address space + object kind
and location/discriminator. A separate session lease controls whether a handle
is still usable. Identity, existence, lookup success and readiness are distinct.
## Hypotheses to try to falsify
- Fine-grained getters can lose to coherent fact batches through repeated
envelopes, schemas and round trips.
- A small workflow can be built from the same facts without maintaining a second
analysis implementation. Explicit selection must survive composition.
- Batching at MCP may reduce agent turns without saving any native analysis.
The engine's unit of work must be measured separately from the fact/read unit.
- Richer structured relations can improve investigation completeness more than
another wrapper around whole-function C text.
- On complex targets, a narrow result can omit the decisive evidence. Compare
coverage and final answers, counting all extra reads and failed paths.
## First prototype
`facts.mjs` consumes REA's existing normalized observations. It does not parse
assembly/C or guess ownership. The same methods support individual reads,
homogeneous batches, and one string-reference-function workflow. Native analysis,
target admission, caches, cancellation and full Evidence ownership stay in REA.
`server.mjs` adds experimental tools to the **complete** production MCP catalog.
It does not hide unrelated schemas to manufacture catalog savings. The additional
schemas are included in costs. Its operations are static reads; it never executes
the target or mutates an attached analysis database.
`benchmark.mjs` compares production calls, individual fact calls, batches and
composition using actual MCP clients and actual Kuna/Ghidra. Verification runs
are separate from timed equivalent-work runs. Each performance run repeats the
task in one session to distinguish first queries and warm native caches. Cold
Ghidra import/open cost must also be reported, not hidden in the query timer.
The prototype projects the current address-only `xrefs` contract. Reference kinds
stay unknown. Unsupported containing-function resolution stays unknown without
futile retries. This is intentionally a way to expose integration gaps rather
than claim that the adapter can recover information already discarded upstream.
Full Evidence is explicitly readable while its existing session owner retains
it; closing the binary expires those records. Paging covers the provider-returned
set and never establishes complete artifact discovery.
This is not a production contract proposal. Overlays, non-default address spaces,
typed edges, native batching and live agent decision quality need further work.
No measured claim about improved complex-program reasoning is established yet.
## Native experiments
After the build described below, with `REA_KUNA_COMMAND` configured:
```sh
mkdir -p .cache/atomic-re-tools
gcc -O2 -fno-pie -no-pie -o .cache/atomic-re-tools/interior-reference \
experiments/atomic-re-tools/interior-reference.c
# Run once for each variant: per-address, then inventory.
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/atomic-re-tools/native-relations.mjs \
.cache/atomic-re-tools/interior-reference HEAD:REA per-address \
.cache/atomic-re-tools/native-interior-per-address
python3 experiments/atomic-re-tools/verify-interior.py \
.cache/atomic-re-tools/interior-reference \
.cache/atomic-re-tools/native-interior-per-address.report.json \
.cache/atomic-re-tools/native-interior-inventory.report.json
```
Use the installed zlib/SQLite paths and `incorrect`/`malformed` to repeat the
native plan comparison. Append `--code-slice` to the per-address experiment for
an exact instruction-to-line probe. `mapped-code.mjs` rejects unsupported address
precision and never substitutes a nearby line for a missing mapping.
The experiment adds an opt-in typed reference projection at the existing Kuna
protocol owner. The existing public address-only result and its retained cache
shape remain the same; ordinary queries do not materialize the richer facts.
Experimental MCP tools have live-validated input schemas, but no canonical
output-schema/product-catalog integration. They are not a production SDK/API.
The v2 facade labels reference scope explicitly and connects **locations** in its
point-reference edges. A result's `target` preserves the caller's selector; it
is not a separate observation proving that an object exists at that address.
All experimental MCP results carry REA's canonical untrusted-target-content
marker. The v1 measurement table is retained with its revision label; it is not
relabelled as a measurement of this later response shape.
## Reproduce
Use the repository's pinned Node/npm and run `npm ci`, then
`npm run build:cached`. Supply existing installations through `REA_KUNA_COMMAND`,
`GHIDRA_INSTALL_DIR` and `JAVA_HOME`; this experiment installs no provider.
```sh
node --test experiments/atomic-re-tools/facts.test.mjs
# Existing bounded runner: one CPU, 4 GiB hard memory cap, early stop at 3 GiB.
# Its payload-exit counters include all processes in the owned scope.
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/atomic-re-tools/benchmark.mjs \
kuna batch /usr/lib/x86_64-linux-gnu/libz.so.1 incorrect \
.cache/atomic-re-tools/kuna-batch
```
Use `baseline`, `fine`, `batch`, or `workflow`, and `ghidra` or `kuna`.
Append `--verify` for additional lifecycle, paging, negative and Evidence checks;
those runs must not be used as equivalent-work performance samples. Binaries,
provider projects, captured catalogs/traffic and local measurements stay ignored
under `.cache/atomic-re-tools/`, never in commits. Summarize reproducible results
and limitations here as experiments finish.
+375
View File
@@ -0,0 +1,375 @@
import assert from "node:assert/strict";
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import Ajv2020 from "ajv/dist/2020.js";
import { requireMcpOperationResult } from "../../scripts/lib/mcp-verifier-results.mjs";
import {
createVerifierRun,
completeVerifierRun,
assertVerifierCleanupComplete,
} from "../../scripts/lib/verifier-run.mjs";
const [provider, variant, binary, pattern, prefix, verification] =
process.argv.slice(2);
assert.ok(["ghidra", "kuna"].includes(provider));
assert.ok(["baseline", "fine", "batch", "workflow"].includes(variant));
assert.ok(binary && pattern && prefix);
await mkdir(dirname(prefix), { recursive: true });
const verifier = createVerifierRun();
const client = new Client({ name: "rea-atomic-experiment", version: "0.1.0" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [
resolve(
variant === "baseline"
? "dist/main.js"
: "experiments/atomic-re-tools/server.mjs",
),
],
env: {
...process.env,
REA_LOG_LEVEL: "error",
REA_MCP_OUTPUT_SCHEMA_PROFILE:
process.env.REA_MCP_OUTPUT_SCHEMA_PROFILE ?? "full",
REA_KUNA_CACHE_DIR: resolve(`${prefix}.kuna-cache`),
REA_KUNA_JOBS: "1",
GHIDRA_HEADLESS_MAXMEM: "1G",
REA_GHIDRA_STARTUP_TIMEOUT_MS: "120000",
},
stderr: "pipe",
});
let stderr = "";
transport.stderr?.on("data", (part) => {
stderr = (stderr + part).slice(-32_768);
});
const wire = [],
calls = [],
checks = [],
failures = [];
let phase = "lifecycle";
const call = async (name, arguments_ = {}, expectError = false) => {
const started = performance.now();
const response = await client.callTool(
{ name, arguments: arguments_ },
{ timeout: 180000 },
);
wire.push({ request: { name, arguments: arguments_ }, response, phase });
if (name.startsWith("experiment_"))
assert.equal(
response._meta?.provenance?.untrusted_content,
true,
"Experimental results lost the canonical target-content marker",
);
calls.push({ name, phase, elapsed_ms: performance.now() - started });
if (expectError) {
assert.equal(response.isError, true);
return JSON.parse(
response.content.find((part) => part.type === "text").text,
);
}
assert.notEqual(
response.isError,
true,
JSON.stringify(response).slice(0, 2000),
);
return name.startsWith("experiment_")
? JSON.parse(response.content.find((part) => part.type === "text").text)
: requireMcpOperationResult(response, name);
};
const contextInput = (context) => ({ ref: context.ref, lease: context.lease });
const stringLimit = 3,
referenceLimit = 4;
const query = { pattern, mode: "literal", case_sensitive: true };
const identity = (row) => ({
kind: "string",
address: row.address,
encoding: row.string?.encoding ?? null,
});
const readBatch = async (name, common, { key, items, individually }) => {
if (!individually) return call(name, { ...common, [key]: items });
const results = [];
for (const item of items)
results.push(...(await call(name, { ...common, [key]: [item] })).results);
return { results };
};
const readExperimentalTask = async (supportsOwners) => {
const input = { query, limit: stringLimit, fields: ["value"] };
if (variant === "workflow") {
const traced = await call("experiment_trace_string_uses", {
...input,
reference_limit: referenceLimit,
});
return {
found: traced.strings,
refs: traced.references,
owners: traced.functions,
};
}
const found = await call("experiment_find_strings", input);
const context = contextInput(found.context);
const refs = await readBatch(
"experiment_incoming_refs",
{ context, limit: referenceLimit },
{
key: "targets",
items: found.items.map((row) => row.id),
individually: variant === "fine",
},
);
const locations = [
...new Map(
refs.results
.flatMap((row) => row.items ?? [])
.map((row) => [row.from.address, row.from]),
).values(),
];
const owners = await readBatch(
"experiment_resolve_functions",
{ context },
{
key: "locations",
items: locations,
individually: variant === "fine" && supportsOwners,
},
);
return { found, refs, owners };
};
let catalog,
firstSourceId,
firstContext,
outcomes = [];
try {
await client.connect(transport);
catalog = await client.listTools();
const expectedOutputSchemas =
process.env.REA_MCP_OUTPUT_SCHEMA_PROFILE === "off"
? 0
: catalog.tools.filter((tool) => !tool.name.startsWith("experiment_"))
.length;
assert.equal(
catalog.tools.filter((tool) => Object.hasOwn(tool, "outputSchema")).length,
expectedOutputSchemas,
"Advertised output-schema profile differs from the requested profile",
);
const ajv = new Ajv2020({ strict: false, validateFormats: false });
for (const tool of catalog.tools.filter((tool) =>
tool.name.startsWith("experiment_"),
)) {
assert.equal(
ajv.validateSchema(tool.inputSchema),
true,
JSON.stringify(ajv.errors),
);
ajv.compile(tool.inputSchema);
}
await call("open_binary", { path: resolve(binary), provider_id: provider });
const status = await call("binary_session");
const supportsOwners = status.capabilities.some(
(row) => row.operation === "resolve_containing_procedure" && row.available,
);
for (phase of ["cold", "warm"]) {
const started = performance.now();
let strings, references, functions;
if (variant === "baseline") {
const all = await call("search_strings", query);
strings = all
.slice(0, stringLimit)
.map((row) => ({ id: identity(row), value: row.value }));
references = [];
for (const string of strings)
references.push({
target: string.id,
items: (await call("xrefs", { address: string.id.address })).slice(
0,
referenceLimit,
),
});
const locations = [...new Set(references.flatMap((row) => row.items))];
functions = [];
for (const address of locations)
functions.push(
supportsOwners
? {
address,
resolution: await call("resolve_containing_procedure", {
address,
}),
}
: { address, status: "unknown" },
);
} else {
const { found, refs, owners } =
await readExperimentalTask(supportsOwners);
assert.equal(found.status, "ok");
assert.ok(refs.results.every((row) => row.status === "ok"));
firstSourceId ??= found.evidence_id;
firstContext ??= found.context;
strings = found.items.map((row) => ({ id: row.id, value: row.value }));
references = refs.results.map((row) => ({
target: row.target,
items: row.items.map((edge) => edge.from.address),
}));
functions = owners.results.map((row) =>
row.status === "ok"
? { address: row.location.address, resolution: row.resolution }
: { address: row.location.address, status: "unknown" },
);
}
outcomes.push({
phase,
elapsed_ms: performance.now() - started,
strings,
references,
functions,
});
}
assert.deepEqual(outcomes[0].strings, outcomes[1].strings);
assert.deepEqual(outcomes[0].references, outcomes[1].references);
assert.deepEqual(outcomes[0].functions, outcomes[1].functions);
assert.ok(outcomes[0].strings.length > 0);
phase = "verification";
if (variant !== "baseline" && verification === "--verify") {
const raw = await call("experiment_read_evidence", {
evidence_id: firstSourceId,
});
assert.equal(raw.evidence.operation, "search_strings");
assert.equal(
raw.evidence.analysis_profile.digest,
firstContext.analysis_profile_digest,
);
assert.equal(
raw.evidence.subject.digest.sha256,
createHash("sha256")
.update(await readFile(binary))
.digest("hex"),
);
assert.deepEqual(
raw.evidence.normalized_result
.slice(0, stringLimit)
.map((row) => ({ id: identity(row), value: row.value })),
outcomes[0].strings,
);
const duplicate = await call("experiment_incoming_refs", {
context: contextInput(firstContext),
targets: [outcomes[0].strings[0].id, outcomes[0].strings[0].id],
limit: 1,
});
assert.deepEqual(duplicate.results[0], duplicate.results[1]);
const nextPage = await call("experiment_find_strings", {
query,
offset: stringLimit,
limit: stringLimit,
});
assert.deepEqual(
nextPage.items.map((row) => row.id),
raw.evidence.normalized_result
.slice(stringLimit, stringLimit * 2)
.map(identity),
);
const negative = await call("experiment_find_strings", {
query: { pattern: "\u0000REA-atomic-negative\u0000" },
});
assert.equal(negative.items.length, 0);
assert.equal(negative.coverage.status, "unknown");
const stale = await call(
"experiment_incoming_refs",
{
context: {
ref: "different-artifact-or-profile",
lease: firstContext.lease,
},
targets: [outcomes[0].strings[0].id],
},
true,
);
assert.equal(stale.status, "error");
await call("close_binary");
assert.equal(
(
await call(
"experiment_read_evidence",
{ evidence_id: firstSourceId },
true,
)
).error.code,
"not_found",
);
await call("open_binary", { path: resolve(binary), provider_id: provider });
const expired = await call(
"experiment_incoming_refs",
{
context: contextInput(firstContext),
targets: [outcomes[0].strings[0].id],
},
true,
);
assert.equal(expired.status, "error");
checks.push(
"retained-evidence",
"paging",
"duplicate-targets",
"negative-unknown",
"foreign-context",
"expired-lease",
);
}
} catch (error) {
failures.push(String(error.stack ?? error));
}
phase = "lifecycle";
try {
await call("close_binary");
} catch (error) {
failures.push(String(error));
}
await client.close().catch((error) => failures.push(String(error)));
await transport.close().catch((error) => failures.push(String(error)));
const lifecycle = await completeVerifierRun(verifier);
try {
assertVerifierCleanupComplete(lifecycle.process_lineage);
} catch (error) {
failures.push(String(error));
}
const report = {
provider,
variant,
binary: resolve(binary),
pattern,
node_version: process.versions.node,
target_sha256: createHash("sha256")
.update(await readFile(binary))
.digest("hex"),
string_limit: stringLimit,
reference_limit: referenceLimit,
outcomes,
calls,
checks,
failures,
verification_only: verification === "--verify",
lifecycle,
};
await writeFile(`${prefix}.wire.json`, JSON.stringify(wire));
await writeFile(`${prefix}.catalog.json`, JSON.stringify(catalog));
await writeFile(`${prefix}.report.json`, JSON.stringify(report, null, 2));
await writeFile(`${prefix}.stderr.txt`, stderr);
console.log(
JSON.stringify({
provider,
variant,
calls: calls.length,
outcomes: outcomes.map(({ strings, references, functions, ...rest }) => ({
...rest,
strings: strings.length,
references: references.flatMap((row) => row.items).length,
functions: functions.length,
})),
failures,
}),
);
assert.deepEqual(failures, []);
@@ -0,0 +1,140 @@
// Capture the existing public MCP function dossier, without changing its contract.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import {
requireMcpOperationResult,
requireMcpToolError,
} from "../../scripts/lib/mcp-verifier-results.mjs";
import {
createVerifierRun,
completeVerifierRun,
assertVerifierCleanupComplete,
} from "../../scripts/lib/verifier-run.mjs";
const [provider, binary, procedure, prefix] = process.argv.slice(2);
assert.ok(
["ghidra", "kuna"].includes(provider) && binary && procedure && prefix,
);
await mkdir(dirname(prefix), { recursive: true });
const verifier = createVerifierRun();
const client = new Client({
name: "rea-relation-experiment",
version: "0.1.0",
});
const transport = new StdioClientTransport({
command: process.execPath,
args: [resolve("dist/main.js")],
env: {
...process.env,
REA_LOG_LEVEL: "error",
REA_MCP_OUTPUT_SCHEMA_PROFILE: "off",
GHIDRA_HEADLESS_MAXMEM: "1G",
REA_GHIDRA_STARTUP_TIMEOUT_MS: "120000",
REA_KUNA_CACHE_DIR: resolve(`${prefix}.kuna-cache`),
REA_KUNA_JOBS: "1",
},
stderr: "pipe",
});
let stderr = "";
transport.stderr?.on("data", (part) => {
stderr = (stderr + part).slice(-32_768);
});
const wire = [],
calls = [],
failures = [];
let catalog;
const call = async (name, arguments_ = {}, acceptRetained = false) => {
const start = performance.now();
const response = await client.callTool(
{ name, arguments: arguments_ },
{ timeout: 180000 },
);
wire.push({ request: { name, arguments: arguments_ }, response });
calls.push({ name, elapsed_ms: performance.now() - start });
if (response.isError === true && acceptRetained) {
const error = requireMcpToolError(response);
const source = error.details?.reported_limits?.evidence_reference;
assert.equal(error.code, "resource_constraint");
assert.equal(source?.kind, "retained-evidence");
await call("inspect_analysis_view", {
source,
view: { kind: "native", facet: "value_flow_summary" },
});
return { retained: source, initial_error: error };
}
assert.notEqual(
response.isError,
true,
JSON.stringify(response).slice(0, 1600),
);
return requireMcpOperationResult(response, name);
};
try {
await client.connect(transport);
catalog = await client.listTools();
assert.ok(
catalog.tools.every((tool) => !Object.hasOwn(tool, "outputSchema")),
);
await call("open_binary", { path: resolve(binary), provider_id: provider });
await call("binary_session");
for (const phase of ["first", "warm"]) {
const dossier = await call("analyze_function", { procedure }, true);
await writeFile(`${prefix}.${phase}.json`, JSON.stringify(dossier), {
flag: "wx",
});
}
await call("export_evidence_bundle", {
path: resolve(`${prefix}.bundle.json`),
});
} catch (error) {
failures.push(String(error.stack ?? error));
} finally {
try {
await call("close_binary");
} catch (error) {
failures.push(String(error));
}
await client.close().catch((error) => failures.push(String(error)));
await transport.close().catch((error) => failures.push(String(error)));
}
const lifecycle = await completeVerifierRun(verifier);
let cleanupVerified = false;
try {
assertVerifierCleanupComplete(lifecycle.process_lineage);
cleanupVerified = true;
} catch (error) {
failures.push(String(error));
}
await writeFile(`${prefix}.wire.json`, JSON.stringify(wire), { flag: "wx" });
await writeFile(`${prefix}.catalog.json`, JSON.stringify(catalog), {
flag: "wx",
});
await writeFile(`${prefix}.stderr.txt`, stderr, { flag: "wx" });
const report = {
provider,
binary: resolve(binary),
procedure,
target_sha256: createHash("sha256")
.update(await readFile(binary))
.digest("hex"),
node_version: process.versions.node,
calls,
failures,
lifecycle,
};
await writeFile(`${prefix}.report.json`, JSON.stringify(report, null, 2), {
flag: "wx",
});
console.log(
JSON.stringify({
provider,
calls,
failures,
cleanup_verified: cleanupVerified,
}),
);
if (failures.length) process.exitCode = 1;
@@ -0,0 +1,109 @@
// Native diagnostic probes; these text dumps are not a typed provider contract.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { parseBinaryTarget } from "../../dist/application/BinaryTargetResolver.js";
import { inspectKunaInstallation } from "../../dist/kuna/KunaInstallation.js";
import { KunaTargetCommands } from "../../dist/kuna/KunaTargetCommands.js";
import { AnalysisError } from "../../dist/domain/analysisErrorBase.js";
import { projectAnalysisError } from "../../dist/domain/analysisErrorProjection.js";
import {
createVerifierRun,
completeVerifierRun,
assertVerifierCleanupComplete,
} from "../../scripts/lib/verifier-run.mjs";
const [binary, procedure, view, prefix] = process.argv.slice(2);
assert.ok(binary && /^0x[0-9a-f]+$/u.test(procedure) && prefix);
assert.ok(["raw", "regions", "assembly"].includes(view));
await mkdir(dirname(prefix), { recursive: true });
const verifier = createVerifierRun();
const lifetime = new AbortController();
const target = await parseBinaryTarget(binary);
assert.ok(target.ok);
const installed = await inspectKunaInstallation({
...process.env,
REA_KUNA_CACHE_DIR: resolve(`${prefix}.cache`),
});
assert.ok(installed.ok);
const commands = new KunaTargetCommands(target.value, {
signal: lifetime.signal,
});
const failures = [],
calls = [];
try {
const arguments_ = (input) =>
view === "assembly"
? ["disassemble", input, procedure, "--addr", "--json"]
: [
"decompile",
input,
procedure,
"--addr",
`--${view}`,
"--max-fn-memory",
"1024",
];
const start = performance.now();
const outcome = await commands.run({
installation: installed.value,
operation: "experiment:structure",
arguments: arguments_,
outputBytes: 16 * 1024 * 1024,
timeoutMs: 120000,
signal: lifetime.signal,
});
if (!outcome.ok) throw outcome.error;
const snapshot = outcome.value;
const file = `${prefix}.native.txt`;
await writeFile(file, snapshot.stdout.text, { flag: "wx" });
calls.push({
arguments: arguments_(snapshot.input),
elapsed_ms: performance.now() - start,
file,
bytes: Buffer.byteLength(snapshot.stdout.text),
sha256: createHash("sha256").update(snapshot.stdout.text).digest("hex"),
});
} catch (error) {
failures.push(
error instanceof AnalysisError
? projectAnalysisError(error)
: String(error.stack ?? error),
);
} finally {
lifetime.abort();
await commands.close();
}
const lifecycle = await completeVerifierRun(verifier);
let cleanupVerified = false;
try {
assertVerifierCleanupComplete(lifecycle.process_lineage);
cleanupVerified = true;
} catch (error) {
failures.push(String(error));
}
const report = {
binary: resolve(binary),
procedure,
view,
calls,
failures,
lifecycle,
engine: { version: installed.value.version, sha256: installed.value.sha256 },
target_sha256: createHash("sha256")
.update(await readFile(binary))
.digest("hex"),
};
await writeFile(`${prefix}.report.json`, JSON.stringify(report, null, 2), {
flag: "wx",
});
console.log(
JSON.stringify({
view,
calls,
failures,
cleanup_verified: cleanupVerified,
}),
);
if (failures.length) process.exitCode = 1;
+210
View File
@@ -0,0 +1,210 @@
// Experimental application layer. Provider parsing and native caches remain in REA.
import { createHash } from "node:crypto";
export const contextFor = (artifactSha, profile, lease) => {
const identity = {
artifact_sha256: artifactSha,
analysis_profile_digest: profile.digest,
address_space: "provider-default",
};
return {
ref: createHash("sha256").update(JSON.stringify(identity)).digest("hex"),
...identity,
provider: profile.provider,
lease,
};
};
const failure = (code, message) => ({
status: "error",
error: { code, message },
});
const source = (evidence) => ({
evidence_id: evidence.evidence_id,
confidence: evidence.confidence,
limitations: evidence.limitations,
});
const pageOf = (items, offset, limit) => ({
items: items.slice(offset, offset + limit),
selection: {
matched: items.length,
offset,
returned: Math.max(0, Math.min(limit, items.length - offset)),
next_offset: offset + limit < items.length ? offset + limit : null,
},
});
/** One fact can be read alone or in a batch; a batch is not a new fact kind. */
export class FactQueries {
constructor(port) {
this.port = port;
}
async findStrings({ query, offset = 0, limit = 20, fields = ["value"] }) {
const observed = await this.port.observe("search_strings", query);
if (!observed.ok) return { status: "error", error: observed.error };
const evidence = observed.value;
const all = evidence.normalized_result;
const page = pageOf(all, offset, limit);
return {
status: "ok",
context: this.port.context(),
...source(evidence),
selection: page.selection,
// Paging is complete over this provider's answer, not all artifact strings.
coverage: {
status: "unknown",
reason:
"Provider inventory is not proof of every string in the artifact.",
},
items: page.items.map((row) => ({
id: {
kind: "string",
address: row.address,
encoding: row.string?.encoding ?? null,
},
...Object.fromEntries(
fields
.filter((key) => Object.hasOwn(row, key))
.map((key) => [key, row[key]]),
),
})),
};
}
validContext(context) {
const current = this.port.context();
return context.ref === current.ref && context.lease === current.lease;
}
async incomingReferences({ context, targets, offset = 0, limit = 20 }) {
if (!this.validContext(context))
return failure(
"invalid_request",
"Fact context or session lease is stale; find facts again on the selected target.",
);
// Share duplicate targets within this batch. Native/session cache ownership
// remains with the provider; this layer adds no long-lived result cache.
const observations = new Map();
const results = [];
for (const target of targets) {
let observed = observations.get(target.address);
if (observed === undefined) {
observed = await this.port.observe("xrefs", {
address: target.address,
});
observations.set(target.address, observed);
}
if (!observed.ok) {
results.push({ target, status: "error", error: observed.error });
continue;
}
const evidence = observed.value;
const page = pageOf(evidence.normalized_result, offset, limit);
results.push({
target,
status: "ok",
scope: { kind: "exact-address", address: target.address },
...source(evidence),
selection: page.selection,
coverage: {
status: "unknown",
reason:
"The existing xrefs contract does not carry reference completeness.",
},
items: page.items.map((address) => ({
from: { kind: "location", address },
to: { kind: "location", address: target.address },
kind: "unknown",
kind_reason: "The existing xrefs contract projects addresses only.",
})),
});
}
return {
status: results.every((row) => row.status === "ok") ? "ok" : "partial",
context: this.port.context(),
results,
};
}
async resolveFunctions({ context, locations }) {
if (!this.validContext(context))
return failure(
"invalid_request",
"Fact context or session lease is stale; find facts again on the selected target.",
);
if (!this.port.supports("resolve_containing_procedure"))
return {
status: locations.length === 0 ? "ok" : "partial",
context: this.port.context(),
results: locations.map((location) => ({
location,
status: "unknown",
error: {
code: "unsupported",
message:
"The selected provider does not advertise containing-procedure resolution.",
},
})),
};
const observations = new Map();
const results = [];
for (const location of locations) {
let observed = observations.get(location.address);
if (observed === undefined) {
observed = await this.port.observe("resolve_containing_procedure", {
address: location.address,
});
observations.set(location.address, observed);
}
results.push(
observed.ok
? {
location,
status: "ok",
...source(observed.value),
// Preserve the owner's complete tagged found/negative outcome.
resolution: observed.value.normalized_result,
}
: { location, status: "unknown", error: observed.error },
);
}
return {
status: results.every((row) => row.status === "ok") ? "ok" : "partial",
context: this.port.context(),
results,
};
}
async traceStringUses(input) {
const strings = await this.findStrings(input);
if (strings.status !== "ok") return strings;
const references = await this.incomingReferences({
context: strings.context,
targets: strings.items.map((row) => row.id),
offset: 0,
limit: input.reference_limit ?? 20,
});
if (references.status === "error") return references;
const locations = [
...new Map(
references.results
.flatMap((row) => row.items ?? [])
.map((row) => [row.from.address, row.from]),
).values(),
];
const functions =
input.include_functions === false
? null
: await this.resolveFunctions({ context: strings.context, locations });
return {
status:
references.status === "partial" || functions?.status === "partial"
? "partial"
: "ok",
strings,
references,
functions,
};
}
}
+160
View File
@@ -0,0 +1,160 @@
import assert from "node:assert/strict";
import test from "node:test";
import { FactQueries, contextFor } from "./facts.mjs";
const profile = {
digest: "p1",
provider: { id: "test", name: "Test", version: "1" },
};
const context = contextFor("artifact-a", profile, "lease-1");
const evidence = (normalized_result) => ({
evidence_id: "ev:test",
confidence: "derived",
limitations: ["fixture coverage only"],
normalized_result,
});
const port = (observe, extra = {}) => ({
context: () => context,
supports: () => true,
observe,
...extra,
});
const string = {
kind: "string",
address: "0x20000000000001",
encoding: "utf8",
};
test("artifact, profile and lease boundaries reject joins before invoking analysis", async () => {
let calls = 0;
const facts = new FactQueries(
port(async () => {
calls++;
throw new Error("must not run");
}),
);
for (const foreign of [
contextFor("artifact-b", profile, "lease-1"),
contextFor("artifact-a", { ...profile, digest: "p2" }, "lease-1"),
contextFor("artifact-a", profile, "lease-2"),
]) {
assert.equal(
(await facts.incomingReferences({ context: foreign, targets: [string] }))
.status,
"error",
);
assert.equal(
(await facts.resolveFunctions({ context: foreign, locations: [] }))
.status,
"error",
);
}
assert.equal(calls, 0);
assert.equal(contextFor("artifact-a", profile, "lease-2").ref, context.ref);
});
test("partial batches retain good observations and exact failure reasons", async () => {
const cancelled = {
code: "cancelled",
message: "caller cancelled",
details: { stage: "native" },
};
const facts = new FactQueries(
port(async (_, parameters) =>
parameters.address === "0xbad"
? { ok: false, error: cancelled }
: { ok: true, value: evidence(["0x20000000000002"]) },
),
);
const result = await facts.incomingReferences({
context,
targets: [string, { ...string, address: "0xbad" }],
});
assert.equal(result.status, "partial");
assert.equal(result.results[0].items[0].from.address, "0x20000000000002");
assert.equal(result.results[0].items[0].kind, "unknown");
assert.deepEqual(result.results[1].error, cancelled);
assert.equal(result.results[0].coverage.status, "unknown");
});
test("duplicate point lookups share work without turning selectors into observed objects", async () => {
let calls = 0;
const facts = new FactQueries(
port(async () => {
calls++;
return { ok: true, value: evidence(["0x20"]) };
}),
);
const otherEncoding = { ...string, encoding: "utf16le" };
const result = await facts.incomingReferences({
context,
targets: [string, otherEncoding, string],
});
assert.equal(calls, 1);
assert.deepEqual(
result.results.map((row) => row.target),
[string, otherEncoding, string],
);
assert.deepEqual(
result.results.map((row) => row.items[0].to),
Array(3).fill({ kind: "location", address: string.address }),
);
assert.equal(result.results[0].scope.kind, "exact-address");
});
test("paging and field selection retain identities, source and unknown coverage", async () => {
const rows = [
{
address: string.address,
value: "😀 marker",
string: {
encoding: "utf8",
encoding_status: "observed",
byte_length: 12,
},
},
{ address: "0x20000000000002", value: "other" },
];
const facts = new FactQueries(
port(async () => ({ ok: true, value: evidence(rows) })),
);
const page = await facts.findStrings({
query: { pattern: "" },
limit: 1,
fields: ["string"],
});
assert.deepEqual(page.items, [{ id: string, string: rows[0].string }]);
assert.equal(page.selection.next_offset, 1);
assert.equal(page.evidence_id, "ev:test");
assert.equal(page.coverage.status, "unknown");
const beyond = await facts.findStrings({
query: { pattern: "" },
offset: 99,
limit: 1,
});
assert.equal(beyond.selection.matched, 2);
assert.equal(beyond.selection.returned, 0);
assert.equal(beyond.coverage.status, "unknown");
});
test("unsupported function resolution remains unknown without futile native calls", async () => {
const facts = new FactQueries(
port(
async () => {
throw new Error("unsupported capability must not run");
},
{ supports: () => false },
),
);
const result = await facts.resolveFunctions({
context,
locations: [{ kind: "location", address: "0x12" }],
});
assert.equal(result.status, "partial");
assert.equal(result.results[0].status, "unknown");
assert.equal(result.results[0].error.code, "unsupported");
assert.equal(
(await facts.resolveFunctions({ context, locations: [] })).status,
"ok",
);
});
@@ -0,0 +1,12 @@
/* Static fixture: compile and inspect, never execute. The only reference into
* this string lands after its first byte. Point xrefs and object uses differ. */
static const char marker[] = "HEAD:REA atomic interior-only marker";
__attribute__((noinline)) const char *give_suffix(void) {
return marker + 5;
}
int main(int argc, char **argv) {
(void)argv;
return (unsigned char)*give_suffix() == (unsigned char)argc;
}
+300
View File
@@ -0,0 +1,300 @@
// Decoder for the pinned 1.833 print_raw format, not a production Kuna API.
// Relations come from native SSA annotations, never from reconstructed C text.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
const pc = "0x[0-9a-f]{1,16}";
const seq = `${pc}:[0-9a-f]{1,16}`;
const operationLine = new RegExp(`^(${seq}):\\s+(.+)$`, "u");
const blockLine = new RegExp(`^Basic Block (\\d+) (${pc})-(${pc})$`, "u");
const definitionToken = new RegExp(`\\((${seq})\\)`, "gu");
const constantToken = /#(0x[0-9a-f]{1,16})(?::(\d+))?(?![0-9a-f])/gu;
const destination = `Block_(\\d+):(${pc})`;
const conditional = new RegExp(
`^goto ${destination} if (.+) else ${destination}$`,
"u",
);
const direct = new RegExp(`^goto ${destination}$`, "u");
const implied = new RegExp(`^(${pc}):\\s+\\[ goto ${destination} \\]$`, "u");
const address = (value) => `0x${BigInt(value).toString(16)}`;
const operationId = (value) => {
const [location, time] = value.split(":");
return `${address(location)}:${BigInt(`0x${time}`).toString(16)}`;
};
const indexInto = (index, key, row) => {
if (!index.has(key)) index.set(key, []);
index.get(key).push(row);
};
const ordinal = (value) => {
const n = Number(value);
assert.ok(Number.isSafeInteger(n) && n >= 0);
return n;
};
const freeze = (value) => {
if (value && typeof value === "object" && !Object.isFrozen(value)) {
Object.freeze(value);
for (const child of Object.values(value)) freeze(child);
}
return value;
};
const validateDefinitions = (operations, issues) => {
for (const row of operations.values()) {
for (const relation of row.definitions) {
if (!operations.has(relation.definition))
issues.push({
source_line: row.source_line,
reason: `Missing definition ${relation.definition}`,
});
}
}
};
const parseOperation = (match, blockId, sourceLine) => {
const id = operationId(match[1]);
const statement = match[2];
const split = statement.indexOf(" = ");
const lhs = split < 0 ? "" : statement.slice(0, split);
const rhs = split < 0 ? statement : statement.slice(split + 3);
const kind = lhs.startsWith("*(")
? "store"
: rhs.startsWith("*(")
? "load"
: rhs.startsWith("call ")
? "call"
: statement.startsWith("goto ")
? "branch"
: statement.startsWith("switch ")
? "switch"
: statement.startsWith("return(")
? "return"
: rhs.includes(" ? ")
? "phi"
: "value";
// STORE's address is an input. An ordinary assignment's LHS is its output.
const inputText = kind === "store" ? statement : rhs;
const definitions = [...inputText.matchAll(definitionToken)].map(
(token, occurrence) => ({
definition: operationId(token[1]),
use: id,
occurrence,
// This is printed-varnode order, NOT a p-code input_index or phi predecessor.
}),
);
const literals = [...inputText.matchAll(constantToken)].map((token) => ({
value: address(token[1]),
printed_size_bytes: token[2] === undefined ? null : ordinal(token[2]),
}));
const row = {
id,
address: id.split(":")[0],
block: blockId,
kind,
statement,
definitions,
constants: literals,
source_line: sourceLine,
};
return row;
};
const controlEdges = (row) => {
const { statement } = row;
const edges = [];
const branch = conditional.exec(statement);
const jump = direct.exec(statement);
if (branch) {
for (const [target, targetAddress, polarity] of [
[branch[1], branch[2], "true"],
[branch[4], branch[5], "false"],
]) {
edges.push({
from: row.block,
to: ordinal(target),
target_address: address(targetAddress),
kind: "conditional",
polarity,
predicate: branch[3],
operation: row.id,
source_line: row.source_line,
});
}
} else if (jump) {
edges.push({
from: row.block,
to: ordinal(jump[1]),
target_address: address(jump[2]),
kind: "direct",
operation: row.id,
source_line: row.source_line,
});
}
return edges;
};
const graphStart = (lines) => {
const first = lines.findIndex(
(line, i) => line.startsWith("Basic Block 0 ") && lines[i - 1] === "0",
);
assert.ok(first >= 0, "Missing native SSA block graph header");
return first;
};
export const parseKunaSsaDump = (text) => {
assert.ok(
Buffer.byteLength(text) <= 16 * 1024 * 1024,
"SSA dump exceeds capture budget",
);
const lines = text.split("\n");
// The CLI places C before the root block graph header. Ignore that C entirely.
const first = graphStart(lines);
const operations = new Map(),
blocks = new Map(),
constants = new Map();
const users = new Map(),
incoming = new Map(),
edges = [],
issues = [];
let block;
for (let i = first; i < lines.length; i++) {
const line = lines[i];
const header = blockLine.exec(line);
if (header) {
const id = ordinal(header[1]);
assert.ok(!blocks.has(id), "Duplicate block identity");
// Stop is the last instruction's address, NOT an exclusive byte bound.
block = {
id,
start: address(header[2]),
stop_instruction: address(header[3]),
operations: [],
source_line: i + 1,
};
blocks.set(id, block);
continue;
}
if (!line.trim()) continue;
const fallthrough = implied.exec(line);
if (fallthrough) {
edges.push({
from: block.id,
to: ordinal(fallthrough[2]),
target_address: address(fallthrough[3]),
kind: "printed-implied-goto",
operation: null,
source_line: i + 1,
});
continue;
}
const match = operationLine.exec(line);
if (!match) {
issues.push({
source_line: i + 1,
reason: "Unrecognized native dump line",
});
continue;
}
const row = parseOperation(match, block.id, i + 1);
assert.ok(!operations.has(row.id), "Duplicate operation identity");
operations.set(row.id, row);
block.operations.push(row.id);
for (const literal of new Set(row.constants.map((c) => c.value)))
indexInto(constants, literal, row.id);
for (const edge of row.definitions) indexInto(users, edge.definition, edge);
edges.push(...controlEdges(row));
}
validateDefinitions(operations, issues);
for (const edge of edges) {
assert.equal(
blocks.get(edge.to)?.start,
edge.target_address,
"Branch target identity mismatch",
);
indexInto(incoming, edge.to, edge);
}
const outgoing = new Set(edges.map((edge) => edge.from));
const unrepresented = [...blocks.values()]
.filter((b) => {
const tail = operations.get(b.operations.at(-1));
return tail?.kind !== "return" && !outgoing.has(b.id);
})
.map((b) => b.id);
const digest = createHash("sha256").update(text).digest("hex");
for (const table of [operations, blocks, constants, users, incoming])
for (const value of table.values()) freeze(value);
return freeze({
identity: { kind: "native-ssa-snapshot", sha256: digest },
coverage: {
scope: "printed-SSA",
operations: operations.size,
blocks: blocks.size,
explicit_control_edges: edges.length,
unrepresented_outgoing_blocks: unrepresented,
issues,
control_flow: "partial",
machine_semantic_equivalence: "unknown",
},
// The owner parses/indexes once; all compositions reuse the same facts.
operation: (id) => operations.get(id) ?? null,
block: (id) => blocks.get(id) ?? null,
findConstant: (value) => [...(constants.get(address(value)) ?? [])],
uses: (id) => users.get(id) ?? [],
incoming: (id) => incoming.get(id) ?? [],
});
};
// Dependency-closed only within the caller's selected budget and boundaries.
export const backwardSlice = (
snapshot,
seeds,
{ maxOperations = 48, stopAt = [], stopKinds = ["phi", "load"] } = {},
) => {
assert.ok(Number.isSafeInteger(maxOperations) && maxOperations > 0);
const stopped = new Set(stopAt),
kinds = new Set(stopKinds);
const queue = [...new Set(seeds)],
queued = new Set(queue),
seen = new Set(),
selected = [],
boundaries = [],
missing = [];
for (let cursor = 0; cursor < queue.length; cursor++) {
const id = queue[cursor];
if (seen.has(id)) continue;
seen.add(id);
const row = snapshot.operation(id);
if (!row) {
missing.push(id);
continue;
}
if (selected.length >= maxOperations) {
boundaries.push({ id, reason: "operation-budget" });
continue;
}
selected.push(row);
if (stopped.has(id) || kinds.has(row.kind)) {
boundaries.push({
id,
reason: stopped.has(id) ? "caller-selected" : row.kind,
});
} else
for (const edge of row.definitions) {
if (!queued.has(edge.definition)) {
queue.push(edge.definition);
queued.add(edge.definition);
}
}
}
return {
operations: selected,
coverage: {
status:
boundaries.length || missing.length || snapshot.coverage.issues.length
? "partial"
: "complete-for-printed-definitions",
boundaries,
missing,
source_issues: snapshot.coverage.issues.length,
runtime_reachability: "unknown",
memory_alias_dependencies: "unknown",
},
};
};
@@ -0,0 +1,120 @@
import assert from "node:assert/strict";
import test from "node:test";
import { backwardSlice, parseKunaSsaDump } from "./kuna-ssa.mjs";
import { queryLiteralUses } from "./query-ssa.mjs";
// Minimal native print_raw syntax, including a STORE pointer on its LHS.
const dump = `ignored C text
0
Basic Block 0 0x100-0x106
0x100:1: r0(0x100:1) = #0xffffffffffffffff
0x102:2: u0(0x102:2) = r0(0x100:1) != #0x0
0x106:3: goto Block_1:0x110 if (u0(0x102:2) != 0) else Block_2:0x120
Basic Block 1 0x110-0x114
0x110:4: u1(0x110:4) = ->(RDI(i),#0x30)
0x112:5: *(ram,u1(0x110:4)) = r0(0x100:1)
0x114:6: return(#0x0)
Basic Block 2 0x120-0x120
0x120:7: switch r0(0x100:1)
`;
test("SSA definitions preserve STORE address dependencies, exact u64 and conditional polarity", () => {
const snapshot = parseKunaSsaDump(dump);
assert.deepEqual(snapshot.findConstant("0xffffffffffffffff"), ["0x100:1"]);
assert.deepEqual(
snapshot.operation("0x112:5").definitions.map((e) => e.definition),
["0x110:4", "0x100:1"],
);
assert.equal(snapshot.operation("0x100:1").definitions.length, 0);
assert.deepEqual(
snapshot.incoming(1).map((e) => [e.from, e.polarity]),
[[0, "true"]],
);
assert.deepEqual(
snapshot.incoming(2).map((e) => [e.from, e.polarity]),
[[0, "false"]],
);
assert.deepEqual(snapshot.coverage.unrepresented_outgoing_blocks, [2]);
assert.equal(snapshot.coverage.control_flow, "partial");
});
test("a bounded slice names budget frontiers; graph cycles terminate", () => {
const snapshot = parseKunaSsaDump(dump);
const result = backwardSlice(snapshot, ["0x112:5"], { maxOperations: 1 });
assert.equal(result.operations.length, 1);
assert.equal(result.coverage.boundaries.length, 2);
assert.equal(result.coverage.status, "partial");
const cyclic = parseKunaSsaDump(
dump.replace("#0xffffffffffffffff", "r0(0x100:1)"),
);
assert.equal(backwardSlice(cyclic, ["0x100:1"]).operations.length, 1);
});
test("snapshot identity changes with source; malformed and missing facts cannot look complete", () => {
assert.notEqual(
parseKunaSsaDump(dump).identity.sha256,
parseKunaSsaDump(dump + "unknown native line\n").identity.sha256,
);
assert.equal(
parseKunaSsaDump(dump + "unknown native line\n").coverage.issues.length,
1,
);
assert.throws(
() => parseKunaSsaDump(dump.replace("Block_1:0x110", "Block_1:0x111")),
/identity mismatch/u,
);
assert.throws(
() => parseKunaSsaDump(dump.replace("0x112:5:", "0x100:1:")),
/Duplicate operation/u,
);
const missing = parseKunaSsaDump(
dump.replace("r0(0x100:1) !=", "r0(0xff:9) !="),
);
assert.equal(missing.coverage.issues.length, 1);
assert.deepEqual(backwardSlice(missing, ["0x102:2"]).coverage.missing, [
"0xff:9",
]);
});
test("phi/load cutoffs preserve boundary rows and do not invent memory or predecessor bindings", () => {
const snapshot = parseKunaSsaDump(
dump.replace("r0(0x100:1) != #0x0", "r0(0x100:1) ? r0(0x100:1)"),
);
const result = backwardSlice(snapshot, ["0x102:2"]);
assert.deepEqual(result.coverage.boundaries, [
{ id: "0x102:2", reason: "phi" },
]);
assert.equal(result.operations.length, 1);
assert.equal(result.operations[0].definitions.length, 2);
assert.equal(result.coverage.memory_alias_dependencies, "unknown");
});
test("cloned blocks share starts; operation identity includes sequence, and facts stay immutable", () => {
const snapshot = parseKunaSsaDump(dump.replaceAll("0x120", "0x110"));
assert.equal(snapshot.block(1).start, snapshot.block(2).start);
assert.notEqual(snapshot.block(1).id, snapshot.block(2).id);
assert.throws(() => {
snapshot.operation("0x100:1").kind = "store";
}, TypeError);
assert.throws(() => snapshot.uses("0x100:1").pop(), TypeError);
const query = queryLiteralUses(snapshot, "0xffffffffffffffff");
assert.deepEqual(query.constant_operations, ["0x100:1"]);
assert.ok(query.direct_uses.includes("0x112:5"));
assert.equal(query.coverage.control_flow, "partial");
});
test("a literal directly in STORE is itself an effect seed; shared block edges are returned once", () => {
const snapshot = parseKunaSsaDump(
dump.replace(
"*(ram,u1(0x110:4)) = r0(0x100:1)",
"*(ram,u1(0x110:4)) = #0x20",
),
);
const query = queryLiteralUses(snapshot, "0x20");
assert.deepEqual(query.effect_seed_operations, ["0x112:5"]);
assert.deepEqual(
query.control_edges.map((e) => e.operation),
["0x106:3"],
);
assert.ok(query.operations.some((row) => row.id === "0x102:2"));
});
+121
View File
@@ -0,0 +1,121 @@
// A view over native printer-to-instruction mappings, never a C-text parser.
import assert from "node:assert/strict";
import { z } from "zod";
const mapping = z.object({
line_number: z.number().int().positive(),
addresses: z.array(z.number().nonnegative()),
});
const documentSchema = z.object({
binary: z.string(),
count: z.number().int().nonnegative(),
functions: z.array(
z.object({
address_hex: z.string(),
address: z.number().nonnegative(),
code: z.string().nullable(),
error: z.string().nullable(),
line_mappings: z.array(mapping).optional(),
}),
),
});
export const mappedCode = async (native, reference, contextLines = 16) => {
if (reference.functions.length !== 1 || !Array.isArray(reference.sites))
return {
status: "unknown",
reason:
"A unique native owning function and exact reference sites are required.",
};
const entry = reference.functions[0].address;
const snapshot = await native((input) => [
"decompile",
input,
entry,
"--addr",
"--json",
"--max-fn-memory",
"1024",
]);
const document = documentSchema.parse(JSON.parse(snapshot.stdout.text));
assert.equal(document.binary, snapshot.input);
assert.equal(document.count, document.functions.length);
const candidates = document.functions.filter(
(row) => row.address_hex === entry,
);
const fn = candidates.length === 1 ? candidates[0] : undefined;
if (fn === undefined || fn.code === null || fn.line_mappings === undefined)
return {
status: "unknown",
reason:
"Native decompilation did not supply a body with instruction-to-line mappings.",
native_error: fn?.error ?? null,
};
// This low-VMA experiment rejects unsafe numeric arrays, rather than rounding
// u64 addresses. General u64 array support would need the owning JSON parser.
if (
!Number.isSafeInteger(fn.address) ||
`0x${fn.address.toString(16)}` !== entry ||
fn.line_mappings.some((row) =>
row.addresses.some((value) => !Number.isSafeInteger(value)),
)
)
return {
status: "unknown",
reason:
"Instruction mapping contains an address outside this prototype's exact safe-integer coverage.",
};
const sites = new Set(reference.sites.map((row) => row.from));
const hits = fn.line_mappings.filter((row) =>
row.addresses.some((value) => sites.has(`0x${value.toString(16)}`)),
);
const mappedSites = new Set(
hits.flatMap((row) =>
row.addresses.map((value) => `0x${value.toString(16)}`),
),
);
const unmappedSites = [...sites].filter((site) => !mappedSites.has(site));
const lines = fn.code.split("\n");
for (const hit of hits) assert.ok(hit.line_number <= lines.length);
if (!hits.length)
return {
status: "unknown",
reason:
"No exact reference site maps to an emitted line; nearest-line guessing is disabled.",
};
const ranges = hits.map((hit) => ({
start_line: Math.max(1, hit.line_number - contextLines),
end_line: Math.min(lines.length, hit.line_number + contextLines),
}));
const slices = [];
for (const range of ranges.sort((a, b) => a.start_line - b.start_line)) {
const previous = slices.at(-1);
if (previous !== undefined && range.start_line <= previous.end_line + 1)
previous.end_line = Math.max(previous.end_line, range.end_line);
else slices.push({ ...range });
}
return {
status: unmappedSites.length || fn.error !== null ? "partial" : "ok",
native_error: fn.error,
entry,
reference_sites: [...sites],
unmapped_reference_sites: unmappedSites,
exact_mapped_lines: hits.map((row) => row.line_number),
total_lines: lines.length,
total_characters: fn.code.length,
slices: slices.map((range) => ({
...range,
text: lines.slice(range.start_line - 1, range.end_line).join("\n"),
})),
coverage: {
status: "partial",
reason:
"Selected text around exact mapped sites; not a proof that every relevant guard or data dependency is included.",
},
limitations: [
"Single-function engine reconstruction, not original source.",
"Printer line mappings are derived and do not prove runtime reachability.",
"Complete native body and mappings remain in the retained source document.",
],
};
};
@@ -0,0 +1,56 @@
import assert from "node:assert/strict";
import test from "node:test";
import { mappedCode } from "./mapped-code.mjs";
const reference = {
functions: [{ address: "0x1000" }],
sites: [{ from: "0x2000" }, { from: "0x2001" }],
};
const document = () => ({
binary: "/private/input",
count: 1,
functions: [
{
address: 4096,
address_hex: "0x1000",
code: "one\ntwo\nthree\nfour\nfive",
error: null,
line_mappings: [{ line_number: 3, addresses: [8192] }],
},
],
});
const native = (value) => async () => ({
input: "/private/input",
stdout: { text: JSON.stringify(value) },
});
test("a mapped subset names unmapped sites instead of implying complete mapping", async () => {
const result = await mappedCode(native(document()), reference, 1);
assert.equal(result.status, "partial");
assert.deepEqual(result.unmapped_reference_sites, ["0x2001"]);
assert.deepEqual(result.slices, [
{ start_line: 2, end_line: 4, text: "two\nthree\nfour" },
]);
assert.equal(result.coverage.status, "partial");
});
test("unmapped locations never select a nearby printed line", async () => {
const result = await mappedCode(native(document()), {
...reference,
sites: [{ from: "0x1fff" }],
});
assert.equal(result.status, "unknown");
});
test("unsafe numeric address arrays stay unknown without lossy conversion", async () => {
const value = document();
value.functions[0].line_mappings[0].addresses = [Number.MAX_SAFE_INTEGER + 1];
const result = await mappedCode(native(value), reference);
assert.equal(result.status, "unknown");
});
test("malformed line locations reject the view before publishing fragments", async () => {
const value = document();
value.functions[0].line_mappings[0].line_number = 99;
await assert.rejects(mappedCode(native(value), reference));
});
@@ -0,0 +1,274 @@
// Native-boundary experiment, not MCP and not a production provider extension.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { z } from "zod";
import { AnalysisError } from "../../dist/domain/analysisErrorBase.js";
import { projectAnalysisError } from "../../dist/domain/analysisErrorProjection.js";
import { mappedCode } from "./mapped-code.mjs";
import { parseBinaryTarget } from "../../dist/application/BinaryTargetResolver.js";
import { inspectKunaInstallation } from "../../dist/kuna/KunaInstallation.js";
import { KunaTargetCommands } from "../../dist/kuna/KunaTargetCommands.js";
import { parseKunaJson } from "../../dist/kuna/KunaJson.js";
import { parseKunaStringInventory } from "../../dist/kuna/KunaStringInventory.js";
import { parseKunaIncomingReferenceFacts } from "../../dist/kuna/KunaReferenceFiles.js";
import {
createVerifierRun,
completeVerifierRun,
assertVerifierCleanupComplete,
} from "../../scripts/lib/verifier-run.mjs";
const [binary, pattern, variant, prefix, codeOption] = process.argv.slice(2);
assert.ok(binary && pattern && prefix);
assert.ok(["per-address", "inventory"].includes(variant));
await mkdir(dirname(prefix), { recursive: true });
const verifier = createVerifierRun();
const lifetime = new AbortController();
const parsed = await parseBinaryTarget(binary);
assert.ok(parsed.ok);
const installed = await inspectKunaInstallation({
...process.env,
REA_KUNA_CACHE_DIR: resolve(`${prefix}.cache`),
});
assert.ok(installed.ok);
const commands = new KunaTargetCommands(parsed.value, {
signal: lifetime.signal,
});
const address = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/u);
const functionSchema = z.object({
name: z.string(),
address,
address_hex: address,
});
const relatedRow = z.object({
address,
address_hex: address,
text: z.string(),
length: z.number().int().nonnegative(),
encoding: z.string(),
byte_length: z.number().int().nonnegative(),
nul_terminated: z.boolean(),
xrefs_count: z.number().int().nonnegative(),
functions: z.array(functionSchema),
});
const sourceKeys = new Set(["address", "from_address", "to_address"]);
const limit = 16 * 1024 * 1024;
const calls = [],
failures = [];
let facts;
const native = async (arguments_) => {
const start = performance.now();
const outcome = await commands.run({
installation: installed.value,
operation: "experiment:native-relations",
arguments: arguments_,
outputBytes: limit,
timeoutMs: 120000,
signal: lifetime.signal,
});
if (!outcome.ok) throw outcome.error;
const snapshot = outcome.value;
const file = `${prefix}.native-${calls.length}.json`;
await writeFile(file, snapshot.stdout.text);
calls.push({
elapsed_ms: performance.now() - start,
file,
arguments: arguments_(snapshot.input),
sha256: createHash("sha256").update(snapshot.stdout.text).digest("hex"),
bytes: Buffer.byteLength(snapshot.stdout.text),
});
return snapshot;
};
try {
// Common producer: validate all string facts once through the existing owner.
const scan = await native((input) => [
"strings",
input,
"--json",
"--encoding",
"all",
"--min-length",
"5",
"--termination",
"any",
"--no-xrefs",
]);
const inventory = parseKunaStringInventory(scan.stdout.text, scan.input);
assert.ok(inventory.ok, inventory.error);
const selected = inventory.value.items
.filter((row) => row.value.includes(pattern))
.slice(0, 3);
assert.ok(selected.length > 0);
const results = [];
if (variant === "per-address") {
for (const item of selected) {
const result = await native((input) => [
"xrefs",
input,
"--to",
item.address,
"--json",
]);
// Consume the owning parser's facts, without reinterpreting raw JSON.
const validated = parseKunaIncomingReferenceFacts(
result.stdout.text,
result.input,
item.address,
);
assert.ok(validated.ok, validated.error);
const rows = validated.value.facts;
const functions = [
...new Map(
rows
.flatMap((row) =>
row.fromFunction === null ? [] : [row.fromFunction],
)
.map((fn) => [fn.address, fn]),
).values(),
];
results.push({
string: item,
scope: "exact-start-address",
xrefs_count: rows.length,
functions,
sites: rows.map((row) => ({
from: row.from,
to: row.to,
kind: row.kind,
instruction: row.instruction,
})),
source: calls.at(-1),
});
}
} else {
const result = await native((input) => [
"strings",
input,
"--json",
"--encoding",
"all",
"--min-length",
"5",
"--termination",
"any",
]);
const document = parseKunaJson(result.stdout.text, limit, sourceKeys);
assert.equal(document.binary, result.input);
assert.equal(document.xrefs, true);
assert.equal(document.encoding, "all");
assert.equal(document.min_length, 5);
assert.equal(document.termination, "any");
assert.equal(document.filter, null);
assert.equal(document.section, null);
assert.equal(document.scanned, inventory.value.scanned);
assert.equal(document.count, document.strings.length);
assert.equal(document.count, inventory.value.items.length);
const known = new Map(
inventory.value.items.map((row) => [
`${row.address}|${row.string.encoding}`,
row,
]),
);
const examined = new Map();
for (const value of document.strings) {
const row = relatedRow.parse(value);
assert.equal(row.address, row.address_hex);
const key = `${row.address}|${row.encoding}`;
assert.ok(!examined.has(key));
const expected = known.get(key);
assert.ok(expected);
assert.equal(row.text, expected.value);
assert.equal(row.length, [...expected.value].length);
assert.equal(row.byte_length, expected.string.byte_length);
assert.equal(
row.nul_terminated,
expected.string.termination === "present_or_not_required",
);
const entries = new Set();
for (const fn of row.functions) {
assert.equal(fn.address, fn.address_hex);
assert.ok(!entries.has(fn.address));
entries.add(fn.address);
}
assert.ok(row.functions.length <= row.xrefs_count);
examined.set(key, row);
}
for (const item of selected) {
const row = examined.get(`${item.address}|${item.string.encoding}`);
results.push({
string: item,
scope: "any-byte-in-string-extent",
xrefs_count: row.xrefs_count,
functions: row.functions,
sites: {
status: "unknown",
reason:
"Native string attribution does not return reference sites or kinds.",
},
source: calls.at(-1),
});
}
}
const code =
codeOption === "--code-slice" ? await mappedCode(native, results[0]) : null;
facts = {
artifact_sha256: parsed.value.sha256,
engine: {
version: installed.value.version,
sha256: installed.value.sha256,
},
inventory_count: inventory.value.items.length,
selected_count: selected.length,
results,
mapped_code: code === null ? null : { ...code, source: calls.at(-1) },
limitations: [
"These are native derived static references, not runtime uses.",
"The per-address and inventory scopes differ; their timings do not establish equivalent-query speedup.",
"An empty function set can mean unowned references, not that a string is unused.",
"Complete native source documents are retained locally by file and SHA-256; this is not an MCP token measurement.",
],
};
} catch (error) {
failures.push(
error instanceof AnalysisError
? projectAnalysisError(error)
: { message: String(error), stack: error.stack ?? null },
);
}
lifetime.abort();
const closed = await commands.close();
if (!closed.ok) failures.push(String(closed.error));
const lifecycle = await completeVerifierRun(verifier);
try {
assertVerifierCleanupComplete(lifecycle.process_lineage);
} catch (error) {
failures.push(String(error));
}
await writeFile(
`${prefix}.report.json`,
JSON.stringify(
{
variant,
binary: resolve(binary),
pattern,
calls,
facts,
failures,
lifecycle,
},
null,
2,
),
);
console.log(
JSON.stringify({
variant,
native_commands: calls.length,
native_ms: calls.reduce((sum, row) => sum + row.elapsed_ms, 0),
selected: facts?.selected_count,
owners: facts?.results.map((row) => row.functions.length),
failures,
}),
);
assert.deepEqual(failures, []);
+168
View File
@@ -0,0 +1,168 @@
// A composition over one validated, immutable snapshot; no new engine work.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { pathToFileURL } from "node:url";
import { backwardSlice, parseKunaSsaDump } from "./kuna-ssa.mjs";
import { assertVerifierCleanupComplete } from "../../scripts/lib/verifier-run.mjs";
export const queryLiteralUses = (snapshot, literal, maxOperations = 48) => {
const allSeeds = snapshot.findConstant(literal);
const seeds = allSeeds.slice(0, 16);
const effectSeeds = seeds.filter((id) =>
["store", "call"].includes(snapshot.operation(id).kind),
);
const allUses = [
...new Set([
...effectSeeds,
...seeds.flatMap((id) => snapshot.uses(id).map((e) => e.use)),
]),
];
const selectedUses = allUses.slice(0, 16);
const consumers = selectedUses.map((id) => snapshot.operation(id));
const controlEdges = [
...new Map(
consumers
.flatMap((row) => snapshot.incoming(row.block))
.map((edge) => [
JSON.stringify([edge.from, edge.to, edge.operation, edge.polarity]),
edge,
]),
).values(),
];
const branchIds = [
...new Set(
controlEdges.flatMap((edge) =>
edge.operation === null ? [] : [edge.operation],
),
),
];
const guards = backwardSlice(snapshot, branchIds, { maxOperations });
const facts = [
...new Map(
[
...seeds.map((id) => snapshot.operation(id)),
...consumers,
...guards.operations,
].map((row) => [row.id, row]),
).values(),
];
const selected = new Set(facts.map((row) => row.id));
return {
literal,
snapshot: snapshot.identity,
constant_operations: seeds,
effect_seed_operations: effectSeeds,
direct_uses: selectedUses,
control_edges: controlEdges,
operations: facts,
coverage: {
status: "partial",
seed_matches: allSeeds.length,
omitted_seed_matches: allSeeds.length - seeds.length,
omitted_direct_uses: allUses.length - selectedUses.length,
dependencies: guards.coverage,
unexpanded_definition_ids: [
...new Set(
facts.flatMap((row) =>
row.definitions
.filter((edge) => !selected.has(edge.definition))
.map((edge) => edge.definition),
),
),
],
control_flow: snapshot.coverage.control_flow,
unrepresented_outgoing_blocks:
snapshot.coverage.unrepresented_outgoing_blocks.length,
source_issues: snapshot.coverage.issues.length,
scope:
"literal equality, effect seeds and direct printed uses, their explicit incoming edges and bounded branch definitions",
address_object_identity: "unknown",
call_argument_bindings: "unknown",
},
};
};
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
const main = async () => {
const [prefix, values, output] = process.argv.slice(2);
assert.ok(prefix && values && output);
const literals = values.split(",");
assert.ok(
literals.length <= 16 &&
literals.every((s) => /^0x[0-9a-f]{1,16}$/u.test(s)),
);
const report = JSON.parse(await readFile(`${prefix}.report.json`, "utf8"));
assert.equal(report.view, "raw");
assert.deepEqual(report.failures, []);
assert.equal(report.calls.length, 1);
assert.equal(
report.engine.version,
"1.833",
"Decoder is pinned to the inspected printer format",
);
assertVerifierCleanupComplete(report.lifecycle.process_lineage);
const source = report.calls[0];
const raw = await readFile(source.file, "utf8");
assert.equal(sha256(raw), source.sha256);
assert.equal(Buffer.byteLength(raw), source.bytes);
assert.equal(
sha256(await readFile(report.binary)),
report.target_sha256,
"Artifact changed since capture",
);
const start = performance.now();
const snapshot = parseKunaSsaDump(raw);
const parseIndexMs = performance.now() - start;
const runs = [];
for (const phase of ["first-projection", "reused-index"]) {
const before = performance.now();
const answers = literals.map((literal) =>
queryLiteralUses(snapshot, literal),
);
const serialized = JSON.stringify(answers);
runs.push({
phase,
query_and_serialization_ms: performance.now() - before,
answers,
serialized_bytes: Buffer.byteLength(serialized),
});
}
assert.deepEqual(runs[0].answers, runs[1].answers);
await writeFile(
output,
JSON.stringify(
{
source: {
artifact_sha256: report.target_sha256,
engine: report.engine,
procedure: report.procedure,
capture_report: `${prefix}.report.json`,
raw: source,
profile:
"one-function native decompile defaults + --raw; no annotations",
},
parse_index_ms: parseIndexMs,
coverage: snapshot.coverage,
runs,
lifetime:
"One local process; validated immutable snapshot reused without native analysis. No cross-session cache.",
},
null,
2,
),
{ flag: "wx" },
);
console.log(
JSON.stringify({
output,
parse_index_ms: parseIndexMs,
runs: runs.map(({ answers, ...timing }) => ({
...timing,
selected_operations: answers.map((a) => a.operations.length),
})),
}),
);
};
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
await main();
+239
View File
@@ -0,0 +1,239 @@
# Control-flow and definition/use experiment
This continues the first-round experiment on the same REA source baseline and
Kuna 1.833. It is an exploratory native-boundary decoder, not a production
provider capability or a new MCP catalog. No PR or merge is intended.
The analyst questions are which local conditions lead to zlib's
`incorrect header check` assignment and SQLite's
`malformed MATCH expression: [%s]` error-formatting call. Runtime reachability,
memory aliasing, phi predecessor bindings and complete entry-state conditions
remain unknown. These are not controlled agent trials.
`capture-function.mjs` records the existing public MCP function dossier. It
preserves transport-budget failures and follows retained Evidence for a value-flow
summary, then exports the complete local Evidence bundle before closing.
`capture-kuna-structure.mjs` uses REA's existing target/process owners to capture
the native `--raw`, `--regions` or disassembly output. All target inspection is
static. The engine and artifact digests, command arguments, raw digest and cleanup
proof are retained locally.
`kuna-ssa.mjs` owns the decoder for the inspected 1.833 SSA diagnostic format.
It indexes explicit definition annotations, literal values, block membership and
printed branch destinations once. Immutable rows carry native source lines;
operation and block IDs are local to the content-digested snapshot. A block's
printed stop is an instruction address, not an exclusive extent. Blocks can share
start addresses. Unknown lines, omitted outgoing relationships, missing definitions
and traversal boundaries remain explicit. Printed operand occurrences do not
claim p-code input indexes or phi-to-predecessor bindings.
`query-ssa.mjs` composes literal equality, direct uses, incoming explicit edges
and bounded backward definitions. It validates the retained capture against its
digest and the current artifact before indexing. It retains complete native
evidence; its local cache lives for one process. Default phi/load boundaries avoid
expanding the whole program state. Selection limits report omissions.
Use the pinned Node from `.nvmrc`, an existing build and caller-supplied Kuna:
```sh
node --test experiments/atomic-re-tools/kuna-ssa.test.mjs
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/atomic-re-tools/capture-kuna-structure.mjs \
/usr/lib/x86_64-linux-gnu/libz.so.1 0xc1e0 raw .cache/atomic-re-tools/relations-zlib
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/atomic-re-tools/query-ssa.mjs .cache/atomic-re-tools/relations-zlib \
0x18fc4 .cache/atomic-re-tools/relations-zlib-query.json
```
The addresses above apply only to the exact zlib artifact in [results](results.md).
Do not reuse them across provider load spaces or binary versions. Complete
measurement and validation notes follow below.
## Observed results
The same artifact and engine digests as [round one](results.md) apply. This is
the existing experiment, not a newly rebased feature branch: newer REA main
`42fc6c53ae188978deb3bedfe88287b6b316ba22` was fetched for inspection, but these
runs retain baseline `5b16f2431062eea6ffd245a9e05512790007cb13` and Kuna 1.833.
The source decoder was checked against Kuna's pinned
[SSA renderer](https://github.com/Noelo-Lab/kuna/blob/fe01d6294eb0d1053ce343c8943646df88e143fc/decompiler/crates/kuna-decomp/src/substrate/funcdata_printraw.rs).
No newer Kuna version was tested.
1. **SSA value origins repair all three failed zlib mappings.** The original
string xrefs are LEAs whose addresses do not appear in printer mappings.
Literal equality in optimized SSA finds a pointer-valued operation; its
explicit def-use points to a STORE at a different instruction origin. Those
origins do appear in the retained JSON line mappings:
| String address | Original LEA | STORE origin | Exact emitted line |
| -------------- | ------------ | ------------ | -----------------: |
| `0x18fc4` | `0xded7` | `0xdee6` | 164 |
| `0x19003` | `0xdc6a` | `0xdc74` | 551 |
| `0x19018` | `0xcd05` | `0xcd0f` | 653 |
The independently captured raw-mode C and JSON-mode C are byte-equal after
removing outer whitespace. The verifier checks both retained source digests,
artifact and engine equality before joining instruction origins. This is
corroboration between two captures, not proof of a shared internal IR revision.
A production interface should export these relationships together.
2. **Control and def-use answer a bounded local question.** For the header error,
the result contains 21 operations: the literal origin, STORE and 19 branch
dependencies. Two explicit incoming edges enter block 446. They correspond
to a missing low wrap bit and a failed divisibility check on the constructed
header value. The branch definitions stop at a memory load and a phi node.
Original machine instructions independently show `test $1` / `je` and the
reciprocal-division sequence / `jne` into the error block. The verifier
exhaustively checks all 65,536 ordinary 16-bit header values against both
the captured arithmetic sequence and modulo 31. No target code is executed.
This agrees with [zlib 1.2.13 source](https://github.com/madler/zlib/blob/v1.2.13/inflate.c#L633)
and [RFC 1950](https://www.rfc-editor.org/rfc/rfc1950.html#section-2.2).
It does not establish all entry-state preconditions or primed bit-buffer states.
3. **The same composition finds a SQLite call guard.** For the first malformed
MATCH string at `0x11dcd8`, five selected operations connect the literal to
`sqlite3Fts3ErrMsg` and the observed condition `EBX == 1`. The original bytes
have `cmp $1,%ebx` and `je` to the formatting path; the corresponding
[3.40.1 source](https://github.com/sqlite/sqlite/blob/version-3.40.1/ext/fts3/fts3_expr.c#L1057)
uses `rc == SQLITE_ERROR`, whose [code is 1](https://www.sqlite.org/rescode.html#error).
The slice stops at the return-code phi: it does not explain why a particular
input caused the parser error. Source-version agreement is corroboration,
not proof of an exact Debian source/build match.
4. **Addresses do not identify IR blocks.** SQLite blocks 20 and 22 both start at
`0x6523e`, yet have distinct identities and paths. Several operations also
share instruction origins. Do not use a block's printed address span as its
membership set, or pair phi inputs with CFG predecessors by printed order.
5. **Retained paging cannot recover producer omissions.** Ghidra `inflate` by
name is ambiguous between PLT and body. Its exact body is `0x10c1e0`, in a
different load space from Kuna's `0xc1e0`. The complete MCP dossier exceeded
the 10 MiB transport budget. The server correctly retained Evidence and
supplied continuation arguments; summary reads and local bundle export worked.
However, high p-code was already truncated to 3,000 operations, with 239 known
omitted edges and at least one omitted operation. The late error literal is
outside that prefix. Its absence remains **unknown**. Raising the response
limit or paging the retained prefix would not repair this missing coverage.
Kuna's raw dump contains 3,860 operations / 472 blocks for `inflate`, and
253 operations / 23 blocks for the SQLite function. All printed definition IDs
resolved in these captures. Nevertheless, 63 / 1 blocks respectively have
unrepresented outgoing relationships. In particular, the diagnostic format does
not enumerate switch successors or every layout fallthrough. Incoming edges are
local observations, not a complete control-dependence or feasible-path proof.
`--regions` exposes structuring output; it did not provide the missing typed CFG.
## Cost and limits of the comparison
The estimates below use tiktoken 0.14.0 / `o200k_base`, not bytes-to-token ratios.
Full C is native text; the selected answer is JSON including source identity,
raw-Evidence access, facts and unknown coverage. No hypothetical MCP wrapper is
counted. These are **representation costs**, not equivalent completed agent tasks.
| Target / question | Full C tokens | Full raw C + SSA tokens | Selected answer tokens |
| ------------------ | ------------: | ----------------------: | ---------------------: |
| zlib header error | 17,726 | 254,265 | 3,270 |
| SQLite MATCH error | 931 | 14,928 | 1,283 |
The three zlib answers together cost 5,485 tokens with one shared source envelope.
The SQLite counterexample matters: a small function is cheaper as complete C than
as this structured slice. A successful atomic API does not force tiny getters or
graph output for every question. A C window around zlib line 164, on the other
hand, still misses the earlier guard at lines 122–124; locating a line alone did
not complete the local explanation.
| Stage | Elapsed s | Family CPU s | Peak family MiB |
| ---------------------------------------------------------- | --------: | -----------: | --------------: |
| zlib native raw, first capture | 6.965 | 6.608 | 208.1 |
| zlib native raw, fresh repeat | 18.749 | 13.462 | 195.9 |
| SQLite native raw | 2.738 | 2.557 | 172.5 |
| zlib validation, index and two projections | 0.423 | 0.227 | 43.7 |
| SQLite validation, index and two projections | 0.322 | 0.095 | 24.9 |
| Ghidra exact selection, retained summary / bundle workflow | 49.170 | 48.536 | 1,466.6 |
The fresh zlib captures have identical raw digests, but different elapsed/CPU
costs; the cause is unisolated. These samples establish neither native-analysis
acceleration nor a cross-provider speed ranking. OS caches were not flushed.
The Ghidra row includes first/warm requests and lifecycle, and is not equivalent
to the one-command native rows. Its analyst question remained unfinished.
Within the final projection runs, parse/index took 130.20 ms / 7.96 ms for
zlib / SQLite. First query plus serialization took 1.10 ms / 0.64 ms; repeat
queries on the same immutable index took 0.29 ms / 0.09 ms. Native analysis did
not run during either projection. This is a useful reuse mechanism, not a claim
that the original native decompilation becomes that fast.
All resource observations use the existing one-CPU, 4 GiB hard limit, 3 GiB early
stop and 192-task owner. Peak memory is the kernel cgroup `memory.peak` for the
whole process family, **including charged file cache**, not RSS. Every captured
run had verified payload-exit cleanup and zero OOM kills. The complete 12-run
ledger includes the ambiguous-name failure, the transport failure, the region
probe, superseded projections and automated verification: 178.051 s wall and
168.094 s family CPU in total. Manual investigation, old discovery/xrefs, objdump,
tokenization and development checks are outside that ledger.
The actual Ghidra retained workflow used the existing complete input-only catalog:
206,954 estimated catalog tokens plus 39,620 captured request/response tokens,
including failures, lifecycle and export. Exported bundle contents stayed local.
These costs do not count as a successful solution or as comparable Kuna savings.
No model reasoning tokens, answer-quality score, controlled agent A/B trial or
end-to-end agent latency was measured.
## What to explore next
The useful candidate is a shared analysis snapshot with typed facts and relations,
then bounded question-specific projections. Stable identity must include artifact,
load space, profile and snapshot; value definitions and block membership need
native identities in addition to instruction origins. Complete control dependence
requires a CFG with coverage; alias effects and phi predecessor bindings need
explicit producer support.
The nearest existing surfaces are `trace_native_values` for analysis and native
facets of `inspect_analysis_view` for retained reads. Retained views should remain
read-only: new analysis belongs to the analysis owner. Seeded selection must happen
before Ghidra's producer limit. Kuna should ideally export a structured IR/CFG
subset from the native owner instead of making this diagnostic-text decoder a
production protocol. Do not add a separate MCP tool for each getter or a custom
query language on the basis of these two functions.
Complex-software superiority is still unproven. The next discriminating question
is why an input reaches a selected error: that needs phi predecessor bindings and
control dependencies beyond these local slices, with a completed-task comparison
against the ordinary C workflow. Keep the small-SQLite counterexample in that
evaluation rather than optimizing only the large-function case.
## Verification and retained evidence
Six distinct SSA module tests plus the existing nine fact/mapping tests passed.
Real static captures, exact mapping replay for three errors, bounded SQLite
composition, 65,536 arithmetic cases, original instruction checks and Ghidra
retained-Evidence recovery passed. The Ghidra late-literal query did not have
sufficient producer coverage. Formatter, targeted lint, syntax and whitespace
checks ran; no production runtime source was changed in this round.
Locally retained evidence is under `.cache/atomic-re-tools/`, including
`relations-*.{report.json,log}`, `relations-*-raw*.native.txt`, the exported Ghidra
bundle, `relations-*-query-v2.json`, `relations-verification-v2.json` and
`relations-summary-v2.json`. Raw captures are not committed. Their SHA-256 values:
- zlib raw: `b1d6bc61755041d2934e4ab173aca4e5fdf40f5bda01ca404458a163b129681b`.
- SQLite raw: `8a945a1549b855802958bd4d33f37b9aaf00781aa57059fe25cb740244a1cdde`.
- zlib JSON body/mappings: `21631d31b4b93aa75c200be71a71ffe9f4b209982a9c329a2120d9f7c7509ea3`.
After producing captures with the prefixes used in the scripts, replay and count:
```sh
node experiments/atomic-re-tools/verify-relations.mjs \
.cache/atomic-re-tools .cache/atomic-re-tools/relations-verification-v2.json
python experiments/atomic-re-tools/summarize-relations.py \
.cache/atomic-re-tools .cache/atomic-re-tools/relations-summary-v2.json
```
The second command requires tiktoken. The checked-in verifier intentionally binds
to these exact artifacts and oracle addresses; it is not a generic correctness
test for arbitrary zlib/SQLite versions. Capture Ghidra with
`capture-function.mjs ghidra <zlib-path> 0x10c1e0 <prefix>`, and SQLite with
`capture-kuna-structure.mjs <sqlite-path> 0x65180 raw <prefix>`. Caller-supplied
engine paths and the existing resource wrapper are required as in round one.
Capture prefixes and output paths must be fresh; exclusive writes preserve any
existing evidence or caller-owned file.
+112
View File
@@ -0,0 +1,112 @@
# What might count as an RE atom?
This is an experiment record, not a claim that a new universal instruction set
has been discovered. APIs already expose many relevant primitives. The open
question here is how to make their evidence, cost and composition useful to an
agent investigating a real program.
## Existing mechanisms and what they establish
- Ghidra has a program database, `ReferenceManager.getReferencesTo`, function-body
ownership through `FunctionManager.getFunctionContaining`, listing objects and
decompiler IR. Fine-grained **engine APIs already exist**. A poor MCP projection
can still discard their metadata or require excessive round trips.
Sources: [ReferenceManager](https://ghidra.re/ghidra_docs/api/ghidra/program/model/symbol/ReferenceManager.html),
[FunctionManager](https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/FunctionManager.html).
- Binary Ninja distinguishes code/data, type and variable references, including
IL-level references. This shows why one undifferentiated `xref` is insufficient
for some questions. It does not establish that an MCP interface using these
APIs will be token-efficient or that another provider can reproduce the facts.
Sources: [type/variable references](https://binary.ninja/2021/04/12/type-and-variable-xrefs.html),
[BinaryView API](https://dev-api.binary.ninja/binaryninja.binaryview-module.html).
- Kuna's Rust engine and SLEIGH compiler derive from Ghidra, while its standalone
loader/analyzer tier and CLI are separate components. Port lineage does not
imply identical observations or a shared live program database.
Source: [NOTICE](https://github.com/Noelo-Lab/kuna/blob/fe01d6294eb0d1053ce343c8943646df88e143fc/NOTICE).
- In **tested Kuna 1.833**, an `xrefs` command loads a program and builds an
`XrefIndex` for its query. A different CLI invocation does that again.
`strings` with attribution instead loads/indexes once and joins references over
each string's byte extent. That is already internal composition, even though
the CLI does not expose an arbitrary persistent query interface.
Sources: [xref producer](https://github.com/Noelo-Lab/kuna/blob/fe01d6294eb0d1053ce343c8943646df88e143fc/decompiler/crates/kuna-cli/src/xrefs.rs),
[string attribution](https://github.com/Noelo-Lab/kuna/blob/fe01d6294eb0d1053ce343c8943646df88e143fc/decompiler/crates/kuna-cli/src/strings.rs).
These are source-derived observations. They are not measurements of current
upstream main, a guarantee of decompilation correctness, or a novelty claim.
## Granularity has at least three independent dimensions
| Dimension | Example | Consequence |
| -------------- | ------------------------------------------------------ | --------------------------------------------------------------------- |
| Fact unit | One typed reference with both endpoints and provenance | Can be retained and joined without reading unrelated C text |
| Transport unit | A batch of references for several selectors | Amortizes headers and agent turns; need not change native work |
| Compute unit | One image load and reference walk | Can serve many fact queries; may cost much more than one returned row |
Calling all three "atomic tools" hides the main optimization opportunity. A tiny
getter can repeat a whole-program analysis. Conversely, one expensive shared
analysis can provide precise, small, independently addressable facts.
## Candidate semantic core
1. **Identified objects and located observations**: artifact, image/address-space,
byte span, decoded instruction, symbol, string candidate, recovered function,
variable and type. Keep width, encoding, storage and confidence where needed
to interpret a value. A recovered object is an analysis claim, not necessarily
a source-language object that existed before compilation.
2. **Relations and constraints**: maps-to, owns/contains, references, calls,
flows-to, defines/uses, argument binding, type/storage association and branch
guard. Keep source and destination, reference site, kind, supporting evidence
and unknown/partial coverage. Do not collapse a call site into just a callee.
3. **Evidence views**: retain the original observations and expose the requested
fields, range or neighborhood. A readable decompilation is an important view;
it does not replace a machine-grounded relation or prove original source.
Useful query operations resemble selection, projection, traversal, join and
aggregation. This is a design vocabulary, **not a proposal to add a SQL/graph
language or Code Mode**. Start with named, typed analyst questions; add more
general execution only if repeated real workflows demonstrate the need.
## Composition rules that prevent false conclusions
- Entity identity and an observation's identity differ. Artifact/profile/location
can identify a function candidate; `evidence_id` identifies one exact claim
about it. Keep disagreeing claims instead of deduplicating them by address.
- A session lease prevents using a handle after close/reopen. It is not a frozen
analysis revision. In particular, Kuna's whole-project and early single-function
reconstructions can differ under the same selected profile. Target admission
prevents a target switch; it does not establish a consistent analysis snapshot.
- A string spans bytes. A point-address xref and a reference anywhere inside that
object answer different questions. A data-pointer reference is not automatically
a runtime call, string consumption, or a reachable execution path.
- Omission, an unsupported operation, cancellation, an unowned reference and an
examined empty set are different outcomes. No empty set proves global absence
unless the producing analysis actually establishes that coverage.
- Field selection must not detach a value from the metadata that makes it
meaningful. For example, the first prototype keeps REA's complete `string`
metadata record as one selectable field, including its encoding status.
- Expensive producer work should be shared with explicit identity, lifetime,
invalidation and cancellation ownership. A client-side batch is insufficient
evidence that native work was shared.
## What complex-program effectiveness would require
Navigation is only the first tier. Finding a string and its users is weaker than
explaining why a branch rejects input or how data reaches a privileged operation.
The next tier needs definition/use, call-argument and memory-flow relationships,
with explicit analysis assumptions. The existing prototypes do **not** supply
those relationships, and slicing C text cannot manufacture them.
For every harder trial, specify the analyst question, independent answer/evidence
oracle, input artifact/profile, initial knowledge and allowed budget. Compare:
- whether the actual question was answered correctly;
- decisive evidence recovered and relevant evidence omitted;
- remaining unknowns and unsafe conclusions;
- all calls, follow-up reads, failures, latency, CPU, family memory and estimated
token traffic, including tool discovery and retained Evidence reads.
Source inspection can establish architectural opportunities. Scripted equivalent
workflows can establish transport and engine costs. Neither alone establishes
that a live agent reasons better on complex software; a controlled agent trial
is a separate validation step. Likewise, byte/token reductions alone do not
establish improved correctness or analyst productivity.
+216
View File
@@ -0,0 +1,216 @@
# First-round results
Continued in the [control-flow / SSA experiment](relations.md). The results below
remain the fixed first-round baseline, including its failed mapping composition.
This is a completed, bounded first round of exploration, including failures.
It does **not** establish that an agent solves complex RE tasks better than with
the existing tools. No PR, merge, target execution or executable mutation occurred;
Ghidra used its normal ephemeral analysis databases.
## Inputs and measurement
- REA base: `5b16f2431062eea6ffd245a9e05512790007cb13`; experiment commits retain
the runnable code and tests. Node 24.18.0 / npm 11.16.0.
- Kuna 1.833, executable SHA-256
`d858918b0766435cd96b999a77f2d5a31a0cc4f07256421c883b5b9dc4dd58e4`.
These results do not verify newer Kuna releases.
- Ghidra 12.1.4, Temurin JDK 21.0.12.1+1, Java heap 1 GiB.
- Installed zlib: `/usr/lib/x86_64-linux-gnu/libz.so.1.2.13`, SHA-256
`7e2a72b4c4b38c61e6962de6e3f4a5e9ae692e732c68deead10a7ce2135a7f68`.
- Installed SQLite: `/usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6`, SHA-256
`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`.
- Real-provider jobs used the existing scope runner: one CPU, affinity to one
allowed core, 4 GiB hard memory limit, early stop at 3 GiB. They ran serially.
Build/type/test gates used separate bounded scopes, up to two CPUs / 4 GiB.
- CPU is complete owned-cgroup CPU at payload exit. Memory is kernel
`memory.peak` for the process family, **including charged file cache**, not RSS.
The observer verified cleanup and resource counters while its scope was alive.
- Tokens below are **offline `o200k_base` estimates**, tiktoken 0.14.0, over actual
captured MCP payloads, including text and structured copies. They are not
model-billed tokens. Model reasoning, tool selection, host rendering, cache
billing and JSON-RPC framing were not measured.
- Each table row is **one sample**, not a median or controlled causal estimate.
OS caches were not flushed. Startup and native work varied substantially on
this shared host. No universal latency improvement is established.
## Experiment 1: transport granularity
Task: case-sensitive literal search (`incorrect` in zlib, `malformed` in SQLite),
select the first three provider-ordered string candidates, read at most four
exact-start reference sites per candidate, then resolve their containing
functions when advertised. Repeat in the same session to measure warm reads.
Unsupported Kuna function resolution remains unknown, rather than guessed.
The four variants use the same application/provider observations. Native query
planning was unchanged. `fine` reads individual facts; `batch` groups homogeneous
reads; `workflow` composes those same fact methods in one call.
These captures measure the **v1 fact response shape**, introduced at `fa8d6a7a2`.
Later selector/scope and transport-provenance clarifications are not retroactive
performance measurements; rerun the harness to measure their current payload.
| Provider / target | Variant | Calls per task | Query payload tokens, first / warm | First query task ms | Warm task ms | Session CPU s | Peak family MiB |
| ----------------- | -------- | -------------: | ---------------------------------: | ------------------: | -----------: | ------------: | --------------: |
| Ghidra / zlib | existing | 10 | 41,565 / 41,565 | 34,926 | 429 | 50.00 | 1,444 |
| Ghidra / zlib | fine | 10 | 23,483 / 23,483 | 34,260 | 120 | 47.37 | 1,376 |
| Ghidra / zlib | batch | 3 | 20,114 / 20,114 | 75,202 | 218 | 52.63 | 1,360 |
| Ghidra / zlib | workflow | 1 | 19,824 / 19,824 | 75,491 | 198 | 51.76 | 1,276 |
| Kuna / zlib | existing | 4 | 5,638 / 5,638 | 4,079 | 44 | 16.12 | 905 |
| Kuna / zlib | fine | 5 | 4,702 / 4,702 | 2,857 | 51 | 15.61 | 927 |
| Kuna / zlib | batch | 3 | 3,774 / 3,774 | 2,843 | 59 | 15.14 | 911 |
| Kuna / zlib | workflow | 1 | 3,509 / 3,509 | 1,376 | 11 | 14.34 | 922 |
| Kuna / SQLite | existing | 4 | 6,039 / 6,039 | 5,874 | 24 | 19.15 | 994 |
| Kuna / SQLite | fine | 5 | 4,753 / 4,753 | 13,058 | 47 | 20.40 | 989 |
| Kuna / SQLite | batch | 3 | 3,829 / 3,829 | 25,868 | 217 | 25.85 | 1,022 |
| Kuna / SQLite | workflow | 1 | 3,560 / 3,560 | 36,231 | 39 | 58.19 | 1,004 |
Session CPU/memory include MCP startup, catalog/status discovery, both tasks and
close/cleanup. Query timers exclude the earlier `open_binary`; Ghidra lazily
performs import/preparation during its first query. Whole-session elapsed times
for existing/workflow were 50.67/104.15 s (Ghidra zlib), 32.46/16.63 s (Kuna zlib),
and 19.46/159.81 s (Kuna SQLite). The SQLite workflow's `open_binary` itself took
64.98 s versus 7.31 s in the existing-tools sample, before any experimental fact
query. Cause is unresolved; neither host contention nor adapter overhead has
been established as the explanation. Do not interpret these samples as a
controlled speedup or slowdown attributable to atomization.
**Failure retained:** the first Ghidra batch attempt hit its 120 s provider
request deadline during cold preparation. Its log ended in `ReaGhidraNoReturnFix`
after reporting successful auto-analysis. The attempt cost 141.45 s elapsed,
106.19 CPU s and 1,211 MiB peak; cleanup was verified and OOM kills were zero.
The successful batch row excludes that failed attempt; any retry-cost comparison
must add it. All twelve successful rows agreed on the selected strings,
reference sites and function-resolution outcomes within each provider/target.
That is adapter equivalence, not an independent proof of native discovery.
### Fixed context costs
The existing catalog had 207 tools; the prototype added five, retaining every
existing tool. Actual default catalogs included output schemas: about 842k /
843k estimated tokens. Removing only output schemas gives about 207k / 208k.
This input-only projection is **not** an assertion about any particular model
host: hosts can omit output schemas, deduplicate content or cache declarations.
For a session containing both completed tasks plus lifecycle/status traffic,
the input-only-catalog estimate changed as follows:
| Provider / target | Existing tools | Workflow |
| ----------------- | -------------: | -------: |
| Ghidra / zlib | 319,907 | 277,458 |
| Kuna / zlib | 245,476 | 242,249 |
| Kuna / SQLite | 245,803 | 241,870 |
Thus a large reduction in query payload need not produce a large whole-task
context reduction. Catalog/discovery and repeated envelopes deserve separate
attention. This experiment has not yet tested a client that loads declarations
on demand or an explicitly smaller, separately declared service surface.
## Experiment 2: compute sharing in native Kuna
Both plans first obtain a complete reference-free scan, validated by REA's
existing string-fact owner. Then select three matching strings. The point plan
runs `xrefs --to` separately; the inventory plan runs `strings` with attribution
once, letting Kuna share its reference index. Full source JSON is retained
locally, while only selected facts need be exposed to a caller.
| Target | Plan | Native commands | Elapsed s | CPU s | Peak family MiB |
| ------ | ------------------ | --------------: | --------: | ----: | --------------: |
| zlib | per address | 4 | 3.846 | 1.796 | 110.4 |
| zlib | shared attribution | 2 | 2.838 | 1.341 | 114.4 |
| SQLite | per address | 4 | 13.221 | 6.497 | 196.1 |
| SQLite | shared attribution | 2 | 6.261 | 3.028 | 196.6 |
For these selected samples, both plans found one owner per string. Their scopes
are nevertheless **different**: point references cover the first address;
attribution covers any byte in the string extent. Attribution also does not
return individual sites/kinds. These figures establish costs of the two real
plans, not a universally equivalent-query speedup. They support investigating
producer-level work sharing, with query selectivity and coverage kept explicit.
This is a native-boundary experiment, not an MCP/model token measurement.
## Experiment 3: the point/object distinction changes correctness
`interior-reference.c` is compiled with `gcc -O2 -fno-pie -no-pie` and never run.
Its `give_suffix` returns a pointer five bytes into `marker`. In this build,
`nm` reports `marker=0x402020`, `give_suffix=0x401120`; `objdump` independently
shows `mov $0x402025,%eax` followed by `ret` in `give_suffix`.
- Exact-start xref query: zero references, no owner.
- Extent attribution: one reference, owner `give_suffix` at `0x401120`.
`verify-interior.py` checks the compiled symbols, input and retained-source
digests, and both native results. This is a genuine counterexample to treating
"no xref to the start address" as "the string has no users". Neither tool was
wrong about its own scope. The compositional representation must carry the
object span and reference offset/scope instead of silently equating the questions.
## Experiment 4: existing relations do not always bridge into code
On real zlib, the three selected string references belong to `inflate` at
`0xc1e0`; their reference instructions are at `0xded7`, `0xdc6a` and `0xcd05`.
The native single-function JSON contains 52,431 characters / 1,620 code lines,
1,167 mapping records and 1,568 mapped instruction addresses. **None of those
three reference instructions occurs in the printer's line mappings.**
The exact-address composition therefore returns **unknown**, not a guessed
nearby line. The mapped-code run cost 8.387 s / 8.091 CPU s / 202.0 MiB, including
discovery/references and decompilation. It did not finish the analyst question
"which condition triggers the error?" and must not count as a successful
context-saving answer. Single-function work uses REA's owned command deadline
and the native `--max-fn-memory` option shown by the tested CLI's actual help.
The retained C text does contain the error literal at line 164, but a small
fixed window there omits the earlier check around lines 122–124. Literal search
and a larger manual read can help, but they are **reconstructed-text evidence**,
not the missing machine-to-source relation. This is not proof that Kuna's line
mapping is buggy: the contract does not promise every contributing instruction
survives transformation into an emitted line.
The next useful relations to test are instruction/block ownership, control-flow
guards and definition/use or contributing-origin edges across transformations.
For a stronger zlib question, an independent expected mechanism is the FCHECK
constraint in [RFC 1950 §2.2](https://www.rfc-editor.org/rfc/rfc1950.html#section-2.2):
the network-order CMF/FLG value must be divisible by 31. Testing an agent's
explanation against that mechanism would be a separate trial; this experiment
does not claim to have completed that trial.
## Validation
- All **191 Kuna regression cases in 20 files passed with one worker**; the
owning reference parser's 18 cases include typed endpoints, distinct kinds,
unknown ownership, alias exclusion and late malformed rows. A prior two-worker
CPU-limited run had three timing-sensitive failures in existing project/export
tests. Those tests were not edited; the isolated cause is not established.
- Nine experiment boundary cases passed, including stale artifact/profile/lease,
partial batches, duplicate point lookups, paging, unsupported ownership,
unmapped/unsafe addresses and invalid line locations.
- Typecheck passed with one checker. Changed experiment files passed formatter
and lint; the existing graph parser's complexity warning remains unrelated.
- A real Kuna MCP run with the actual output-schema **off** profile passed
paging, duplicate-selector, negative/unknown, retained-Evidence, foreign-context
and expired-lease checks. Its live catalog had 212 tools and zero output
schemas. Verification runs are excluded from the performance table.
- The v2 facade was also exercised through a real Kuna MCP client: edges carry
explicit point scope and location endpoints, caller selectors stay separate,
and successful/error results retain the canonical untrusted-target marker.
The same six additional verification groups passed. This v2 verification is
not relabelled as a new performance comparison.
- All real-provider measurements reported verified cleanup and zero OOM kills.
No default parallel-suite stability, live-model reasoning improvement, general
high-address code mapping, overlay support or newer Kuna compatibility is claimed.
## What this round supports
- A coherent fact includes the metadata needed to interpret it. One-field
getters are not necessarily useful atoms.
- Fact, transport and compute units are different. Batches/workflows can share
one fact implementation while the producer independently shares analysis.
- Identity and evidence are necessary, but scope and uncertainty determine
whether joins answer the analyst's actual question.
- Complex-program effectiveness remains **unproven**. The experiments found a
real coverage distinction and a real composition gap, not a universal RE API.
Raw captures, projects, binaries and counters remain ignored under
`.cache/atomic-re-tools/`. `summarize.py` checks selected-answer equivalence,
payload-exit cleanup and OOM counters, and includes failed runs separately.
+40
View File
@@ -0,0 +1,40 @@
"""Run comparable real-provider tasks serially using REA's existing scope owner."""
import argparse
import json
from pathlib import Path
import subprocess
import sys
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider", choices=["kuna", "ghidra"], required=True)
parser.add_argument("--binary", required=True)
parser.add_argument("--pattern", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--variants", nargs="+", choices=["baseline", "fine", "batch", "workflow"],
default=["baseline", "fine", "batch", "workflow"])
options = parser.parse_args()
root = Path(options.output).resolve()
root.mkdir(parents=True, exist_ok=True)
completed = []
for variant in options.variants:
prefix = root / (options.provider + "-" + variant)
command = [sys.executable, "experiments/analysis-efficiency/resource-run.py", "--observe-payload-exit", "--",
"node", "experiments/atomic-re-tools/benchmark.mjs", options.provider, variant,
options.binary, options.pattern, str(prefix)]
with Path(str(prefix) + ".log").open("w") as output:
result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, check=False)
if result.returncode:
raise SystemExit(f"{variant} failed ({result.returncode}); retained {prefix}.log")
report = json.loads(Path(str(prefix) + ".report.json").read_text())
if report["failures"] or report["verification_only"]:
raise SystemExit("Not a successful comparable performance sample: " + str(prefix))
completed.append(str(prefix))
(root / (options.provider + "-completed.json")).write_text(json.dumps(completed, indent=2) + "\n")
print(json.dumps({"completed": variant, "provider": options.provider,
"query_ms": [round(row["elapsed_ms"], 2) for row in report["outcomes"]]}), flush=True)
if __name__ == "__main__":
main()
+202
View File
@@ -0,0 +1,202 @@
// Experimental tools are added to the complete production catalog, never shipped.
import { serveStdio } from "@modelcontextprotocol/server/stdio";
import { z } from "zod";
import { run } from "../../dist/main.js";
import { createServer } from "../../dist/server/createServer.js";
import { createEvidence } from "../../dist/domain/evidence.js";
import { projectAnalysisError } from "../../dist/domain/analysisErrorProjection.js";
import { toolContract } from "../../dist/contracts/toolContracts.js";
import { TARGET_CONTENT_RESULT_META } from "../../dist/server/toolResultEncoding.js";
import { expandProviderCapabilities } from "../../dist/contracts/toolOutputSchemaPrimitives.js";
import { FactQueries, contextFor } from "./facts.mjs";
const contextSchema = z.strictObject({ ref: z.string(), lease: z.string() });
const location = z.strictObject({
kind: z.literal("location"),
address: z.string().min(1),
});
const stringId = z.strictObject({
kind: z.literal("string"),
address: z.string().min(1),
encoding: z.string().nullable(),
});
const paging = {
offset: z.number().int().nonnegative().default(0),
limit: z.number().int().positive().default(20),
};
const stringsInput = z.strictObject({
query: toolContract("search_strings").inputSchema,
...paging,
fields: z.array(z.enum(["value", "string"])).default(["value"]),
});
const contracts = [
[
"experiment_find_strings",
"Find string facts with scoped identities, selected fields, paging and retained source Evidence. Experimental; provider discovery coverage remains unknown.",
stringsInput,
"findStrings",
],
[
"experiment_incoming_refs",
"Read incoming reference facts for a batch of string identities from this session. Preserve per-target failures and unknown reference kinds. Reuse the context ref and lease returned by find_strings.",
z.strictObject({
context: contextSchema,
targets: z.array(stringId),
...paging,
}),
"incomingReferences",
],
[
"experiment_resolve_functions",
"Resolve a batch of instruction locations to provider-analyzed containing functions. Unsupported or failed resolution stays unknown; never guess the nearest symbol.",
z.strictObject({ context: contextSchema, locations: z.array(location) }),
"resolveFunctions",
],
[
"experiment_trace_string_uses",
"Compose the same string, incoming reference and containing-function facts in one call. Page limits and partial outcomes remain explicit.",
stringsInput.extend({
reference_limit: z.number().int().positive().default(20),
include_functions: z.boolean().default(true),
}),
"traceStringUses",
],
];
const encode = (value) => ({
_meta: TARGET_CONTENT_RESULT_META,
content: [{ type: "text", text: JSON.stringify(value) }],
structuredContent: value,
...(value.status === "error" ? { isError: true } : {}),
});
const createFacts = (session, analysis, signal) => {
const status = session.status();
const context = contextFor(
session.activeTarget().sha256,
session.analysisProfile("search_strings"),
status.analysis_run.run_id,
);
const capabilities = expandProviderCapabilities(status.capabilities);
const facts = new FactQueries({
supports: (operation) =>
capabilities.some(
(capability) =>
capability.operation === operation && capability.available,
),
context: () => context,
observe: async (operation, parameters) => {
const outcome = await analysis.execute(
operation,
toolContract(operation).inputSchema.parse(parameters),
{ signal: signal },
);
if (!outcome.ok)
return {
ok: false,
error: projectAnalysisError(outcome.error),
};
const execution = outcome.value;
const evidence = createEvidence(
execution.subject ?? session.activeTarget(),
execution.provider,
{
operation,
parameters,
result: execution.result,
analysisProfile: execution.analysisProfile,
rawResult: execution.rawResult,
limitations: execution.limitations,
relationshipCoverage: execution.relationshipCoverage,
locations: execution.locations,
},
);
const recorded = session.recordEvidence(evidence);
return recorded.ok
? { ok: true, value: evidence }
: {
ok: false,
error: projectAnalysisError(recorded.error),
};
},
});
return facts;
};
await run({
env: process.env,
serve: serveStdio,
writeStderr: (value) => process.stderr.write(value),
setExitCode: (value) => {
process.exitCode = value;
},
registerShutdown: (handler) => {
const registrations = [
[process, "SIGINT"],
[process, "SIGTERM"],
[process.stdin, "end"],
[process.stdin, "close"],
];
for (const [emitter, event] of registrations) emitter.once(event, handler);
return () => {
for (const [emitter, event] of registrations) emitter.off(event, handler);
};
},
createServer(source, options) {
const server = createServer(source, options);
const session = source.session;
for (const [name, description, inputSchema, method] of contracts) {
server.registerTool(
name,
{ description, inputSchema, annotations: { readOnlyHint: true } },
async (input, request) => {
const admitted = await session.withAdmittedAnalysis(
name,
request.mcpReq.signal,
async (analysis) => {
const facts = createFacts(
session,
analysis,
request.mcpReq.signal,
);
return facts[method](inputSchema.parse(input));
},
);
return encode(
admitted.ok
? admitted.value
: {
status: "error",
error: projectAnalysisError(admitted.error),
},
);
},
);
}
server.registerTool(
"experiment_read_evidence",
{
description:
"Read complete retained source Evidence explicitly by evidence_id while the existing session record owner retains it. Closing the binary expires these records.",
inputSchema: z.strictObject({ evidence_id: z.string().min(1) }),
annotations: { readOnlyHint: true },
},
(input) => {
const evidence = session.evidenceById(input.evidence_id);
return encode(
evidence === undefined
? {
status: "error",
error: {
code: "not_found",
message: "Evidence is not retained in this server.",
},
}
: { status: "ok", evidence },
);
},
);
return server;
},
});
@@ -0,0 +1,94 @@
"""Representation estimates and a complete ledger of this round's captured runs.
Run with the tokenizer environment from the first round. This does not measure
model billing, model reasoning, or equivalent completed agent tasks.
"""
import importlib.metadata
import json
from pathlib import Path
import sys
import tiktoken
def load(path):
return json.loads(path.read_text())
def resource(path):
records = [json.loads(line) for line in path.read_text().splitlines()
if line.startswith('{"resource_scope":')]
assert len(records) == 1, path
record = records[0]
observation = record['payload_resource_observation']
assert record['payload_resource_observation_verified'] is True
assert observation['payload_cleanup_verified'] is True
return {
'log': str(path), 'elapsed_s': record['elapsed_seconds'],
'cpu_s': observation['cpu_stat']['usage_usec'] / 1_000_000,
'family_peak_mib_including_file_cache': observation['memory_peak_bytes'] / 2**20,
'exit_code': observation['payload_exit_code'],
'oom_kills': observation['resource_events']['memory.events']['oom_kill'],
}
def main():
root, output = map(Path, sys.argv[1:])
encoder = tiktoken.get_encoding('o200k_base')
def tokens(value):
return len(encoder.encode(value, disallowed_special=()))
def encoded(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
rows = []
for name, prefix, query_file in [
('zlib', 'relations-kuna-zlib-raw-v2', 'relations-zlib-query-v2.json'),
('sqlite', 'relations-kuna-sqlite-raw', 'relations-sqlite-query-v2.json'),
]:
capture = load(root / (prefix + '.report.json'))
query = load(root / query_file)
raw = Path(capture['calls'][0]['file']).read_text()
code = raw.split('\n0\nBasic Block', 1)[0].strip()
answers = query['runs'][0]['answers']
rows.append({
'target': name, 'artifact_sha256': capture['target_sha256'],
'full_code_tokens': tokens(code), 'full_code_bytes': len(code.encode()),
'full_native_dump_tokens': tokens(raw),
'selected_answer_tokens_with_source': [tokens(encoded({'source': query['source'], 'answer': a})) for a in answers],
'batch_answer_tokens_with_source': tokens(encoded({'source': query['source'], 'answers': answers})),
'parse_index_ms': query['parse_index_ms'],
'projection_runs': [{k: v for k, v in r.items() if k != 'answers'} for r in query['runs']],
'snapshot_coverage': query['coverage'],
})
ghidra = root / 'relations-ghidra-zlib-retained'
wire = load(ghidra.with_suffix('.wire.json'))
report = load(ghidra.with_suffix('.report.json'))
resources = [resource(p) for p in sorted(root.glob('relations-*.log'))]
summary = {
'encoding': 'o200k_base', 'tiktoken': importlib.metadata.version('tiktoken'),
'representation_estimates': rows,
'ghidra_retained_workflow': {
'wire_tokens_including_errors_and_exports': tokens(encoded(wire)),
'input_only_catalog_tokens': tokens(encoded(load(ghidra.with_suffix('.catalog.json')))),
'calls': report['calls'], 'analyst_task_completed': False,
'reason': 'Late literal not covered by the producer-truncated high-p-code prefix.',
},
'run_ledger': resources,
'scope': [
'Raw captures are fresh native analyses; OS caches were not flushed.',
'Reused-index runs reuse parsed facts, not a warm native analysis.',
'Token counts cover the stated representations. The native probe has no new MCP catalog.',
'No controlled agent A/B task, model reasoning tokens or model latency measured.',
'Discovery, old xrefs, manual oracle construction, objdump and tokenizer CPU are outside the native/projection runs.',
'The ledger retains failed and superseded probes rather than excluding their costs.',
],
}
with output.open('x') as destination:
destination.write(json.dumps(summary, indent=2) + '\n')
print(json.dumps({'output': str(output), 'targets': len(rows), 'runs': len(resources)}))
if __name__ == '__main__':
main()
+84
View File
@@ -0,0 +1,84 @@
"""Compare completed equivalent tasks; token counts are offline estimates only."""
import argparse
import importlib.metadata
import json
from pathlib import Path
import tiktoken
def main():
parser = argparse.ArgumentParser()
parser.add_argument("roots", nargs="+")
options = parser.parse_args()
encoding = tiktoken.get_encoding("o200k_base")
def tokens(value):
return len(encoding.encode(json.dumps(value, ensure_ascii=False, separators=(",", ":")), disallowed_special=()))
results = []
failed_runs = []
equivalence = {}
for root in map(Path, options.roots):
for report_path in sorted(root.glob("*.report.json")):
report = json.loads(report_path.read_text())
if report["verification_only"]:
continue
prefix = str(report_path).removesuffix(".report.json")
catalog = json.loads(Path(prefix + ".catalog.json").read_text())
wire = json.loads(Path(prefix + ".wire.json").read_text())
resources = [json.loads(line) for line in Path(prefix + ".log").read_text().splitlines()
if line.startswith('{"resource_scope":')][-1]
assert resources["payload_resource_observation_verified"], prefix
counters = resources["payload_resource_observation"]
assert counters["payload_cleanup_verified"], prefix
assert counters["resource_events"]["memory.events"]["oom_kill"] == 0, prefix
if report["failures"]:
failed_runs.append({"provider": report["provider"], "variant": report["variant"], "target": report["binary"],
"elapsed_seconds": resources["elapsed_seconds"], "cpu_seconds": round(counters["cpu_stat"]["usage_usec"] / 1e6, 3),
"peak_process_family_cgroup_mib": round(counters["memory_peak_bytes"] / 1024 ** 2, 1),
"failure_count": len(report["failures"]), "retained_prefix": prefix,
"cleanup_verified": True, "oom_kills": 0})
continue
assert counters["payload_exit_code"] == 0, prefix
key = (report["provider"], report["target_sha256"], report["pattern"])
outcome = [{name: row[name] for name in ["strings", "references", "functions"]} for row in report["outcomes"]]
if key in equivalence:
assert outcome == equivalence[key], "Equivalent-task answers differ: " + prefix
else:
equivalence[key] = outcome
phase_traffic = {phase: [{"request": row["request"], "response": row["response"]} for row in wire if row["phase"] == phase]
for phase in ["cold", "warm", "lifecycle"]}
used = {row["request"]["name"] for row in wire}
input_only_catalog = {**catalog, "tools": [{key: value for key, value in tool.items() if key != "outputSchema"} for tool in catalog["tools"]]}
task_traffic = [row for row in wire if row["phase"] in ["cold", "warm", "lifecycle"]]
rows = {phase: {"calls": len(traffic), "estimated_tokens": tokens(traffic)} for phase, traffic in phase_traffic.items()}
results.append({
"provider": report["provider"], "variant": report["variant"], "target_sha256": report["target_sha256"],
"pattern": report["pattern"], "catalog_tools": len(catalog["tools"]), "catalog_estimated_tokens": tokens(catalog),
"observed_output_schema_count": sum("outputSchema" in tool for tool in catalog["tools"]),
"input_only_catalog_estimated_tokens": tokens(input_only_catalog),
"phases": rows,
"query_elapsed_ms": {row["phase"]: round(row["elapsed_ms"], 2) for row in report["outcomes"]},
"session_elapsed_seconds_including_two_tasks": resources["elapsed_seconds"],
"session_cpu_seconds_including_two_tasks": round(counters["cpu_stat"]["usage_usec"] / 1e6, 3),
"peak_process_family_cgroup_mib": round(counters["memory_peak_bytes"] / 1024 ** 2, 1),
"whole_session_estimated_tokens": tokens(catalog) + tokens([{"request": row["request"], "response": row["response"]} for row in task_traffic]),
"whole_session_with_input_only_catalog_estimated_tokens": tokens(input_only_catalog) + tokens([{"request": row["request"], "response": row["response"]} for row in task_traffic]),
"hypothetical_selected_declarations_estimated_tokens": tokens([tool for tool in catalog["tools"] if tool["name"] in used]),
"answer": {"strings": len(outcome[0]["strings"]), "reference_sites": sum(len(row["items"]) for row in outcome[0]["references"]), "locations": len(outcome[0]["functions"])},
"cleanup_verified": True, "oom_kills": 0,
})
summary = {
"token_method": {"encoding": "o200k_base", "tiktoken_version": importlib.metadata.version("tiktoken"),
"estimated_not_billed": True, "scope": "Captured MCP tool catalog and request/response payloads, including text and structured copies; excludes JSON-RPC framing, model reasoning, host prompts and model tool selection."},
"memory_method": "Kernel memory.peak for the complete owned process-family cgroup, including charged file cache; not summed RSS. Payload-exit counters verified while the observer remains alive.",
"workload": "One cold and one warm selected navigation task per fresh session, plus startup, discovery/status, close and cleanup. Additional --verify runs excluded.",
"equivalence": "All included variants agree on selected strings, reference sites and per-location function resolutions within each provider/target/query. This is not cross-provider truth or a live-agent reasoning evaluation.",
"results": results,
"failed_runs": failed_runs,
}
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
main()
@@ -0,0 +1,26 @@
"""Independent compiled-symbol oracle for the interior-only reference experiment."""
import hashlib
import json
from pathlib import Path
import subprocess
import sys
binary, point_report, extent_report = map(Path, sys.argv[1:])
point, extent = [json.loads(path.read_text()) for path in [point_report, extent_report]]
symbols = {}
for line in subprocess.check_output(["nm", "--defined-only", "--format=posix", str(binary)], text=True).splitlines():
fields = line.split()
if fields[0] in ["marker", "give_suffix"]:
symbols[fields[0]] = "0x" + format(int(fields[2], 16), "x")
assert set(symbols) == {"marker", "give_suffix"}
for report in [point, extent]:
assert not report["failures"]
assert report["facts"]["artifact_sha256"] == hashlib.sha256(binary.read_bytes()).hexdigest()
for call in report["calls"]:
assert hashlib.sha256(Path(call["file"]).read_bytes()).hexdigest() == call["sha256"]
assert report["facts"]["results"][0]["string"]["address"] == symbols["marker"]
assert point["facts"]["results"][0]["xrefs_count"] == 0
assert point["facts"]["results"][0]["functions"] == []
assert extent["facts"]["results"][0]["xrefs_count"] == 1
assert [fn["address"] for fn in extent["facts"]["results"][0]["functions"]] == [symbols["give_suffix"]]
print(json.dumps({"verified": True, "point_references": 0, "extent_references": 1, "symbols": symbols}))
@@ -0,0 +1,163 @@
// Replay real captures against independent machine-byte and arithmetic checks.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { mappedCode } from "./mapped-code.mjs";
import { parseKunaSsaDump } from "./kuna-ssa.mjs";
import { queryLiteralUses } from "./query-ssa.mjs";
const [root, output] = process.argv.slice(2);
assert.ok(root && output);
const json = async (name) =>
JSON.parse(await readFile(join(root, name), "utf8"));
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
const zlib = await json("relations-zlib-query-v2.json");
const sqlite = await json("relations-sqlite-query-v2.json");
assert.deepEqual(zlib.runs[0].answers, zlib.runs[1].answers);
assert.deepEqual(sqlite.runs[0].answers, sqlite.runs[1].answers);
for (const query of [zlib, sqlite]) {
const source = await readFile(query.source.raw.file, "utf8");
assert.equal(sha256(source), query.source.raw.sha256);
const snapshot = parseKunaSsaDump(source);
for (const answer of query.runs[0].answers)
assert.deepEqual(queryLiteralUses(snapshot, answer.literal), answer);
}
const old = await json("native-zlib-code-fixed.report.json");
assert.equal(zlib.source.artifact_sha256, old.facts.artifact_sha256);
assert.deepEqual(zlib.source.engine, old.facts.engine);
const bodySource = old.calls.find((call) => call.arguments[0] === "decompile");
const body = await readFile(bodySource.file, "utf8");
assert.equal(sha256(body), bodySource.sha256);
const bodyDocument = JSON.parse(body);
const raw = await readFile(zlib.source.raw.file, "utf8");
assert.equal(sha256(raw), zlib.source.raw.sha256);
assert.equal(
raw.split("\n0\nBasic Block", 1)[0].trim(),
bodyDocument.functions[0].code.trim(),
"Cross-surface emitted C differs",
);
const native = async () => ({
input: bodyDocument.binary,
stdout: { text: body },
});
const mappings = [];
for (const [i, expectedLine] of [164, 551, 653].entries()) {
const answer = zlib.runs[0].answers[i];
const original = old.facts.results[i];
assert.equal(answer.literal, original.string.address);
const failed = await mappedCode(native, original);
assert.equal(failed.status, "unknown");
const sites = answer.direct_uses.map((id) => ({
from: answer.operations.find((op) => op.id === id).address,
}));
const mapped = await mappedCode(native, {
functions: original.functions,
sites,
});
assert.equal(mapped.status, "ok");
assert.deepEqual(mapped.exact_mapped_lines, [expectedLine]);
mappings.push({
literal: answer.literal,
original_sites: original.sites.map((s) => s.from),
semantic_sites: sites.map((s) => s.from),
line: expectedLine,
});
}
const header = zlib.runs[0].answers[0];
assert.deepEqual(
header.control_edges.map((e) => [e.from, e.to, e.polarity]),
[
[182, 446, "false"],
[183, 446, "false"],
],
);
assert.equal(header.operations.length, 21);
assert.equal(header.coverage.dependencies.boundaries.length, 2);
assert.ok(header.operations.some((op) => op.statement.includes("& #0x1:4")));
assert.ok(header.operations.some((op) => op.statement.includes("/ #0x1f")));
const match = sqlite.runs[0].answers[0];
assert.equal(
match.operations.find((op) => op.id === match.direct_uses[0]).kind,
"call",
);
assert.ok(match.operations.some((op) => op.statement.includes("!= #0x1:4")));
assert.deepEqual(
match.control_edges.map((e) => e.polarity),
["false"],
);
const assemblyProofs = [];
for (const [file, patterns] of [
[
"relations-zlib-guard.objdump.txt",
[/d9ec:.*je\s+ded7/u, /da32:.*jne\s+ded7/u, /0x842108421084211/u],
],
[
"relations-sqlite-error.objdump.txt",
[
/65239:.*cmp\s+\$0x1,%ebx/u,
/6523c:.*je\s+65281/u,
/65284:.*lea.*11dcd8/u,
],
],
]) {
const text = await readFile(join(root, file), "utf8");
for (const pattern of patterns) assert.match(text, pattern);
assemblyProofs.push({ file, sha256: sha256(text) });
}
// Exhaustive ordinary 16-bit header domain. This executes arithmetic, not targets.
for (let n = 0n; n < 65536n; n++) {
const high = (n * 0x842108421084211n) >> 64n;
const machineQ = (((n - high) >> 1n) + high) >> 4n;
const ssaQ = (((n - n / 31n) >> 1n) + n / 31n) >> 4n;
assert.equal(machineQ, n / 31n);
assert.equal(ssaQ, machineQ);
assert.equal(n !== machineQ * 31n, n % 31n !== 0n);
}
const ghidra = await json("relations-ghidra-zlib-retained.bundle.json");
const flow = ghidra.records.find((r) => r.normalized_result?.native_value_flow)
?.normalized_result.native_value_flow;
assert.equal(flow.truncated, true);
assert.equal(flow.operations.length, 3000);
assert.equal(flow.known_omitted_edges, 239);
assert.ok(
!flow.operations.some((op) =>
op.inputs.some((v) => v.constant_hex === "118fc4"),
),
);
await writeFile(
output,
JSON.stringify(
{
mappings,
zlib_header_selected_operations: header.operations.length,
sqlite_selected_operations: match.operations.length,
assemblyProofs,
arithmetic_oracle: {
tested_values: 65536,
domain: "ordinary 16-bit header value",
target_executed: false,
},
ghidra: {
retained_operations: 3000,
known_omitted_edges: 239,
late_literal_coverage: "unknown due to truncation",
},
limitations: [
"Printed-SSA observations are derived, not runtime behavior.",
"Entry state, complete control dependencies, phi predecessor and memory alias bindings remain unknown.",
"Source-version agreement does not prove an exact Debian source/build match.",
],
},
null,
2,
),
{ flag: "wx" },
);
console.log(
JSON.stringify({
output,
mapped_errors: mappings.length,
arithmetic_values: 65536,
}),
);
+1
View File
@@ -10,6 +10,7 @@
"scripts/verify/fridaBindingsCompatibility.ts",
"scripts/verify/termux/browser.mjs",
"scripts/verify-real-ghidra-noreturn.mjs",
"scripts/run-exact-tests.mjs",
"tests/processOwnershipRunner.ts"
],
"project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"],
+4 -11
View File
@@ -111,14 +111,9 @@ automatically. Alternatively, build it using the Linux/MinGW lane above.
- Cleanup walks locked directory objects, removes files by handle, and unlinks
reparse entries without traversing their targets. Failed removal retains the
root and ancestor leases for retry; a partially removed runtime accepts only
cleanup, and its registry entry remains until removal succeeds. The runtime
owner retries only native close failures with Win32 sharing/lock codes 32/33,
for at most ten attempts with a 100 ms delay between attempts. Other native
errors are returned immediately; exhausting the retry budget preserves the
runtime owner for a later explicit close. These cleanup retries do not use an
analysis cancellation signal and schedule up to 900 ms of backoff, in
addition to native cleanup time. The owned root handle includes attribute-write access so cleanup can clear a
read-only marker, matching the child cleanup handles. Normal close, cancellation,
cleanup, and its registry entry remains until removal succeeds. The owned root
handle includes attribute-write access so cleanup can clear a read-only marker,
matching the child cleanup handles. Normal close, cancellation,
startup failure, and timeout release provider resources. Abrupt owner death
kills the owned job. The Android provider additionally uses a detached helper
that owns its private runtime directory and cleans it when the owner pipe
@@ -138,9 +133,7 @@ text do not establish a safe retry. Analysis errors propagate the producer's
retry disposition through JavaScript, source-map and WABT workflows; transient
failures report `category: unavailable` and retain the original reason.
Incomplete cleanup blocks an operation retry until its owned resources settle.
The close-stage retry is separate from allocation and analysis retries: it
repeats only the existing owned cleanup operation, and does not retry provider
work or create another runtime.
REA adds no automatic retry loop or new analysis cache for this behavior.
Ordinary drive-absolute paths accept backslashes, forward slashes, or mixed
separators. File identity retains the original requested spelling. Separator
-2
View File
@@ -214,7 +214,6 @@
"verify:javascript:digests": "npm run build:cached && node scripts/verify/javascript/digests.mjs",
"verify:javascript:output": "npm run build:cached && node scripts/verify/javascript/output.mjs",
"verify:javascript:module-references": "npm run build:cached && node scripts/verify/javascript/module-references.mjs",
"verify:javascript:native-handoffs": "npm run build:cached && node scripts/verify/javascript/native-handoffs.mjs",
"verify:application-workflows": "npm run build:cached && node scripts/verify/javascript/application-workflows.mjs",
"verify:managed": "npm run build:cached && node scripts/verify/managed/conformance.mjs",
"verify:firmware": "npm run build:cached && node scripts/verify/firmware/analysis.mjs",
@@ -273,7 +272,6 @@
"verify:windows-native": "node scripts/verify-windows-native.mjs",
"verify:ghidra:windows:package": "node scripts/verify-packaged-ghidra-windows.mjs",
"build:ghidra:nativeaot": "node scripts/build-ghidra-nativeaot.mjs",
"verify:ghidra:bridge-sources": "node scripts/verify-ghidra-bridge-sources.mjs",
"build:fixtures:nativeaot": "node scripts/generate-nativeaot-fixtures.mjs",
"verify:ghidra:nativeaot": "npm run build:cached && node scripts/verify-real-ghidra-nativeaot.mjs",
"verify:browser:scripts": "npm run build:cached && node scripts/verify-browser-scripts.mjs",
+1 -1
View File
@@ -157,7 +157,7 @@ function propertyValue(content, name) {
return null;
}
/** Release line supported by the separately pinned NativeAOT integration. */
/** Same 12.1 line as GhidraInstallationPolicy, including a distro suffix. */
function ghidraReleaseLineAccepted(raw) {
const match = /^(\d+)\.(\d+)(?:\.(\d+))?([-+].*)?$/u.exec(raw ?? "");
return match !== null && Number(match[1]) === 12 && Number(match[2]) === 1;
-9
View File
@@ -104,7 +104,6 @@ export const SCOPES = {
"tests/boundary/filesystem/npmRelease.test.ts",
"tests/boundary/process/releaseArtifactCanary.test.ts",
"src/windows/**",
"src/ghidra/GhidraBridgeSources.ts",
"native/windows/**",
"src/process/capture/FilesystemSnapshot.ts",
"src/process/capture/FilesystemSnapshot.test.ts",
@@ -437,14 +436,6 @@ export const SCOPES = {
"tests/fixtures/nativeaot/**",
".github/workflows/nativeaot-fixtures.yml",
],
ghidra_bridge: [
"bridge/ghidra/**",
"src/ghidra/GhidraBridgeSources.ts",
"src/ghidra/GhidraJavaProperties.ts",
"src/ghidra/GhidraReleasePolicy.ts",
"scripts/verify-ghidra-bridge-sources.mjs",
".github/workflows/ghidra-bridge-sources.yml",
],
website: [
"website/**",
"scripts/*website*.py",
@@ -1,28 +0,0 @@
import type { z } from "zod";
export interface StructuredScopeExpectation {
readonly operation:
| "parse-package-json"
| "parse-json-module"
| "parse-html-script-references"
| "parse-local-source-map";
readonly analysis_status: "interrupted" | "not-started";
}
export interface StructuredProcessingScope {
readonly properties: Readonly<Record<string, unknown>>;
readonly evidence: {
readonly authority: string;
readonly state: string;
readonly confidence: string;
};
}
export const structuredScopeExpectationsSchema: z.ZodType<
Record<string, StructuredScopeExpectation>
>;
export function matchesStructuredScopeExpectation(
scopes: readonly StructuredProcessingScope[],
expected: StructuredScopeExpectation | undefined,
): boolean;
@@ -1,28 +0,0 @@
import { z } from "zod";
/** Independent expected outcomes supplied by the verification scenario. */
export const structuredScopeExpectationsSchema = z.record(
z.string().min(1),
z.strictObject({
operation: z.enum([
"parse-package-json",
"parse-json-module",
"parse-html-script-references",
"parse-local-source-map",
]),
analysis_status: z.enum(["interrupted", "not-started"]),
}),
);
/** Every retained gap must match the named phase and its unavailable status. */
export const matchesStructuredScopeExpectation = (scopes, expected) =>
expected !== undefined &&
scopes.length > 0 &&
scopes.every(
({ evidence, properties }) =>
evidence.authority === "artifact-bytes" &&
evidence.state === "unavailable" &&
evidence.confidence === "unknown" &&
properties.operation === expected.operation &&
properties.analysis_status === expected.analysis_status,
);
+1 -1
View File
@@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [
group: "source",
include: [
"src/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
],
},
{
+4 -27
View File
@@ -76,30 +76,13 @@ function runCommand() {
env: process.env,
stdio: "inherit",
});
let spawned = false;
let childError;
let hasChildError = false;
child.once("spawn", () => {
spawned = true;
});
const rememberChildError = (cause) => {
if (!hasChildError) {
hasChildError = true;
childError = cause;
}
if (!spawned) {
reject(childError);
}
};
child.on("error", rememberChildError);
child.once("error", reject);
child.once("close", (code) => {
child.off("error", rememberChildError);
// During POSIX cancellation the group leader must stay alive, retaining
// both its group identity and SQLite ownership until the entire group is
// killed. A completed command can still have uncooperative descendants.
if (shutdownSignal !== undefined && process.platform !== "win32") return;
if (hasChildError) reject(childError);
else resolveResult({ code });
if (shutdownSignal === undefined || process.platform === "win32")
resolveResult({ code });
});
});
}
@@ -116,13 +99,7 @@ function handleDisconnect() {
function handleSignal(signal) {
if (shutdownSignal !== undefined) return;
shutdownSignal = signal;
if (child === undefined) return;
if (process.platform === "win32") {
// Windows has no POSIX process groups. Stop only the command owned by this
// supervisor, and keep the SQLite transaction until its close event.
child.kill(signal);
return;
}
if (child === undefined || process.platform === "win32") return;
// Establish cancellation before signalling the group. This supervisor is
// its live leader, so the group identifier cannot be reused during cleanup.
process.kill(-process.pid, signal);
-41
View File
@@ -322,47 +322,6 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
self.assertTrue(any("missing Markdown fragment" in error for error in errors))
self.assertTrue(any("missing repository target" in error for error in errors))
def test_repository_links_accept_aliased_roots_and_reject_escaping_links(self):
with tempfile.TemporaryDirectory() as directory:
root = (Path(directory) / "repository").resolve()
(root / "website/public").mkdir(parents=True)
(root / "docs").mkdir()
(root / "docs/guide.md").write_text("# Guide\n")
page = root / "website/README.md"
page.write_text("[Guide](../docs/guide.md#guide)\n")
alias = Path(directory) / "repository-alias"
alias.symlink_to(root, target_is_directory=True)
self.assertEqual(VERIFY.check_repository_links(root), [])
self.assertEqual(VERIFY.check_repository_links(alias), [])
(root.parent / "outside.md").write_text("# Outside\n")
page.write_text("[Outside](../../outside.md)\n")
for boundary in (root, alias):
with self.subTest(boundary=boundary):
self.assertEqual(VERIFY.check_repository_links(boundary), [
"website/README.md: missing repository target: ../../outside.md"
])
def test_site_links_preserve_containment_through_aliased_roots(self):
with tempfile.TemporaryDirectory() as directory:
site = (Path(directory) / "site").resolve()
site.mkdir()
page = site / "index.html"
page.write_text('<html lang="en"><head><title>Fixture</title><meta name="robots" content="noindex"></head><body><h1>Fixture</h1><a href="guide.html#answer">Guide</a></body></html>')
(site / "guide.html").write_text('<html lang="en"><head><title>Guide</title><meta name="robots" content="noindex"></head><body><h1 id="answer">Answer</h1></body></html>')
(site / "sitemap.xml").write_text('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"/>')
(site / "robots.txt").write_text("User-agent: *\nAllow: /\nSitemap: https://rea.tools/sitemap.xml\n")
alias = Path(directory) / "site-alias"
alias.symlink_to(site, target_is_directory=True)
# The tiny link fixture intentionally omits the social preview.
expected = ["Social preview PNG is missing; run scripts/prepare-website.py"]
self.assertEqual(VERIFY.check_site(site)[3], expected)
self.assertEqual(VERIFY.check_site(alias)[3], expected)
(site.parent / "outside.html").write_text("<p>Outside</p>")
page.write_text(page.read_text().replace("guide.html#answer", "../outside.html"))
for boundary in (site, alias):
with self.subTest(boundary=boundary):
self.assertIn("index.html: reference leaves the website: ../outside.html", VERIFY.check_site(boundary)[3])
def test_preview_preserves_pages_prefix_for_assets_and_directory_redirects(self):
with tempfile.TemporaryDirectory() as directory:
site = Path(directory)
+305 -328
View File
@@ -5,7 +5,6 @@ import { join } from "node:path";
import { AdbDeviceAnalysisService } from "../dist/application/adb/AdbDeviceAnalysisService.js";
import { createAdbDeviceAnalysisProvider } from "../dist/composition/adb.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
const options = parseArguments(process.argv.slice(2));
if (options.help) {
@@ -40,365 +39,343 @@ const run = async (operation, input) => {
return result.value.normalized_result;
};
let primaryFailure;
try {
const client = await run("inspect_adb_client", {});
if (client !== null) {
const client = await run("inspect_adb_client", {});
if (client !== null) {
check(
"adb client reports a version",
typeof client.client.version === "string" && client.client.version !== "",
`binary=${client.client.binary_path} version=${client.client.version}`,
);
}
const listing = await run("list_adb_devices", {});
let serial = options.serial ?? null;
if (listing !== null) {
check(
"device listing parsed",
Array.isArray(listing.devices),
`${listing.devices.length} device(s)`,
);
const usable = listing.devices.filter((device) => device.state === "device");
if (serial === null && usable.length > 0) serial = usable[0].serial;
if (serial === null) {
process.stdout.write(
"[FAIL] no usable device is attached; start an emulator or connect a phone\n",
);
failures += 1;
} else {
const selected = listing.devices.find((device) => device.serial === serial);
check(
"adb client reports a version",
typeof client.client.version === "string" && client.client.version !== "",
`binary=${client.client.binary_path} version=${client.client.version}`,
"selected device present with a kind inference",
selected !== undefined &&
["emulator", "physical", "unknown"].includes(selected.kind),
`serial=${serial} kind=${selected?.kind} basis=${selected?.kind_basis}`,
);
}
}
if (serial !== null) {
const device = await run("inspect_adb_device", { serial });
if (device !== null) {
check(
"device build identity projected",
device.properties.some(
(property) => property.name === "ro.build.version.sdk",
),
`sdk=${
device.properties.find((p) => p.name === "ro.build.version.sdk")?.value
} emulator_observed=${String(device.emulator_observed)}`,
);
}
const listing = await run("list_adb_devices", {});
let serial = options.serial ?? null;
if (listing !== null) {
// Observation tools: processes, window, display, settings, logcat.
const processes = await run("list_adb_processes", { serial });
if (processes !== null) {
check(
"device listing parsed",
Array.isArray(listing.devices),
`${listing.devices.length} device(s)`,
"process listing is non-empty with a header",
processes.processes.length > 0 && typeof processes.columns === "string",
`${processes.processes.length} process(es)`,
);
const usable = listing.devices.filter(
(device) => device.state === "device",
}
const window = await run("inspect_adb_window", { serial });
if (window !== null)
check(
"window focus observation completed",
typeof window.focused_window === "string" ||
window.focused_window === null,
`focused_window=${String(window.focused_window).slice(0, 60)} (null is legitimate on headless devices)`,
);
if (serial === null && usable.length > 0) serial = usable[0].serial;
if (serial === null) {
process.stdout.write(
"[FAIL] no usable device is attached; start an emulator or connect a phone\n",
);
failures += 1;
} else {
const selected = listing.devices.find(
(device) => device.serial === serial,
);
const display = await run("inspect_adb_display", { serial });
if (display !== null)
check(
"display reports a physical size",
display.physical_size !== null &&
display.physical_size.width > 0 &&
display.physical_size.height > 0,
`${display.physical_size?.width}x${display.physical_size?.height} @${String(display.physical_density)}`,
);
const setting = await run("read_adb_setting", {
serial,
namespace: "global",
key: "adb_enabled",
});
if (setting !== null)
check(
"settings read returns the device's own value",
setting.value === "1",
`adb_enabled=${String(setting.value)}`,
);
const logcat = await run("read_adb_logcat", {
serial,
count: 20,
buffer: "main",
});
if (logcat !== null)
check(
"logcat dump returns bounded lines",
logcat.lines.length <= 20,
`${logcat.lines.length} line(s), coverage=${logcat.coverage}`,
);
const features = await run("list_adb_features", { serial });
if (features !== null)
check(
"feature inventory is non-empty",
features.coverage === "complete" && features.features.length > 0,
`${features.features.length} feature(s)`,
);
const services = await run("list_adb_services", { serial });
if (services !== null)
check(
"service inventory is non-empty",
services.coverage === "complete" && services.services.length > 0,
`${services.services.length} service(s)`,
);
const directory = await run("list_adb_directory", {
serial,
device_path: "/data/local/tmp",
});
if (directory !== null)
check(
"directory listing completed",
directory.coverage === "complete",
`${directory.entries.length} entr(ies)`,
);
const packages = await run("list_adb_packages", {
serial,
scope: "third_party",
});
let pullCandidate = options.pull ?? null;
if (packages !== null) {
// A third-party scope may legitimately be empty (clean emulator images);
// completeness is about unparsed lines, not row count.
check(
"third-party package listing reports complete coverage",
packages.coverage === "complete",
`${packages.packages.length} package(s)`,
);
const allPackages = await run("list_adb_packages", {
serial,
scope: "all",
});
if (allPackages !== null) {
check(
"selected device present with a kind inference",
selected !== undefined &&
["emulator", "physical", "unknown"].includes(selected.kind),
`serial=${serial} kind=${selected?.kind} basis=${selected?.kind_basis}`,
"full package inventory is non-empty",
allPackages.coverage === "complete" && allPackages.packages.length > 0,
`${allPackages.packages.length} package(s)`,
);
}
if (pullCandidate === null && packages.packages.length > 0)
pullCandidate = packages.packages[0].package_name;
}
if (serial !== null) {
const device = await run("inspect_adb_device", { serial });
if (device !== null) {
check(
"device build identity projected",
device.properties.some(
(property) => property.name === "ro.build.version.sdk",
),
`sdk=${
device.properties.find((p) => p.name === "ro.build.version.sdk")
?.value
} emulator_observed=${String(device.emulator_observed)}`,
);
}
// Observation tools: processes, window, display, settings, logcat.
const processes = await run("list_adb_processes", { serial });
if (processes !== null) {
check(
"process listing is non-empty with a header",
processes.processes.length > 0 && typeof processes.columns === "string",
`${processes.processes.length} process(es)`,
);
}
const window = await run("inspect_adb_window", { serial });
if (window !== null)
check(
"window focus observation completed",
typeof window.focused_window === "string" ||
window.focused_window === null,
`focused_window=${String(window.focused_window).slice(0, 60)} (null is legitimate on headless devices)`,
);
const display = await run("inspect_adb_display", { serial });
if (display !== null)
check(
"display reports a physical size",
display.physical_size !== null &&
display.physical_size.width > 0 &&
display.physical_size.height > 0,
`${display.physical_size?.width}x${display.physical_size?.height} @${String(display.physical_density)}`,
);
const setting = await run("read_adb_setting", {
serial,
namespace: "global",
key: "adb_enabled",
});
if (setting !== null)
check(
"settings read returns the device's own value",
setting.value === "1",
`adb_enabled=${String(setting.value)}`,
);
const logcat = await run("read_adb_logcat", {
serial,
count: 20,
buffer: "main",
});
if (logcat !== null)
check(
"logcat dump returns bounded lines",
logcat.lines.length <= 20,
`${logcat.lines.length} line(s), coverage=${logcat.coverage}`,
);
const features = await run("list_adb_features", { serial });
if (features !== null)
check(
"feature inventory is non-empty",
features.coverage === "complete" && features.features.length > 0,
`${features.features.length} feature(s)`,
);
const services = await run("list_adb_services", { serial });
if (services !== null)
check(
"service inventory is non-empty",
services.coverage === "complete" && services.services.length > 0,
`${services.services.length} service(s)`,
);
const directory = await run("list_adb_directory", {
serial,
device_path: "/data/local/tmp",
});
if (directory !== null)
check(
"directory listing completed",
directory.coverage === "complete",
`${directory.entries.length} entr(ies)`,
);
const packages = await run("list_adb_packages", {
serial,
scope: "third_party",
});
let pullCandidate = options.pull ?? null;
if (packages !== null) {
// A third-party scope may legitimately be empty (clean emulator images);
// completeness is about unparsed lines, not row count.
check(
"third-party package listing reports complete coverage",
packages.coverage === "complete",
`${packages.packages.length} package(s)`,
);
const allPackages = await run("list_adb_packages", {
serial,
scope: "all",
});
if (allPackages !== null) {
check(
"full package inventory is non-empty",
allPackages.coverage === "complete" &&
allPackages.packages.length > 0,
`${allPackages.packages.length} package(s)`,
);
}
if (pullCandidate === null && packages.packages.length > 0)
pullCandidate = packages.packages[0].package_name;
}
if (pullCandidate !== null) {
const directory = await mkdtemp(join(tmpdir(), "rea-adb-verify-"));
try {
const pull = await run("pull_adb_package", {
serial,
package: pullCandidate,
output_directory: directory,
});
if (pull !== null) {
// Installed /data/app packages name their APKs base.apk and
// split_*.apk; system packages keep arbitrary names, so the lane
// checks a non-empty artifact set and verifies each file on disk.
check(
"pull returned a non-empty artifact set",
pull.artifacts.length > 0,
`${pull.artifacts.length} artifact(s), coverage=${pull.coverage}, roles=${pull.artifacts.map((a) => a.role).join(",")}`,
);
for (const artifact of pull.artifacts) {
const bytes = await stat(artifact.local_path).then(
(info) => info.size,
);
const digest = createHash("sha256")
.update(await readFile(artifact.local_path))
.digest("hex");
check(
`artifact digest matches on disk: ${artifact.file_name}`,
bytes === artifact.bytes && digest === artifact.sha256,
`${bytes} bytes`,
);
}
}
} finally {
await rm(directory, { recursive: true, force: true });
}
}
// File transfer: push one lane-owned file, verify both digests, pull it
// back, then clean up the device copy through the same fixed argv surface.
const transferRoot = await mkdtemp(join(tmpdir(), "rea-adb-transfer-"));
if (pullCandidate !== null) {
const directory = await mkdtemp(join(tmpdir(), "rea-adb-verify-"));
try {
const payload = join(transferRoot, "rea-verify.txt");
const payloadBytes = "rea adb transfer verification\n";
await import("node:fs/promises").then((fs) =>
fs.writeFile(payload, payloadBytes),
);
const deviceTarget = `/data/local/tmp/rea-verify-${Date.now()}.txt`;
const pushed = await run("push_adb_file", {
const pull = await run("pull_adb_package", {
serial,
local_path: payload,
device_path: deviceTarget,
overwrite: false,
package: pullCandidate,
output_directory: directory,
});
if (pushed !== null) {
if (pull !== null) {
// Installed /data/app packages name their APKs base.apk and
// split_*.apk; system packages keep arbitrary names, so the lane
// checks a non-empty artifact set and verifies each file on disk.
check(
"push reports matching device-side digest",
pushed.device_digest_source === "device_sha256sum" &&
pushed.device_sha256 === pushed.sha256,
`${pushed.bytes} bytes`,
"pull returned a non-empty artifact set",
pull.artifacts.length > 0,
`${pull.artifacts.length} artifact(s), coverage=${pull.coverage}, roles=${pull.artifacts.map((a) => a.role).join(",")}`,
);
const pulled = await run("pull_adb_file", {
serial,
device_path: deviceTarget,
output_directory: transferRoot,
});
if (pulled !== null)
check(
"pulled file digest matches the pushed digest",
pulled.sha256 === pushed.sha256,
`${pulled.bytes} bytes`,
for (const artifact of pull.artifacts) {
const bytes = await stat(artifact.local_path).then(
(info) => info.size,
);
}
const screenRoot = await mkdtemp(join(tmpdir(), "rea-adb-screen-"));
try {
const screen = await run("capture_adb_screen", {
serial,
output_directory: screenRoot,
});
if (screen !== null)
const digest = createHash("sha256")
.update(await readFile(artifact.local_path))
.digest("hex");
check(
"screen capture is a sized PNG",
screen.width !== null && screen.height !== null && screen.bytes > 0,
`${screen.width}x${screen.height}, ${screen.bytes} bytes`,
`artifact digest matches on disk: ${artifact.file_name}`,
bytes === artifact.bytes && digest === artifact.sha256,
`${bytes} bytes`,
);
} finally {
await rm(screenRoot, { recursive: true, force: true });
}
if (options.pull !== null) {
const details = await run("inspect_adb_package", {
serial,
package: options.pull,
});
if (details !== null)
check(
"dumpsys package projection completed",
details.coverage === "complete",
`versionCode=${String(details.version_code)}`,
);
}
if (options.installApk !== null && options.installApk !== undefined) {
const probePackage = "com.rea.adb.probe";
const installedProbe = await run("install_adb_package", {
serial,
apk_path: options.installApk,
replace: false,
});
if (installedProbe !== null) {
check(
"install reports the APK digest",
typeof installedProbe.sha256 === "string" &&
installedProbe.sha256.length === 64,
`${installedProbe.bytes} bytes`,
);
const resolved = await run("resolve_adb_packages", {
serial,
query: "rea.adb.probe",
});
if (resolved !== null)
check(
"resolve finds the installed probe uniquely",
resolved.matches.length === 1 &&
resolved.matches[0]?.package_name === probePackage,
`${resolved.matches.length} match(es)`,
);
const started = await run("start_adb_app", {
serial,
package: probePackage,
});
if (started !== null)
check(
"app start resolves and launches the launcher activity",
typeof started.component === "string" &&
started.component.startsWith(probePackage),
String(started.component),
);
const observedProcesses = await run("list_adb_processes", { serial });
if (observedProcesses !== null)
check(
"started app appears in the process listing",
observedProcesses.processes.some(
(process) => process.name === probePackage,
),
);
const stopped = await run("stop_adb_app", {
serial,
package: probePackage,
});
if (stopped !== null)
check(
"force-stop completes the dynamic-analysis run",
stopped.package_name === probePackage,
);
const removed = await run("uninstall_adb_package", {
serial,
package: probePackage,
});
if (removed !== null)
check(
"uninstall completes the lifecycle",
removed.package_name === probePackage,
);
const afterRemoval = await run("resolve_adb_packages", {
serial,
query: "rea.adb.probe",
});
if (afterRemoval !== null)
check(
"resolve reports zero matches after removal",
afterRemoval.matches.length === 0,
);
}
}
} finally {
await rm(transferRoot, { recursive: true, force: true });
await rm(directory, { recursive: true, force: true });
}
}
} catch (cause) {
primaryFailure = cause;
// File transfer: push one lane-owned file, verify both digests, pull it
// back, then clean up the device copy through the same fixed argv surface.
const transferRoot = await mkdtemp(join(tmpdir(), "rea-adb-transfer-"));
try {
const payload = join(transferRoot, "rea-verify.txt");
const payloadBytes = "rea adb transfer verification\n";
await import("node:fs/promises").then((fs) =>
fs.writeFile(payload, payloadBytes),
);
const deviceTarget = `/data/local/tmp/rea-verify-${Date.now()}.txt`;
const pushed = await run("push_adb_file", {
serial,
local_path: payload,
device_path: deviceTarget,
overwrite: false,
});
if (pushed !== null) {
check(
"push reports matching device-side digest",
pushed.device_digest_source === "device_sha256sum" &&
pushed.device_sha256 === pushed.sha256,
`${pushed.bytes} bytes`,
);
const pulled = await run("pull_adb_file", {
serial,
device_path: deviceTarget,
output_directory: transferRoot,
});
if (pulled !== null)
check(
"pulled file digest matches the pushed digest",
pulled.sha256 === pushed.sha256,
`${pulled.bytes} bytes`,
);
}
const screenRoot = await mkdtemp(join(tmpdir(), "rea-adb-screen-"));
try {
const screen = await run("capture_adb_screen", {
serial,
output_directory: screenRoot,
});
if (screen !== null)
check(
"screen capture is a sized PNG",
screen.width !== null && screen.height !== null && screen.bytes > 0,
`${screen.width}x${screen.height}, ${screen.bytes} bytes`,
);
} finally {
await rm(screenRoot, { recursive: true, force: true });
}
if (options.pull !== null) {
const details = await run("inspect_adb_package", {
serial,
package: options.pull,
});
if (details !== null)
check(
"dumpsys package projection completed",
details.coverage === "complete",
`versionCode=${String(details.version_code)}`,
);
}
if (options.installApk !== null && options.installApk !== undefined) {
const probePackage = "com.rea.adb.probe";
const installedProbe = await run("install_adb_package", {
serial,
apk_path: options.installApk,
replace: false,
});
if (installedProbe !== null) {
check(
"install reports the APK digest",
typeof installedProbe.sha256 === "string" &&
installedProbe.sha256.length === 64,
`${installedProbe.bytes} bytes`,
);
const resolved = await run("resolve_adb_packages", {
serial,
query: "rea.adb.probe",
});
if (resolved !== null)
check(
"resolve finds the installed probe uniquely",
resolved.matches.length === 1 &&
resolved.matches[0]?.package_name === probePackage,
`${resolved.matches.length} match(es)`,
);
const started = await run("start_adb_app", {
serial,
package: probePackage,
});
if (started !== null)
check(
"app start resolves and launches the launcher activity",
typeof started.component === "string" &&
started.component.startsWith(probePackage),
String(started.component),
);
const observedProcesses = await run("list_adb_processes", { serial });
if (observedProcesses !== null)
check(
"started app appears in the process listing",
observedProcesses.processes.some(
(process) => process.name === probePackage,
),
);
const stopped = await run("stop_adb_app", {
serial,
package: probePackage,
});
if (stopped !== null)
check(
"force-stop completes the dynamic-analysis run",
stopped.package_name === probePackage,
);
const removed = await run("uninstall_adb_package", {
serial,
package: probePackage,
});
if (removed !== null)
check(
"uninstall completes the lifecycle",
removed.package_name === probePackage,
);
const afterRemoval = await run("resolve_adb_packages", {
serial,
query: "rea.adb.probe",
});
if (afterRemoval !== null)
check(
"resolve reports zero matches after removal",
afterRemoval.matches.length === 0,
);
}
}
} finally {
await rm(transferRoot, { recursive: true, force: true });
}
}
let cleanupFailure;
try {
await closeVerifierResources([() => provider.close()], primaryFailure);
} catch (cause) {
cleanupFailure = cause;
await provider.close();
} catch {
// The provider owns nothing long-lived; closing is best-effort.
}
if (primaryFailure !== undefined || cleanupFailure !== undefined) {
const causes =
cleanupFailure instanceof AggregateError
? cleanupFailure.errors
: [cleanupFailure ?? primaryFailure];
process.stderr.write(
`ADB device verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`,
);
process.exitCode = 1;
} else if (failures > 0) {
if (failures > 0) {
process.stdout.write(`\n${failures} check(s) failed\n`);
process.exitCode = 1;
} else {
process.stdout.write("\nall adb device checks passed\n");
process.exit(1);
}
process.stdout.write("\nall adb device checks passed\n");
function parseArguments(argv) {
const parsed = { help: false, serial: null, pull: null, installApk: null };
+14 -41
View File
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdtemp, readdir, readFile, rmdir } from "node:fs/promises";
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
@@ -8,9 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
import { OwnedRegularFileCleanupFailure } from "../dist/filesystem/RegularFile.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
const options = parseArguments(process.argv.slice(2));
if (options.help) {
@@ -41,17 +39,6 @@ Object.assign(environment, {
TEMP: root,
});
const client = new Client({ name: "flutter-verifier", version: "1" });
let primaryError;
const hashOwners = new Map();
const hashOwned = async (...args) => {
try {
return await hashStableFile(...args);
} catch (cause) {
if (cause instanceof OwnedRegularFileCleanupFailure)
hashOwners.set(cause.owner, cause.path);
throw cause;
}
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
@@ -62,7 +49,7 @@ const execute = promisify(execFile);
try {
await client.connect(transport);
const apk = resolve(options.apk);
const identity = await hashOwned(apk);
const identity = await hashStableFile(apk);
const identified = await run("identify_flutter_build", { path: apk });
assert.deepEqual(identified.target, { path: apk, ...identity });
assert.equal(identified.flutter_detected, true);
@@ -189,46 +176,32 @@ try {
const negative = await run("identify_flutter_build", { path: plain });
assert.deepEqual(negative.target, {
path: plain,
...(await hashOwned(plain)),
...(await hashStableFile(plain)),
});
assert.equal(negative.flutter_detected, false);
assert.equal(negative.coverage, "complete");
}
assert.deepEqual(
await hashOwned(apk),
await hashStableFile(apk),
identity,
"Verification must preserve its input",
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
for (const [owner, path] of hashOwners) {
cleanup.push(async () => {
try {
await owner.close();
hashOwners.delete(owner);
} catch (cause) {
throw new Error(`Could not close hash file handle for ${path}`, {
cause,
});
}
});
}
cleanup.push(async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
]);
// Server shutdown retries retained provider cleanup and must settle before success.
try {
try {
await client.close();
} finally {
await transport.close();
}
assert.deepEqual(
await readdir(root),
[],
`Provider snapshots remain under ${root}`,
);
await rmdir(root);
});
await closeVerifierResources(cleanup, primaryError);
} finally {
await rm(root, { recursive: true, force: true });
}
}
process.stdout.write(
"all asserted Flutter CLI/MCP checks passed; unverified facets are reported above\n",
-62
View File
@@ -5,11 +5,6 @@ import { tmpdir } from "node:os";
import { join, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import Ajv2020 from "ajv/dist/2020.js";
import {
schemaArrays,
schemaChildren,
schemaMaps,
} from "../dist/contracts/inputSchemaPresentation.js";
import {
TOOL_CONTRACTS,
toolContract,
@@ -22,8 +17,6 @@ const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini";
const mode = process.argv[2] ?? "call";
const schemaProfile = process.env.REA_VERIFY_GEMINI_SCHEMA_PROFILE ?? "full";
assert(["full", "compact"].includes(schemaProfile));
// npm installs of Gemini CLI provide a JavaScript entry point; on Windows only a gemini.cmd shim reaches
// PATH, which Node does not spawn without a shell, so a selected .js entry runs under this Node. It is
// resolved once, because the preflight runs from the caller's directory and the client from the workspace.
@@ -67,7 +60,6 @@ const environment = {
GEMINI_CLI_NO_RELAUNCH: "1",
GEMINI_CLI_TRUST_WORKSPACE: "true",
REA_PROCESS_RUN_ID: verifier.run_id,
REA_MCP_INPUT_SCHEMA_PROFILE: schemaProfile,
};
let clientVersion;
try {
@@ -169,43 +161,8 @@ let skillRequested = false;
let finalSeen = false;
let catalogSize = 0;
let schemasChecked = 0;
let examplesChecked = 0;
let artifactDigest;
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
const booleanLiteralPath = (schema, path = "$", seen = new Set()) => {
if (!schema || typeof schema !== "object" || seen.has(schema))
return undefined;
seen.add(schema);
for (const [key, child] of Object.entries(schema)) {
const childPath = `${path}.${key}`;
if (key === "const" && typeof child === "boolean") return childPath;
if (
key === "enum" &&
Array.isArray(child) &&
child.some((item) => typeof item === "boolean")
)
return childPath;
const schemaChildrenAtKey =
schemaMaps.has(key) &&
child &&
typeof child === "object" &&
!Array.isArray(child)
? Object.entries(child).map(([name, item]) => [
`${childPath}.${name}`,
item,
])
: schemaArrays.has(key) && Array.isArray(child)
? child.map((item, index) => [`${childPath}[${index}]`, item])
: schemaChildren.has(key)
? [[childPath, child]]
: [];
for (const [nestedPath, nested] of schemaChildrenAtKey) {
const found = booleanLiteralPath(nested, nestedPath, seen);
if (found !== undefined) return found;
}
}
return undefined;
};
const extractEvidence = (value) => {
if (typeof value === "string") {
if (
@@ -269,22 +226,6 @@ const fixture = await createGeminiModelFixture({
schemaValidator.validateSchema(tool.parametersJsonSchema),
`${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`,
);
const booleanLiteral = booleanLiteralPath(tool.parametersJsonSchema);
assert.equal(
booleanLiteral,
undefined,
`${tool.name}: Gemini SDK declaration retains a boolean const/enum at ${booleanLiteral}`,
);
const name = tool.name.slice(prefix.length);
const contract = toolContract(name);
const validate = schemaValidator.compile(tool.parametersJsonSchema);
for (const example of contract.examples) {
assert(
validate(example.input),
`${tool.name}: Gemini SDK declaration rejects canonical example ${example.title}: ${JSON.stringify(schemaValidator.errors)}`,
);
examplesChecked++;
}
schemasChecked++;
}
}
@@ -415,9 +356,6 @@ try {
client: "Gemini CLI",
clientVersion,
mode,
schemaProfile,
schemasChecked,
examplesChecked,
passed: failure === undefined,
artifacts: lab,
runtimeRoot,
-182
View File
@@ -1,182 +0,0 @@
#!/usr/bin/env node
// Compile the packaged Ghidra bridge sources against an existing Ghidra
// installation's jars. Ghidra's script provider compiles the whole bridge
// directory when a session starts, so one unresolved symbol disables every
// operation and surfaces only after import and auto-analysis. This check
// reports the same javac diagnostics without launching Ghidra or selecting,
// importing or executing a target.
import { spawnSync } from "node:child_process";
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
// The pinned Node toolchain strips types from these dependency-free owners;
// this preflight therefore uses runtime facts without needing npm ci or a build.
import { GHIDRA_BRIDGE_SOURCES } from "../src/ghidra/GhidraBridgeSources.ts";
import { parseGhidraJavaProperties } from "../src/ghidra/GhidraJavaProperties.ts";
import {
SUPPORTED_GHIDRA_VERSION,
SUPPORTED_GHIDRA_JAVA_MAJOR,
ghidraReleaseAccepted,
ghidraReleaseLine,
} from "../src/ghidra/GhidraReleasePolicy.ts";
const args = process.argv.slice(2);
if (
args.length > 1 ||
(args.length === 1 &&
!["--check-sdk-pin", "--verified-sdk"].includes(args[0]))
)
fail(
"Usage: verify-ghidra-bridge-sources.mjs [--check-sdk-pin|--verified-sdk]",
);
if (args[0] === "--check-sdk-pin") {
let url;
try {
url = new URL(process.env.GHIDRA_RELEASE_URL);
} catch {
fail("GHIDRA_RELEASE_URL must name the checksum-pinned official release");
}
const prefix = `/NationalSecurityAgency/ghidra/releases/download/Ghidra_${SUPPORTED_GHIDRA_VERSION}_build/ghidra_${SUPPORTED_GHIDRA_VERSION}_PUBLIC_`;
if (
url.origin !== "https://github.com" ||
url.username ||
url.password ||
url.search ||
url.hash ||
!url.pathname.startsWith(prefix) ||
!url.pathname.endsWith(".zip") ||
!/^[a-f0-9]{64}$/iu.test(process.env.GHIDRA_RELEASE_SHA256 ?? "")
)
fail(
`SDK URL and SHA256 must match verified Ghidra ${SUPPORTED_GHIDRA_VERSION}`,
);
console.log(`GHIDRA_JAVA_MAJOR=${SUPPORTED_GHIDRA_JAVA_MAJOR}`);
process.exit(0);
}
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const bridge = join(root, "bridge/ghidra");
const install = process.env.GHIDRA_INSTALL_DIR;
if (!install)
fail(
`Set GHIDRA_INSTALL_DIR to an existing Ghidra ${ghidraReleaseLine()} installation (verified with ${SUPPORTED_GHIDRA_VERSION}). REA does not install Ghidra.`,
);
const properties = await readFile(
join(install, "Ghidra/application.properties"),
"latin1",
).catch((cause) =>
fail(
`GHIDRA_INSTALL_DIR has no readable Ghidra/application.properties: ${cause.message}`,
),
);
const parsedProperties = parseGhidraJavaProperties(properties);
if (!parsedProperties.ok) fail(parsedProperties.error);
const version = parsedProperties.value.get("application.version");
if (!ghidraReleaseAccepted(version))
fail(
`Bridge sources require Ghidra ${ghidraReleaseLine()} (verified with ${SUPPORTED_GHIDRA_VERSION}). Observed application.version=${version === undefined ? "missing" : JSON.stringify(version)}.`,
);
if (args[0] === "--verified-sdk" && version !== SUPPORTED_GHIDRA_VERSION)
fail(
`CI SDK must be verified Ghidra ${SUPPORTED_GHIDRA_VERSION}; observed ${JSON.stringify(version)}`,
);
const sourceNames = (await readdir(bridge, { withFileTypes: true }))
.filter((entry) => entry.isFile() && entry.name.endsWith(".java"))
.map((entry) => entry.name);
for (const name of Object.values(GHIDRA_BRIDGE_SOURCES))
if (!sourceNames.includes(name))
fail(`Required Ghidra bridge source is missing: bridge/ghidra/${name}`);
const requiredNames = new Set(Object.values(GHIDRA_BRIDGE_SOURCES));
const unregistered = sourceNames.filter((name) => !requiredNames.has(name));
if (unregistered.length)
fail(
`Ghidra bridge sources absent from the manifest: ${unregistered.join(", ")}`,
);
const sources = sourceNames
.map((name) => relative(root, join(bridge, name)))
.sort();
const javac = process.env.JAVA_HOME
? join(process.env.JAVA_HOME, "bin", "javac")
: "javac";
const banner = run(javac, ["-version"]);
if (banner.error || banner.status !== 0)
fail(`javac is unavailable: ${banner.error?.message ?? output(banner)}`);
const major = Number(/^javac (\d+)/mu.exec(output(banner))?.[1]);
if (!Number.isInteger(major) || major < SUPPORTED_GHIDRA_JAVA_MAJOR)
fail(
`Ghidra ${ghidraReleaseLine()} compiles bridge scripts with Java ${SUPPORTED_GHIDRA_JAVA_MAJOR} or newer. Observed: ${output(banner) || "no version"}. REA does not install or upgrade Java.`,
);
const jars = (
await readdir(join(install, "Ghidra"), {
recursive: true,
withFileTypes: true,
})
)
.filter((entry) => entry.isFile() && entry.name.endsWith(".jar"))
.map((entry) => join(entry.parentPath, entry.name))
.sort();
if (jars.length === 0) fail(`No jars found under ${join(install, "Ghidra")}.`);
const scratch = await mkdtemp(join(tmpdir(), "rea-ghidra-bridge-"));
try {
// An argument file keeps the jar class path below Windows' command length.
const argumentFile = join(scratch, "javac.args");
await writeFile(
argumentFile,
[
"-proc:none",
"-encoding",
"UTF-8",
"--release",
String(SUPPORTED_GHIDRA_JAVA_MAJOR),
"-cp",
jars.join(process.platform === "win32" ? ";" : ":"),
"-d",
join(scratch, "classes"),
...sources,
]
.map((s) => `"${s.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`)
.join("\n"),
);
// Sources are repository-relative, so diagnostics name bridge/ghidra/<file>:<line>.
const result = run(javac, ["-J-Xmx1g", `@${argumentFile}`]);
if (result.error || result.status !== 0) {
// Set the status instead of exiting so the scratch directory is removed.
process.stderr.write(
`${output(result) || result.error?.message || "javac failed"}\nGhidra ${version} could not compile the packaged bridge sources; the provider would fail at session start.\n`,
);
process.exitCode = 1;
} else {
console.log(
JSON.stringify({
ghidra_version: version,
javac: output(banner),
class_path_jars: jars.length,
sources,
}),
);
}
} finally {
await rm(scratch, { recursive: true, force: true });
}
function run(command, args) {
return spawnSync(command, args, {
cwd: root,
encoding: "utf8",
timeout: 300_000,
maxBuffer: 16 * 1024 * 1024,
});
}
function output(result) {
return `${result.stdout ?? ""}${result.stderr ?? ""}`.trim();
}
function fail(message) {
process.stderr.write(`${message}\n`);
process.exit(1);
}
+1 -4
View File
@@ -17,10 +17,7 @@ export async function verifyPackagePack({ root, workspace }) {
})
).stdout.trim(),
);
const listing = (await exec("tar", ["-tf", tarball])).stdout;
// A bare LF identifies LF output; preceding CRs then belong to filenames.
const lineEnding = /(^|[^\r])\n/u.test(listing) ? "\n" : "\r\n";
const packedFiles = listing.split(lineEnding);
const packedFiles = (await exec("tar", ["-tf", tarball])).stdout.split("\n");
const packedManifest = JSON.parse(
(await exec("tar", ["-xOf", tarball, "package/package.json"])).stdout,
);
+1 -6
View File
@@ -2,7 +2,6 @@
import { join } from "node:path";
import { mkdir } from "node:fs/promises";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { exec, execNpm } from "./lib/verify-package-core.mjs";
import { WABT_WORKFLOW_TIMEOUT_MS } from "./lib/wabt-verifier-timeouts.mjs";
import { verifyPackagePack } from "./verify-package-pack.mjs";
@@ -10,7 +9,6 @@ import { verifyPackagePack } from "./verify-package-pack.mjs";
const root = await PrivateRuntimeRoot.create({
prefix: "rea-wabt-package-verifier-",
});
let primaryError;
try {
const { tarball } = await verifyPackagePack({
root: process.cwd(),
@@ -58,9 +56,6 @@ try {
},
);
process.stdout.write(verified.stdout);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => root.close()], primaryError);
await root.close();
}
+5 -4
View File
@@ -1,20 +1,21 @@
import { execFile } from "node:child_process";
import { join } from "node:path";
import { promisify } from "node:util";
import { GHIDRA_BRIDGE_SOURCES } from "../src/ghidra/GhidraBridgeSources.ts";
const exec = promisify(execFile);
const packedHopperBridge = "package/bridge/hopper_bridge.py";
const packedCutterBridge = "package/bridge/cutter/rea_bridge.py";
const packedGhidraBridge = "package/bridge/ghidra/ReaGhidraBridge.java";
const packedGhidraActivator =
"package/bridge/ghidra/ReaGhidraBundleActivator.java";
/** Check shipped bridge files; real-provider lanes verify runtime behavior. */
export async function verifyPackedBridge({ workspace, tarball, packedFiles }) {
for (const path of [
packedHopperBridge,
packedCutterBridge,
...Object.values(GHIDRA_BRIDGE_SOURCES).map(
(name) => `package/bridge/ghidra/${name}`,
),
packedGhidraBridge,
packedGhidraActivator,
]) {
if (!packedFiles.includes(path)) throw new Error(`package omitted ${path}`);
}
+3 -45
View File
@@ -12,7 +12,6 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -107,7 +106,6 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await writeFile(path, source);
await writeFile(bluetooth, bluetoothSource);
@@ -265,27 +263,6 @@ try {
assert.ok(pcapAjv.validateSchema(pcapTool.inputSchema));
assert.ok(pcapAjv.validateSchema(pcapTool.outputSchema));
for (const capturePath of [path, bluetooth]) {
const capinfosOutput = (
await execute(
process.env.REA_CAPINFOS_COMMAND ?? "capinfos",
["-M", "--", capturePath],
{
env: { ...environment, LC_ALL: "C", LANG: "C" },
timeout: 60000,
maxBuffer: 64 * 1024,
},
)
).stdout;
const expectedCaptureStart =
/^(?:Earliest|First) packet time:[ \t]*(.+)$/mu
.exec(capinfosOutput)?.[1]
?.trim() ?? null;
const expectedCaptureEnd =
/^(?:Latest|Last) packet time:[ \t]*(.+)$/mu
.exec(capinfosOutput)?.[1]
?.trim() ?? null;
assert.ok(expectedCaptureStart, capinfosOutput);
assert.ok(expectedCaptureEnd, capinfosOutput);
const response = await client.callTool({
name: pcapTool.name,
arguments: { path: capturePath, detail: "summary" },
@@ -296,27 +273,12 @@ try {
pcapAjv.errorsText(),
);
const summary = parseEvidence(response.structuredContent).normalized_result;
assert.equal(summary.summary.capture_start, expectedCaptureStart);
assert.equal(summary.summary.capture_end, expectedCaptureEnd);
assert.equal(
summary.summary.target.sha256,
createHash("sha256")
.update(await readFile(capturePath))
.digest("hex"),
);
const cli = parseEvidence(
JSON.parse(
(
await execute(
process.execPath,
[entrypoint, "inspect-pcap-capture", capturePath, "--json"],
{ env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
)
).stdout,
),
);
assert.equal(cli.normalized_result.capture_start, expectedCaptureStart);
assert.equal(cli.normalized_result.capture_end, expectedCaptureEnd);
const source = {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
@@ -501,14 +463,10 @@ try {
"network with spaces.pcap",
"paged.pcap",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
await client.close();
await transport.close();
await root.close();
}
console.log(
JSON.stringify(
+12 -43
View File
@@ -8,9 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
import { OwnedRegularFileCleanupFailure } from "../dist/filesystem/RegularFile.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
const args = process.argv.slice(2);
if (args.includes("--help") || args.includes("-h")) {
@@ -36,23 +34,12 @@ Object.assign(environment, {
const entrypoint = resolve("scripts/rea.mjs");
const execute = promisify(execFile);
const client = new Client({ name: "pcap-real-verifier", version: "1" });
let primaryError;
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment,
stderr: "pipe",
});
const hashOwners = new Map();
const hashOwned = async (...args) => {
try {
return await hashStableFile(...args);
} catch (cause) {
if (cause instanceof OwnedRegularFileCleanupFailure)
hashOwners.set(cause.owner, cause.path);
throw cause;
}
};
transport.stderr?.resume();
try {
await client.connect(transport);
@@ -61,7 +48,7 @@ try {
if (path === null)
console.log("[unverified] Capture inspection: no --pcap was supplied");
else {
const identity = await hashOwned(path);
const identity = await hashStableFile(path);
const result = await run("inspect_pcap_capture", { path });
assert.deepEqual(result.target, { path, ...identity });
assert(result.packet_count >= 0);
@@ -88,7 +75,7 @@ try {
);
assert.equal(result.coverage, partial ? "partial" : "complete");
assert.deepEqual(
await hashOwned(path),
await hashStableFile(path),
identity,
"Input capture must remain unchanged",
);
@@ -96,36 +83,18 @@ try {
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
);
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
for (const [owner, path] of hashOwners) {
cleanup.push(async () => {
try {
await owner.close();
hashOwners.delete(owner);
} catch (cause) {
throw new Error(`Could not close hash file handle for ${path}`, {
cause,
});
}
});
try {
await client.close();
} finally {
await transport.close();
}
cleanup.push(async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
});
await closeVerifierResources(cleanup, primaryError);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
}
console.log("All asserted pcap CLI/MCP checks passed");
+1 -1
View File
@@ -3,7 +3,7 @@ import { hasTypedSwitchEvidence } from "./verify-real-ghidra-switch-assertions.m
import { access } from "node:fs/promises";
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraReleasePolicy.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraInstallationPolicy.js";
import { GHIDRA_SESSION_CAPABILITIES } from "../dist/ghidra/GhidraSessionValues.js";
export function assertLocalProcedureInfo(info, expectedAddress) {
+1 -1
View File
@@ -11,7 +11,7 @@ import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { parseConfig } from "../dist/config/parseConfig.js";
import { GhidraClient } from "../dist/ghidra/GhidraClient.js";
import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraReleasePolicy.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraInstallationPolicy.js";
import { GhidraProvider } from "../dist/ghidra/GhidraProvider.js";
import { silentLogger } from "../dist/logger.js";
import { cleanupOwnedProcessGroup } from "../dist/process/ProcessOwnership.js";
+1 -1
View File
@@ -19,7 +19,7 @@ import {
parseGhidraInventoryResult,
} from "../dist/ghidra/GhidraInventoryValues.js";
import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraReleasePolicy.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraInstallationPolicy.js";
import { GhidraHeadlessLauncher } from "../dist/ghidra/GhidraLauncher.js";
import { GHIDRA_PROVIDER_IDENTITY } from "../dist/ghidra/GhidraProviderCapabilities.js";
import {
+1 -1
View File
@@ -11,7 +11,7 @@ import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { resolveGhidraAnalysisProfile } from "../dist/ghidra/GhidraAnalysisProfile.js";
import { GhidraClient } from "../dist/ghidra/GhidraClient.js";
import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraReleasePolicy.js";
import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraInstallationPolicy.js";
import { GhidraHeadlessLauncher } from "../dist/ghidra/GhidraLauncher.js";
import { GHIDRA_PROVIDER_IDENTITY } from "../dist/ghidra/GhidraProviderCapabilities.js";
+3 -9
View File
@@ -12,7 +12,6 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -84,7 +83,6 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await writeFile(path, source);
await client.connect(transport);
@@ -385,14 +383,10 @@ try {
"paged.vcd",
"recording with spaces.vcd",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
await client.close();
await transport.close();
await root.close();
}
console.log(
JSON.stringify(
-2
View File
@@ -44,7 +44,6 @@ def markdown_ids(path):
def check_repository_links(root):
"""Check local links in website Markdown and current-repository source links."""
root = root.resolve()
errors = []
sources = {}
for path in authored_markdown(root):
@@ -319,7 +318,6 @@ def check_search_metadata(site, pages):
def check_site(site):
"""Check assets, HTML fragment destinations and SVG syntax."""
site = site.resolve()
pages = {}
errors = []
references = 0
+29 -118
View File
@@ -23,7 +23,6 @@ import { pathToFileURL } from "node:url";
import { verifyWindowsProtocolInput } from "./lib/windows-protocol-input.mjs";
import { verifyWindowsJobMemoryLimit } from "./lib/windows-job-memory-limit.mjs";
import { verifyVolumeRootAppendLease } from "./lib/windows-volume-root-append-lease.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
if (process.platform !== "win32")
throw new Error(
@@ -67,49 +66,6 @@ const exists = (path) =>
() => true,
() => false,
);
const startDeleteSharingDeniedReader = async (path) => {
const child = spawn(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
"$ErrorActionPreference = 'Stop'; " +
"$stream = [IO.File]::Open($env:REA_CLEANUP_HELD_PATH, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite); " +
"try { [Console]::WriteLine('ready'); [Console]::In.ReadLine() | Out-Null } " +
"finally { $stream.Dispose() }",
],
{
env: { ...process.env, REA_CLEANUP_HELD_PATH: path },
stdio: ["pipe", "pipe", "pipe"],
},
);
const closed = once(child, "close");
void closed.catch(() => undefined);
try {
const [ready] = await once(child.stdout, "data", {
signal: AbortSignal.timeout(15_000),
});
assert.equal(String(ready).trim(), "ready");
} catch (cause) {
try {
child.kill();
await closed;
} catch (cleanupCause) {
throw new AggregateError(
[cause, cleanupCause],
"Reader startup and child cleanup failed",
{ cause },
);
}
throw cause;
}
return { child, closed };
};
const stopDeleteSharingDeniedReader = async ({ child, closed }) => {
child.kill();
await closed;
};
const live = (pid) => {
try {
process.kill(pid, 0);
@@ -560,14 +516,32 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
"cleanup-retry-",
);
const heldPath = join(retryRuntime.observation.path, "snapshot.bin");
let holder;
let primaryError;
await retryRuntime.snapshot(source, "snapshot.bin");
const holder = spawn(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
"$ErrorActionPreference = 'Stop'; " +
"$stream = [IO.File]::Open($env:REA_CLEANUP_HELD_PATH, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite); " +
"try { [Console]::WriteLine('ready'); [Console]::In.ReadLine() | Out-Null } " +
"finally { $stream.Dispose() }",
],
{
env: { ...process.env, REA_CLEANUP_HELD_PATH: heldPath },
stdio: ["pipe", "pipe", "pipe"],
},
);
const holderClosed = once(holder, "close");
void holderClosed.catch(() => undefined);
try {
await retryRuntime.snapshot(source, "snapshot.bin");
holder = await startDeleteSharingDeniedReader(heldPath);
const [ready] = await once(holder.stdout, "data", {
signal: AbortSignal.timeout(15_000),
});
assert.equal(String(ready).trim(), "ready");
// The reader denies delete sharing. Cleanup must retain ownership when
// its handle-relative open receives ERROR_SHARING_VIOLATION (32) through
// the bounded automatic retries; a later explicit close must succeed.
// its handle-relative open receives ERROR_SHARING_VIOLATION (32).
await assert.rejects(retryRuntime.close(), { win32Code: 32 });
assert.equal(
windowsPrivateRuntime(retryRuntime.observation.path),
@@ -578,8 +552,8 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
() => retryRuntime.writeFile("after-close.txt", "forbidden"),
/cleanup is pending/u,
);
holder.child.stdin.end("\n");
const [exitCode] = await holder.closed;
holder.stdin.end("\n");
const [exitCode] = await holderClosed;
assert.equal(exitCode, 0);
await retryRuntime.close();
await retryRuntime.close();
@@ -589,73 +563,10 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
/No native private runtime/u,
);
report.controls.cleanupFailureRetainsOwnershipForRetry = true;
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(holder === undefined
? []
: [() => stopDeleteSharingDeniedReader(holder)]),
() => retryRuntime.close(),
],
primaryError,
);
}
const transientRuntime = WindowsPrivateRuntime.create(
workspace,
"cleanup-transient-",
);
const transientHeldPath = join(
transientRuntime.observation.path,
"snapshot.bin",
);
let transientHolder;
let transientPrimaryError;
try {
await transientRuntime.snapshot(source, "snapshot.bin");
transientHolder = await startDeleteSharingDeniedReader(transientHeldPath);
// Keep the reader held through the first close attempt. Without retries,
// close rejects before the next event-loop turn; with retries, it remains
// pending until the caller releases this sharing conflict.
let closeSettled = false;
const closing = transientRuntime.close();
void closing.then(
() => {
closeSettled = true;
},
() => {
closeSettled = true;
},
);
await new Promise((resolve) => setImmediate(resolve));
assert.equal(closeSettled, false);
transientHolder.child.stdin.end("\n");
const [, [exitCode]] = await Promise.all([
closing,
transientHolder.closed,
]);
assert.equal(exitCode, 0);
assert.equal(await exists(transientRuntime.observation.path), false);
assert.throws(
() => windowsPrivateRuntime(transientRuntime.observation.path),
/No native private runtime/u,
);
report.controls.cleanupTransientConflictRetries = true;
} catch (cause) {
transientPrimaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(transientHolder === undefined
? []
: [() => stopDeleteSharingDeniedReader(transientHolder)]),
() => transientRuntime.close(),
],
transientPrimaryError,
);
holder.kill();
await holderClosed.catch(() => undefined);
await retryRuntime.close();
}
report.controls = {
...report.controls,
+1 -4
View File
@@ -14,7 +14,6 @@ import { join } from "node:path";
import { parse as parseJsonc } from "jsonc-parser";
import { supportedClients } from "../dist/application/SupportedClients.js";
import { promisify } from "node:util";
import { GHIDRA_BRIDGE_SOURCES } from "../src/ghidra/GhidraBridgeSources.ts";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
@@ -53,9 +52,7 @@ try {
);
const packagedPaths = new Set(packageResult.files.map(({ path }) => path));
for (const required of [
...Object.values(GHIDRA_BRIDGE_SOURCES).map(
(name) => `bridge/ghidra/${name}`,
),
"bridge/ghidra/ReaGhidraBridge.java",
"native/windows/build/manifest.json",
"native/windows/build/rea-windows-x64.node",
"dist/android/JadxWorkspaceRoot.js",
@@ -1,482 +0,0 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { createHash, randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { readFile, writeFile } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { performance } from "node:perf_hooks";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createEvidence } from "../../../dist/domain/evidence.js";
import { PrivateRuntimeRoot } from "../../../dist/process/PrivateRuntimeRoot.js";
import {
OwnedCommandFailure,
runOwnedCommand,
} from "../../../dist/process/OwnedCommand.js";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import {
matchesStructuredScopeExpectation,
structuredScopeExpectationsSchema,
} from "../../lib/javascript-file-scope-oracle.mjs";
const { values } = parseArgs({
options: {
artifact: { type: "string" },
"addon-path": { type: "string" },
"native-sha256": { type: "string" },
"source-path": { type: "string", multiple: true },
"structured-path": { type: "string", multiple: true },
"structured-expectations": { type: "string" },
"validate-structured-from-export": { type: "boolean", default: false },
"analysis-timeout-ms": { type: "string" },
"expect-timeout": { type: "boolean", default: false },
rounds: { type: "string", default: "3" },
"cli-parity": { type: "boolean", default: false },
"tokenizer-python": { type: "string" },
"require-safe": { type: "boolean", default: false },
},
strict: true,
});
assert.ok(
values.artifact && isAbsolute(values.artifact),
"--artifact must be absolute",
);
assert.ok(
values["addon-path"] &&
!isAbsolute(values["addon-path"]) &&
values["addon-path"]
.split("/")
.every((part) => part !== "" && part !== "." && part !== ".."),
"--addon-path must be a normalized archive-relative path",
);
assert.match(
values["native-sha256"] ?? "",
/^[a-f0-9]{64}$/,
"--native-sha256 pins prior native bytes; it does not prove availability",
);
const rounds = Number(values.rounds);
assert.ok(Number.isSafeInteger(rounds) && rounds > 0);
const analysisTimeoutMs =
values["analysis-timeout-ms"] === undefined
? undefined
: Number(values["analysis-timeout-ms"]);
assert.ok(
analysisTimeoutMs === undefined ||
(Number.isSafeInteger(analysisTimeoutMs) && analysisTimeoutMs > 0),
);
assert.ok(!values["expect-timeout"] || analysisTimeoutMs !== undefined);
assert.ok(!values["validate-structured-from-export"] || values["cli-parity"]);
const structuredExpectations = structuredScopeExpectationsSchema.parse(
JSON.parse(values["structured-expectations"] ?? "{}"),
);
for (const path of Object.keys(structuredExpectations))
assert.ok(
values["structured-path"]?.includes(path),
`Expected structured path was not selected for verification: ${path}`,
);
if (values["expect-timeout"])
for (const path of values["structured-path"] ?? [])
assert.ok(
Object.hasOwn(structuredExpectations, path),
`--structured-expectations must name the expected operation/status for ${path}`,
);
const environment = {
PATH: process.env.PATH ?? "/usr/bin:/bin",
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "auto",
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [fileURLToPath(new URL("../../rea.mjs", import.meta.url)), "mcp"],
env: environment,
stderr: "pipe",
});
const client = new Client({
name: "javascript-native-handoff-verifier",
version: "1",
});
const serialize = (value) => JSON.stringify(value);
const digest = (value) =>
createHash("sha256").update(serialize(value)).digest("hex");
const fileDigest = async (path) => {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk);
return hash.digest("hex");
};
const simulatedNativeEvidence = (path, sha256) =>
createEvidence(
{ path, sha256, format: "raw-binary" },
{ id: "ghidra", name: "Ghidra", version: "simulated" },
{
operation: "binary_overview",
parameters: {},
result: {
summary:
"Simulated subject-linkage input; no native provider executed.",
},
limitations: ["No native provider behavior is established."],
},
);
let temporary;
let primaryError;
let retainTemporary = false;
let report;
const ownedCommand = async (command, arguments_, timeoutMs = 60_000) => {
try {
return await runOwnedCommand(
{
command,
arguments: arguments_,
cwd: temporary.path,
env: environment,
runId: `rea-native-handoff-${randomUUID()}`,
expectedCommand: command === process.execPath ? process.execPath : null,
sweepTokenOwnedProcesses: true,
},
{ timeoutMs, diagnosticBytes: 16 * 1024 * 1024 },
);
} catch (cause) {
retainTemporary ||=
cause instanceof OwnedCommandFailure && cause.cleanupOwner !== undefined;
throw cause;
}
};
try {
const archiveSha256 = await fileDigest(values.artifact);
const companion = join(`${values.artifact}.unpacked`, values["addon-path"]);
let companionSha256 = null;
try {
companionSha256 = await fileDigest(companion);
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
}
if (companionSha256 !== null)
assert.equal(companionSha256, values["native-sha256"]);
const native = simulatedNativeEvidence(companion, values["native-sha256"]);
const container = simulatedNativeEvidence(values.artifact, archiveSha256);
await client.connect(transport);
transport.stderr?.on("data", () => undefined);
const tools = (await client.listTools()).tools;
const catalog = tools.map(({ name, description, inputSchema }) => ({
name,
description,
inputSchema,
}));
const selected = catalog.filter(({ name }) =>
["analyze_javascript_application", "trace_application_feature"].includes(
name,
),
);
assert.equal(selected.length, 2);
const call = async (name, arguments_, allowTimeout = false) => {
const response = await client.callTool(
{ name, arguments: arguments_ },
{ timeout: 360_000 },
);
let projection = response.structuredContent;
if (response.isError === true) {
assert.ok(
allowTimeout,
serialize({ name, arguments: arguments_, response }),
);
const text = response.content.find(({ type }) => type === "text");
assert.ok(text?.type === "text");
projection = JSON.parse(text.text);
assert.equal(projection.error.code, "provider_timeout");
assert.equal(
projection.error.details.partial_observation.kind,
"retained-evidence",
);
} else assert.ok(projection?.evidence_id);
return {
request: { name, arguments: arguments_ },
response: projection,
wire_response: response,
};
};
const prepStart = performance.now();
const preparation = await call(
"analyze_javascript_application",
{
input_path: values.artifact,
detail: "summary",
...(analysisTimeoutMs === undefined
? {}
: { analysis_timeout_ms: analysisTimeoutMs }),
...(values["source-path"] === undefined
? {}
: { source_paths: values["source-path"] }),
},
values["expect-timeout"],
);
const preparationMs = performance.now() - prepStart;
const timedOut = preparation.wire_response.isError === true;
assert.equal(timedOut, values["expect-timeout"]);
if (!timedOut)
assert.equal(preparation.response.subject.digest.sha256, archiveSha256);
const sourceId = timedOut
? preparation.response.error.details.partial_observation.evidence_id
: preparation.response.normalized_result.parent_evidence_id;
assert.match(sourceId, /^ev_[a-f0-9]{64}$/);
const query = {
application: { kind: "retained-evidence", evidence_id: sourceId },
native_observations: [container, native],
seed: {
kind: "string",
value: values["addon-path"],
match: "exact",
case_sensitive: true,
},
direction: "outgoing",
};
const samples = [];
let first;
for (let round = 0; round < rounds; round++) {
const start = performance.now();
const result = await call("trace_application_feature", query);
const elapsedMs = performance.now() - start;
if (first === undefined) first = result;
else assert.equal(result.response.evidence_id, first.response.evidence_id);
samples.push({
round,
elapsed_ms: elapsedMs,
structured_reply_bytes: Buffer.byteLength(serialize(result.response)),
full_tool_reply_bytes: Buffer.byteLength(serialize(result.wire_response)),
});
}
const trace = first.response.normalized_result;
const addons = trace.graph.nodes.filter(
({ kind }) => kind === "native-addon",
);
assert.equal(
addons.length,
1,
"The selected native add-on must remain visible",
);
const addon = addons[0];
const unavailable = addon.observations.some(
({ properties }) => properties.availability === "unavailable",
);
const expectedLinks = [
sourceId,
...(companionSha256 === null ? [] : [native.evidence_id]),
].sort();
const safeIdentity =
serialize(trace.evidence_links) === serialize(expectedLinks) &&
(companionSha256 === null
? unavailable &&
addon.identity.strategy !== "content-digest" &&
trace.coverage.status === "partial" &&
trace.native_handoffs.length === 0
: addon.identity.strategy === "content-digest" &&
addon.identity.sha256 === companionSha256 &&
trace.native_handoffs.length === 1 &&
trace.native_handoffs[0].artifact_sha256 === companionSha256 &&
trace.native_handoffs[0].status === "evidence-linked" &&
serialize(trace.native_handoffs[0].evidence_ids) ===
serialize([native.evidence_id]));
const analysisScopes = (result, path) =>
result.graph.nodes.flatMap((node) =>
node.kind !== "unknown"
? []
: node.observations.flatMap((observation) =>
observation.properties.path === path &&
observation.properties.analysis_failure !== undefined
? [{ node_id: node.node_id, ...observation }]
: [],
),
);
const nativeScopes = analysisScopes(trace, values["addon-path"]);
const structuredQueries = [];
const structuredScopes = [];
for (const path of values["validate-structured-from-export"]
? []
: (values["structured-path"] ?? [])) {
const result = await call("trace_application_feature", {
...query,
native_observations: [],
seed: { ...query.seed, value: path },
});
structuredQueries.push(result);
structuredScopes.push({
source: "retained-feature-trace",
path,
scopes: analysisScopes(result.response.normalized_result, path),
evidence_id: result.response.evidence_id,
source_coverage: result.response.normalized_result.coverage,
graph_coverage: result.response.normalized_result.graph.coverage,
});
}
let cliParity = null;
if (values["cli-parity"]) {
temporary = await PrivateRuntimeRoot.create({
prefix: "rea-native-handoff-verifier-",
});
const bundlePath = join(temporary.path, "bundle.json");
const exported = await client.callTool({
name: "export_evidence_bundle",
arguments: { path: bundlePath },
});
assert.notEqual(exported.isError, true, serialize(exported));
const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
const parent = bundle.records.find(
({ evidence_id }) => evidence_id === sourceId,
);
assert.ok(parent, "Export omitted the retained parent");
assert.equal(parent.normalized_result.root_artifact_sha256, archiveSha256);
if (values["validate-structured-from-export"])
for (const path of values["structured-path"] ?? []) {
const retained = parent.normalized_result;
const matchingNodes = retained.graph.nodes.filter((node) =>
node.observations.some(({ properties }) => properties.path === path),
);
assert.ok(matchingNodes.length > 0, `Missing retained path: ${path}`);
// Diagnostic validation of the complete backend record is distinct
// from the inline native analyst task used by this control's tokens.
structuredScopes.push({
source: "exported-parent-validation",
path,
scopes: analysisScopes(retained, path),
source_evidence_id: parent.evidence_id,
retained_matching_nodes: matchingNodes.length,
graph_coverage: retained.graph.coverage,
});
}
const request = join(temporary.path, "trace.json");
// The CLI's portable file contract takes complete Evidence; MCP resolves a
// same-session reference into the same shared application workflow.
await writeFile(request, serialize({ ...query, application: parent }));
const result = await ownedCommand(
process.execPath,
[
fileURLToPath(new URL("../../rea.mjs", import.meta.url)),
"trace-application-feature",
request,
"--json",
],
120_000,
);
const evidence = JSON.parse(result.stdout.text);
assert.deepEqual(evidence.normalized_result, trace);
assert.equal(evidence.evidence_id, first.response.evidence_id);
cliParity = {
exact_cli_mcp_trace_parity: true,
evidence_id: evidence.evidence_id,
};
}
const safeApplicability =
nativeScopes.length === 0 &&
structuredScopes.every(({ path, scopes }) =>
timedOut
? matchesStructuredScopeExpectation(
scopes,
structuredExpectations[path],
)
: scopes.length === 0,
);
let tokenCosts = null;
if (values["tokenizer-python"]) {
temporary ??= await PrivateRuntimeRoot.create({
prefix: "rea-native-handoff-verifier-",
});
const input = join(temporary.path, "token-input.json");
await writeFile(
input,
serialize({
catalog,
selected,
preparation,
queries: [first, ...structuredQueries],
}),
);
const result = await ownedCommand(values["tokenizer-python"], [
"-c",
`import json,sys,tiktoken
with open(sys.argv[1],encoding='utf-8') as f:d=json.load(f)
enc=tiktoken.get_encoding('cl100k_base')
def n(v):return len(enc.encode(json.dumps(v,ensure_ascii=False,separators=(',',':')),disallowed_special=()))
requests=n(d['preparation']['request'])+sum(n(q['request']) for q in d['queries'])
structured=n(d['preparation']['response'])+sum(n(q['response']) for q in d['queries'])
full_reply=n(d['preparation']['wire_response'])+sum(n(q['wire_response']) for q in d['queries'])
s=n(d['selected']);a=n(d['catalog'])
print(json.dumps(dict(tokenizer='cl100k_base',tiktoken_version=tiktoken.__version__,selected_catalog=s,full_catalog=a,requests=requests,structured_replies=structured,full_tool_replies=full_reply,selected_catalog_structured_task=s+requests+structured,full_catalog_structured_task=a+requests+structured,selected_catalog_full_reply_task=s+requests+full_reply,full_catalog_full_reply_task=a+requests+full_reply)))`,
input,
]);
tokenCosts = JSON.parse(result.stdout.text);
}
report = {
scope:
"Real shipped-ASAR native identity through public CLI/stdio MCP; simulated native Evidence only, no provider/application executed",
artifact: { path: values.artifact, sha256: archiveSha256 },
addon_path: values["addon-path"],
companion_sha256_observed: companionSha256,
prior_native_sha256_supplied: values["native-sha256"],
source_paths: values["source-path"] ?? null,
preparation_outcome: timedOut ? "provider_timeout" : "completed",
analysis_timeout_ms: analysisTimeoutMs ?? null,
parent_evidence_id: sourceId,
trace_evidence_id: first.response.evidence_id,
trace_subject: first.response.subject,
preparation_ms: preparationMs,
samples,
safe_identity: safeIdentity,
safe_applicability: safeApplicability,
native_analysis_scopes: nativeScopes,
structured_scopes: structuredScopes,
structured_expectations: structuredExpectations,
modeled_structured_trace_count: structuredQueries.length,
node: {
node_id: addon.node_id,
identity: addon.identity,
unavailable_observed: unavailable,
basis: addon.observations.map(({ evidence }) => ({
authority: evidence.authority,
state: evidence.state,
confidence: evidence.confidence,
artifact: evidence.artifact,
location: evidence.location,
})),
},
native_handoffs: trace.native_handoffs,
evidence_links: trace.evidence_links,
source_coverage: trace.coverage,
graph_coverage: trace.graph.coverage,
facts_sha256: digest({
node: addon,
native_handoffs: trace.native_handoffs,
evidence_links: trace.evidence_links,
coverage: trace.coverage,
}),
cli_parity: cliParity,
token_costs: tokenCosts,
conditions:
"Fresh MCP session, selected analysis followed by retained traces. Task token counts include one analysis summary or parsed timeout error, one native trace and each requested structured-path trace, with name/description/inputSchema catalog projections. Structured Evidence/parsed error and complete-tool-reply projections are alternatives, not billing. Repeated native rounds, export, CLI parity and tokenizer are validation overhead outside task tokens but inside total resource receipts. Filesystem/build caches uncontrolled. Incorrect identity/applicability is not a successful equivalent task or efficiency credit.",
};
if (values["require-safe"]) {
assert.ok(safeIdentity, "Native identity/linkage oracle failed");
assert.ok(safeApplicability, "File analysis applicability oracle failed");
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
],
primaryError,
);
}
console.log(serialize(report));
-63
View File
@@ -880,42 +880,6 @@ exit 1`,
});
});
it("binds a push digest to the bytes adb consumes after caller-path replacement", async () => {
const { binary, root } = await writeAdbStub(
(stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
if [ "$1" = "-s" ] && [ "$4" = "test" ]; then exit 1; fi
if [ "$1" = "-s" ] && [ "$3" = "push" ]; then
printf 'replacement-payload' > "${stubRoot}/payload.bin"
cat "$4" > "${stubRoot}/consumed.bin" || exit 1
exit 0
fi
if [ "$1" = "-s" ] && [ "$4" = "sha256sum" ]; then exit 1; fi
exit 1`,
);
const localFile = join(root, "payload.bin");
await writeFile(localFile, "original-payload");
const result = await evidenceResult(providerOver(binary), {
operation: "push_adb_file",
input: {
serial: "emulator-5554",
local_path: localFile,
device_path: "/data/local/tmp/payload.bin",
overwrite: false,
},
});
const consumedDigest = createHash("sha256")
.update(await readFile(join(root, "consumed.bin")))
.digest("hex");
expect(result).toMatchObject({
local_path: localFile,
sha256: consumedDigest,
});
expect(await readFile(localFile, "utf8")).toBe("replacement-payload");
expect(await readFile(join(root, "consumed.bin"), "utf8")).toBe(
"original-payload",
);
});
it("reports a digest mismatch after push as a changed artifact", async () => {
const { binary, root } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
@@ -1315,33 +1279,6 @@ exit 1`,
expect(projected.message).toContain("INSTALL_FAILED_ALREADY_EXISTS");
});
it("binds an install digest to the bytes adb consumes after caller-path replacement", async () => {
const { binary, root } = await writeAdbStub(
(stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
if [ "$1" = "-s" ] && [ "$3" = "install" ]; then
printf 'replacement-apk' > "${stubRoot}/app.apk"
cat "$4" > "${stubRoot}/consumed.bin" || exit 1
printf 'Success\\n'
exit 0
fi
exit 1`,
);
const apk = join(root, "app.apk");
await writeFile(apk, "original-apk");
const result = await evidenceResult(providerOver(binary), {
operation: "install_adb_package",
input: { serial: "emulator-5554", apk_path: apk, replace: false },
});
const consumedDigest = createHash("sha256")
.update(await readFile(join(root, "consumed.bin")))
.digest("hex");
expect(result).toMatchObject({ apk_path: apk, sha256: consumedDigest });
expect(await readFile(apk, "utf8")).toBe("replacement-apk");
expect(await readFile(join(root, "consumed.bin"), "utf8")).toBe(
"original-apk",
);
});
it("uninstalls a package and reports the device's refusal for absent ones", async () => {
const { binary } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
+269 -568
View File
File diff suppressed because it is too large. Load diff
+32 -93
View File
@@ -14,7 +14,6 @@ import {
} from "../domain/analysisErrorCore.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import type { AndroidRequest } from "../domain/android/androidAnalysis.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
@@ -27,15 +26,8 @@ import {
snapshotAndroidEngine,
snapshotAndroidTarget,
} from "./AndroidTargetSnapshot.js";
import {
hashStableFile,
StableFileHashCleanupFailure,
} from "../filesystem/StableFileHash.js";
import {
OwnedRegularFileCleanupFailure,
RegularFileChangedError,
} from "../filesystem/RegularFile.js";
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { hashStableFile } from "../filesystem/StableFileHash.js";
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
import {
inspectJadxAvailability,
resolveJadxConfigurationInputs,
@@ -74,38 +66,8 @@ const executionError = (context: {
session: JadxSession | undefined;
}): AnalysisError => {
const { cause, request, timeout, signal, session } = context;
if (cause instanceof OwnedRegularFileCleanupFailure) {
const primary =
signal?.aborted === true
? new AnalysisCancelledError(request.operation, { cause: cause.cause })
: timeout.aborted
? new AnalysisTimeoutError(request.operation, OPERATION_TIMEOUT_MS)
: new ProviderAdapterError("jadx", request.operation, {
cause: cause.cause,
userMessage:
cause.cause instanceof Error
? cause.cause.message
: String(cause.cause),
});
const cleanup = new ProviderCleanupError(
"jadx",
[cause.path],
{
reason:
cause.cleanupCause instanceof Error
? cause.cleanupCause.message
: String(cause.cleanupCause),
...(cause instanceof StableFileHashCleanupFailure &&
cause.completedHash !== undefined
? { completed_hash: cause.completedHash }
: {}),
},
{ operation: request.operation, cause: cause.cleanupCause },
);
return analysisErrorWithCleanupFailure(primary, cleanup, request.operation);
}
if (signal?.aborted === true)
return new AnalysisCancelledError(request.operation, { cause });
return new AnalysisCancelledError(request.operation);
if (timeout.aborted)
return new AnalysisTimeoutError(request.operation, OPERATION_TIMEOUT_MS);
if (cause instanceof AnalysisError) return cause;
@@ -194,73 +156,57 @@ const hashAndroidSource = async (
interface PendingCleanup {
session: JadxSession | undefined;
root: JadxWorkspaceRoot | undefined;
readonly fileHandles: Map<
OwnedFileHandle,
{ readonly path: string; readonly error: AnalysisError }
>;
readonly previousError: AnalysisError | undefined;
}
const cleanup = async (
resources: PendingCleanup,
): Promise<Result<void, AnalysisError>> => {
const { session, root, previousError, fileHandles } = resources;
const { session, root, previousError } = resources;
const diagnostics = {
previous_error:
previousError === undefined ? null : projectAnalysisError(previousError),
};
const failures: { readonly resource: string; readonly cause: unknown }[] = [];
for (const [owner, { path, error }] of fileHandles) {
try {
await owner.close();
error.resolveCleanup();
fileHandles.delete(owner);
} catch (cause: unknown) {
failures.push({ resource: path, cause });
}
}
try {
await session?.close();
resources.session = undefined;
} catch (cause) {
failures.push({ resource: session?.transport.runId ?? "unknown", cause });
}
if (resources.session === undefined && failures.length === 0) {
try {
await root?.close();
resources.root = undefined;
} catch (cause: unknown) {
failures.push({ resource: root?.path ?? "unknown", cause });
}
}
if (failures.length > 0) {
const cleanupCause = new AggregateError(
failures.map(({ cause }) => cause),
"JADX file, session, or workspace cleanup failed",
);
// Retain an uncertain workspace until the caller can resolve process ownership.
const error = new ProviderCleanupError(
"jadx",
failures.map(({ resource }) => resource),
[
session?.transport.runId ?? "unknown",
...(root === undefined ? [] : [root.path]),
],
{
...diagnostics,
reason: failures
.map(({ cause }) =>
cause instanceof Error ? cause.message : String(cause),
)
.join("; "),
provider_cleanup: failures.map(({ cause }) =>
reason: cause instanceof Error ? cause.message : String(cause),
provider_cleanup:
cause instanceof ProviderCleanupError
? (cause.diagnostics ?? null)
: null,
),
},
{
cause: previousError ?? cleanupCause,
...(previousError?.partialObservation === undefined
? {}
: { partialObservation: previousError.partialObservation }),
},
{ cause: previousError ?? cause },
);
if (previousError?.partialObservation !== undefined)
error.retainPartialObservation(previousError.partialObservation);
return err(error);
}
try {
await root?.close();
resources.root = undefined;
} catch (cause) {
const error = new ProviderCleanupError(
"jadx",
[root?.path ?? "unknown"],
{
...diagnostics,
reason: cause instanceof Error ? cause.message : String(cause),
},
{ cause: previousError ?? cause },
);
if (previousError?.partialObservation !== undefined)
error.retainPartialObservation(previousError.partialObservation);
return err(error);
}
return ok(undefined);
@@ -353,7 +299,6 @@ export class JadxProvider implements AndroidAnalysisPort {
this.#pendingCleanup = {
session: this.#retained.session,
root: this.#retained.root,
fileHandles: new Map(),
previousError: undefined,
};
this.#retained = undefined;
@@ -533,12 +478,6 @@ export class JadxProvider implements AndroidAnalysisPort {
signal: options?.signal,
session,
});
const fileHandles: PendingCleanup["fileHandles"] = new Map();
if (cause instanceof OwnedRegularFileCleanupFailure)
fileHandles.set(cause.owner, {
path: cause.path,
error: previousError,
});
const partialObservation = session?.partialObservation();
if (partialObservation !== undefined)
previousError.retainPartialObservation(partialObservation);
@@ -546,7 +485,7 @@ export class JadxProvider implements AndroidAnalysisPort {
root ??= this.#retained?.root;
session ??= this.#retained?.session;
this.#retained = undefined;
this.#pendingCleanup = { session, root, fileHandles, previousError };
this.#pendingCleanup = { session, root, previousError };
const retired = await this.#retire();
return retired.ok ? err(previousError) : retired;
}
-4
View File
@@ -290,8 +290,4 @@ export interface AnalysisProviderCandidate extends AnalysisProvider {
inspectAvailability(): ProviderAvailability;
inspectReadinessSnapshot?(): AnalysisProviderReadinessSnapshot;
inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport;
/** Report retained candidate-owned cleanup without starting work or retrying it. */
hasPendingOwnedResources?(): boolean;
/** Release candidate-owned resources that may exist without a client. */
closeOwnedResources?(): Promise<Result<null, AnalysisError>>;
}
@@ -32,46 +32,3 @@ describe("app bundle Info.plist magic", () => {
}
});
});
describe("app bundle program directory containment", () => {
it("rejects an escaping programs directory before resolving its child", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-app-programs-"));
try {
const bundle = join(root, "Fixture.app");
const programs = join(bundle, "Contents", "MacOS");
const executable = join(programs, "Missing");
const canonicalBundle = join(root, "Canonical.app");
const outsidePrograms = join(root, "outside");
const plist =
"<plist><dict><key>CFBundleExecutable</key><string>Missing</string></dict></plist>";
await mkdir(join(bundle, "Contents"), { recursive: true });
await writeFile(join(bundle, "Contents", "Info.plist"), plist);
let childResolved = false;
const result = await resolveAppBundleExecutable(bundle, {
readFile: async () => Buffer.from(plist),
realpath: async (path) => {
if (path === bundle) return canonicalBundle;
if (path === programs) return outsidePrograms;
if (path === executable) {
childResolved = true;
throw Object.assign(new Error("missing executable"), {
code: "ENOENT",
});
}
return path;
},
});
expect(result).toMatchObject({
ok: false,
error: {
reason: "app program directory leaves Contents/MacOS",
},
});
expect(childResolved).toBe(false);
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
+4 -17
View File
@@ -219,24 +219,11 @@ const resolveLayoutExecutable = async (
);
const programs = join(bundle, ...layout.programs);
const executable = join(programs, name);
let canonicalBundle: string;
try {
canonicalBundle = await fileSystem.realpath(bundle);
} catch (cause: unknown) {
return err(
new BinaryTargetError(bundle, errorReason(cause, bundle), { cause }),
);
}
try {
const canonicalPrograms = await fileSystem.realpath(programs);
if (!isPathWithinRoot(canonicalBundle, canonicalPrograms))
return err(
new BinaryTargetError(
bundle,
`app program directory leaves ${layout.programsLabel}`,
),
);
const canonicalExecutable = await fileSystem.realpath(executable);
const [canonicalPrograms, canonicalExecutable] = await Promise.all([
fileSystem.realpath(programs),
fileSystem.realpath(executable),
]);
if (!isPathWithinRoot(canonicalPrograms, canonicalExecutable))
return err(
new BinaryTargetError(
@@ -35,8 +35,6 @@ import type { ExecutionOptions } from "./AnalysisProvider.js";
/** Provider-neutral application boundary for passive browser observation. */
export interface BrowserObservationPort {
/** Release provider-owned transports during command or server shutdown. */
close?(): Promise<void>;
identity(): ProviderIdentity;
listTargets(
input: ListBrowserTargetsInput,
@@ -1,161 +0,0 @@
import { expect, it } from "vitest";
import type { BrowserObservationPort } from "./BrowserObservationPort.js";
import { listBrowserTargets } from "./BrowserObservationService.js";
import type { BrowserTargetList } from "../domain/browserObservationSchemas.js";
import { parseEvidence } from "../domain/evidence.js";
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
import { BrowserObservationError } from "../domain/browserObservationError.js";
import { ok } from "../domain/result.js";
import { passiveNetworkCapture } from "../../tests/fixtures/passiveNetworkCapture.js";
import { inspectWebPage } from "./BrowserObservationService.js";
import { analyzeWebBundleInputSchema } from "../domain/webBundleAnalysis.js";
import { inspectWebPageInputSchema } from "../domain/browserObservation.js";
import { analyzeWebBundle } from "./BrowserObservationService.js";
const input = {
cdp_endpoint: {
transport: "http_discovery" as const,
url: "http://127.0.0.1:9222",
recorded: "http://127.0.0.1:9222",
},
allowed_origins: [],
};
const report: BrowserTargetList = {
browser: {
product: "Chrome",
protocol_version: "1.3",
revision: "1",
user_agent: "Chrome fixture",
js_version: "V8 fixture",
},
targets: [],
excluded: { disallowed_origin: 0, unsupported_url: 0, non_page: 0 },
limitations: [],
};
const provider = (
listTargets: BrowserObservationPort["listTargets"],
): BrowserObservationPort =>
({
identity: () => ({ id: "browser-fixture", name: "Fixture", version: "1" }),
listTargets,
}) as unknown as BrowserObservationPort;
it("retains completed browser inventory Evidence when the provider completes after abort", async () => {
const controller = new AbortController();
const result = await listBrowserTargets(
provider(async () => {
controller.abort();
return ok(report);
}),
input,
{ signal: controller.signal },
);
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error._tag).toBe("AnalysisCancelledError");
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "list_browser_targets",
normalized_result: report,
});
}
});
it("does not let late cancellation hide invalid completed browser output", async () => {
const malformed = {
...report,
limitations: [() => undefined],
} as unknown as BrowserTargetList;
const output = await listBrowserTargets(
provider(async () => ok(malformed)),
input,
);
expect(output).toMatchObject({
ok: false,
error: { _tag: "AnalysisOutputError" },
});
if (!output.ok) expect(output.error).toBeInstanceOf(AnalysisOutputError);
const controller = new AbortController();
const late = await listBrowserTargets(
provider(async () => {
controller.abort();
return ok(malformed);
}),
input,
{ signal: controller.signal },
);
expect(late).toMatchObject({
ok: false,
error: { _tag: "AnalysisOutputError" },
});
if (!late.ok) expect(late.error).toBeInstanceOf(AnalysisOutputError);
});
it("converts a raw page inspection retained after browser cleanup failure", async () => {
const inspection = passiveNetworkCapture();
const failure = new BrowserObservationError(
"inspect_web_page",
"cleanup_failed",
{
cleanup: {
reason: "target detach failed",
resources: ["browser target"],
},
partialObservation: inspection,
},
);
const browser: BrowserObservationPort = {
identity: () => ({ id: "browser-fixture", name: "Fixture", version: "1" }),
listTargets: async () => ok(report),
inspectPage: async () => ({ ok: false, error: failure }),
} as unknown as BrowserObservationPort;
const pageInput = inspectWebPageInputSchema.parse({
cdp_endpoint: input.cdp_endpoint.recorded,
allowed_origins: [],
target_id: "page",
observation_ms: 0,
include_script_sources: false,
});
const result = await inspectWebPage(browser, pageInput);
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toBe(failure);
expect(result.error.cleanupIncomplete).toBe(true);
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "inspect_web_page",
normalized_result: inspection,
});
}
});
it("retains captured page Evidence when bundle analysis fails after capture", async () => {
const inspection = passiveNetworkCapture();
const failure = new BrowserObservationError(
"analyze_web_bundle",
"cancelled",
{ partialObservation: inspection },
);
const browser = {
identity: () => ({ id: "browser-fixture", name: "Fixture", version: "1" }),
analyzeBundle: async () => ({ ok: false as const, error: failure }),
} as unknown as BrowserObservationPort;
const bundleInput = analyzeWebBundleInputSchema.parse({
...input,
cdp_endpoint: input.cdp_endpoint.recorded,
target_id: "page",
observation_ms: 0,
include_script_sources: true,
});
const result = await analyzeWebBundle(browser, bundleInput);
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toBe(failure);
expect(failure.operation).toBe("analyze_web_bundle");
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "inspect_web_page",
normalized_result: inspection,
});
}
});
+54 -131
View File
@@ -14,16 +14,12 @@ import type {
import {
AnalysisCancelledError,
AnalysisCapabilityUnavailableError,
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { err, ok, type Result } from "../domain/result.js";
import type { ExecutionOptions } from "./AnalysisProvider.js";
import type { BrowserObservationPort } from "./BrowserObservationPort.js";
import { createBrowserEvidence } from "./BrowserEvidence.js";
import { jsonValueSchema } from "../domain/jsonValue.js";
import { completeProviderEvidence } from "./ProviderOperationExecution.js";
import { webPageInspectionSchema } from "../domain/browserObservationSchemas.js";
/** List browser targets within the endpoint and origin scope in the request. */
export const listBrowserTargets = async (
@@ -33,16 +29,17 @@ export const listBrowserTargets = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "list_browser_targets");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("list_browser_targets"));
const result = await ready.value.listTargets(input, options);
if (result.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("list_browser_targets"));
return result.ok
? browserEvidence(
"list_browser_targets",
input,
result.value,
ready.value.identity(),
options,
? ok(
createBrowserEvidence(
"list_browser_targets",
input,
result.value,
ready.value.identity(),
),
)
: result;
};
@@ -55,23 +52,17 @@ export const inspectWebPage = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "inspect_web_page");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("inspect_web_page"));
const result = await ready.value.inspectPage(input, options);
if (!result.ok)
return retainPageInspection(
result.error,
input,
ready.value,
"inspect_web_page",
);
if (result.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("inspect_web_page"));
return result.ok
? browserEvidence(
"inspect_web_page",
input,
result.value,
ready.value.identity(),
options,
? ok(
createBrowserEvidence(
"inspect_web_page",
input,
result.value,
ready.value.identity(),
),
)
: result;
};
@@ -84,22 +75,17 @@ export const analyzeWebBundle = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "analyze_web_bundle");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("analyze_web_bundle"));
const analyzed = await ready.value.analyzeBundle(input, options);
if (!analyzed.ok)
return retainPageInspection(
analyzed.error,
input,
ready.value,
if (analyzed.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("analyze_web_bundle"));
if (!analyzed.ok) return analyzed;
return ok(
createBrowserEvidence(
"analyze_web_bundle",
);
return browserEvidence(
"analyze_web_bundle",
input,
analyzed.value,
ready.value.identity(),
options,
input,
analyzed.value,
ready.value.identity(),
),
);
};
@@ -111,16 +97,17 @@ export const observeWebSession = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "observe_web_session");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("observe_web_session"));
const observed = await ready.value.observeSession(input, options);
if (observed.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("observe_web_session"));
return observed.ok
? browserEvidence(
"observe_web_session",
input,
observed.value,
ready.value.identity(),
options,
? ok(
createBrowserEvidence(
"observe_web_session",
input,
observed.value,
ready.value.identity(),
),
)
: observed;
};
@@ -133,16 +120,17 @@ export const discoverWebMcpTools = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "discover_webmcp_tools");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("discover_webmcp_tools"));
const discovered = await ready.value.discoverWebMcpTools(input, options);
if (discovered.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("discover_webmcp_tools"));
return discovered.ok
? browserEvidence(
"discover_webmcp_tools",
input,
discovered.value,
ready.value.identity(),
options,
? ok(
createBrowserEvidence(
"discover_webmcp_tools",
input,
discovered.value,
ready.value.identity(),
),
)
: discovered;
};
@@ -175,16 +163,17 @@ export const captureWebScreenshot = async (
): Promise<Result<Evidence, AnalysisError>> => {
const ready = requireBrowser(browser, "capture_web_screenshot");
if (!ready.ok) return ready;
if (options.signal?.aborted)
return err(new AnalysisCancelledError("capture_web_screenshot"));
const captured = await ready.value.captureScreenshot(input, options);
if (captured.ok && options.signal?.aborted === true)
return err(new AnalysisCancelledError("capture_web_screenshot"));
return captured.ok
? browserEvidence(
"capture_web_screenshot",
input,
captured.value,
ready.value.identity(),
options,
? ok(
createBrowserEvidence(
"capture_web_screenshot",
input,
captured.value,
ready.value.identity(),
),
)
: captured;
};
@@ -222,69 +211,3 @@ const requireBrowser = (
),
)
: ok(browser);
const browserEvidence = (
operation: Parameters<typeof createBrowserEvidence>[0],
input: Parameters<typeof createBrowserEvidence>[1],
result: Parameters<typeof createBrowserEvidence>[2],
provider: Parameters<typeof createBrowserEvidence>[3],
options: ExecutionOptions,
): Result<Evidence, AnalysisError> => {
const json = jsonValueSchema.safeParse(result);
if (!json.success)
return err(
new AnalysisOutputError(
operation,
"Browser provider returned non-JSON observation data.",
{
cause: json.error,
},
),
);
return completeProviderEvidence(
ok(createBrowserEvidence(operation, input, result, provider)),
operation,
options,
);
};
/** Convert a provider-retained page capture without claiming bundle analysis completed. */
const retainPageInspection = (
error: AnalysisError,
input: InspectWebPageInput | AnalyzeWebBundleInput,
provider: BrowserObservationPort,
errorOperation: "inspect_web_page" | "analyze_web_bundle",
): Result<Evidence, AnalysisError> => {
const partial = error.partialObservation;
if (
partial === undefined ||
"evidence_id" in partial ||
!("target" in partial) ||
!("network" in partial) ||
!("capture_window" in partial)
)
return err(error);
const inspection = webPageInspectionSchema.safeParse(partial);
if (!inspection.success) {
if (error instanceof AnalysisCancelledError)
return err(
new AnalysisOutputError(
errorOperation,
"Browser provider retained a malformed completed page inspection.",
{ cause: inspection.error },
),
);
return err(error);
}
const pageInput: InspectWebPageInput =
"fetch_source_maps" in input
? (({ fetch_source_maps: _fetchSourceMaps, ...page }) => page)(input)
: input;
error.partialObservation = createBrowserEvidence(
"inspect_web_page",
pageInput,
inspection.data,
provider.identity(),
);
return err(error);
};
@@ -1,78 +0,0 @@
import { expect, it } from "vitest";
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
import { BrowserObservationError } from "../domain/browserObservationError.js";
import { parseEvidence } from "../domain/evidence.js";
import { browserScenarioInputSchema } from "../domain/browserScenario.js";
import { err } from "../domain/result.js";
import { scriptScenarioFixture } from "../../tests/fixtures/webScriptCapture.js";
import type { BrowserScenarioCapturePort } from "./BrowserScenarioCapturePort.js";
import { captureBrowserScenario } from "./BrowserScenarioCaptureService.js";
const scenario = browserScenarioInputSchema.parse({
browser: { mode: "launch", executable_path: "/fixture/chrome" },
start_url: { url: "https://fixture.test/", query: [] },
actions: [{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }],
});
const capture = scriptScenarioFixture();
it("converts a retained raw scenario capture into partial Evidence", async () => {
const failure = new AnalysisCancelledError("capture_browser_scenario", {
partialObservation: {
kind: "browser-scenario-observation",
capture,
},
});
const provider: BrowserScenarioCapturePort = {
identity: () => ({
id: "scenario-fixture",
name: "Scenario fixture",
version: "1",
}),
captureScenario: async () => err(failure),
};
const result = await captureBrowserScenario(provider, scenario);
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toBe(failure);
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "capture_browser_scenario",
normalized_result: capture,
});
}
});
it("converts a raw scenario capture retained after cleanup failure", async () => {
const failure = new BrowserObservationError(
"capture_browser_scenario",
"cleanup_failed",
{
cleanup: {
reason: "browser close failed",
resources: ["browser process"],
},
partialObservation: {
kind: "browser-scenario-observation",
capture,
},
},
);
const provider: BrowserScenarioCapturePort = {
identity: () => ({
id: "scenario-fixture",
name: "Scenario fixture",
version: "1",
}),
captureScenario: async () => err(failure),
};
const result = await captureBrowserScenario(provider, scenario);
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toBe(failure);
expect(result.error.cleanupIncomplete).toBe(true);
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "capture_browser_scenario",
normalized_result: capture,
});
}
});
@@ -1,17 +1,11 @@
import type { ExecutionOptions } from "./AnalysisProvider.js";
import type { BrowserScenarioCapturePort } from "./BrowserScenarioCapturePort.js";
import { createBrowserScenarioEvidence } from "./BrowserScenarioEvidence.js";
import {
AnalysisCancelledError,
AnalysisCapabilityUnavailableError,
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import type { Evidence } from "../domain/evidence.js";
import type { BrowserScenario } from "../domain/browserScenario.js";
import { err, type Result } from "../domain/result.js";
import { browserScenarioCaptureSchema } from "../domain/browserScenarioCapture.js";
import { completeProviderEvidence } from "./ProviderOperationExecution.js";
const OPERATION = "capture_browser_scenario" as const;
const PROVIDER_ID = "rea-playwright-browser-scenario";
@@ -30,55 +24,15 @@ export const captureBrowserScenario = async (
"browser scenario provider is not configured",
),
);
if (options.signal?.aborted)
return err(new AnalysisCancelledError(OPERATION));
const captured = await provider.captureScenario(scenario, options);
if (!captured.ok) {
const partial = captured.error.partialObservation;
if (
partial !== undefined &&
"kind" in partial &&
partial.kind === "browser-scenario-observation"
) {
const report = browserScenarioCaptureSchema.safeParse(partial.capture);
if (!report.success) {
if (captured.error instanceof AnalysisCancelledError)
return err(
new AnalysisOutputError(
OPERATION,
"Browser scenario provider retained a malformed completed capture.",
{ cause: report.error },
),
);
return captured;
return captured.ok
? {
ok: true,
value: createBrowserScenarioEvidence(
scenario,
captured.value,
provider.identity(),
),
}
captured.error.partialObservation = createBrowserScenarioEvidence(
scenario,
report.data,
provider.identity(),
);
}
return captured;
}
const report = browserScenarioCaptureSchema.safeParse(captured.value);
if (!report.success)
return err(
new AnalysisOutputError(
OPERATION,
"Browser scenario provider returned a malformed completed capture.",
{ cause: report.error },
),
);
return completeProviderEvidence(
{
ok: true,
value: createBrowserScenarioEvidence(
scenario,
report.data,
provider.identity(),
),
},
OPERATION,
options,
);
: captured;
};
+81 -38
View File
@@ -14,14 +14,12 @@ import {
} from "./HermesConfigurationYaml.js";
import {
applyEdits,
createScanner,
findNodeAtLocation,
modify,
parse as parseJsonc,
parseTree,
printParseErrorCode,
type ParseError,
type SyntaxKind,
} from "jsonc-parser";
import { z } from "zod";
@@ -1249,27 +1247,44 @@ const tomlTextHolds = (
}
};
// jsonc-parser declares SyntaxKind as an ambient const enum, so keep its token
// code local and type-checked as in the JSON scanner used by IDA diagnostics.
const JSON_COMMA = 5 satisfies SyntaxKind;
/** Find the next non-trivia JSONC token at or after `offset`. */
const nextJsoncToken = (
text: string,
offset: number,
): { readonly kind: SyntaxKind; readonly offset: number } => {
const scanner = createScanner(text, true);
scanner.setPosition(offset);
const kind = scanner.scan();
return { kind, offset: scanner.getTokenOffset() };
/** Comments in the trivia between two properties. `undefined` when there are none. */
const commentsBetweenProperties = (gap: string): string | undefined => {
const pieces: string[] = [];
let index = 0;
while (index < gap.length) {
const line = gap.indexOf("//", index);
const block = gap.indexOf("/*", index);
if (line < 0 && block < 0) break;
const start = line < 0 ? block : block < 0 ? line : Math.min(line, block);
const lineComment = start === line;
const close = lineComment
? gap.indexOf("\n", start)
: gap.indexOf("*/", start + 2);
if (!lineComment && close < 0) return undefined;
const end = lineComment ? (close < 0 ? gap.length : close) : close + 2;
const comment = gap.slice(start, end);
const lineBreak = gap.lastIndexOf("\n", start - 1);
if (lineBreak < 0) pieces.push(` ${comment}`);
else {
const prefix = gap.slice(lineBreak + 1, start);
let indentEnd = 0;
while (prefix[indentEnd] === " " || prefix[indentEnd] === "\t")
indentEnd += 1;
const newline =
lineBreak > 0 && gap[lineBreak - 1] === "\r" ? "\r\n" : "\n";
pieces.push(`${newline}${prefix.slice(0, indentEnd)}${comment}`);
}
index = end;
}
return pieces.length === 0 ? undefined : pieces.join("");
};
/**
* Delete a property by its JSONC AST span and the separator needed to keep its
* parent object valid. All whitespace and comments outside those tokens stay
* byte-for-byte in place.
* Delete an object property without taking a comment from the gap before it.
* jsonc-parser's removal span includes that gap, so a trailing comment on the
* previous property would disappear. Returns `undefined` to keep the stock edit.
*/
const deleteJsonProperty = (
const deleteJsonPropertyPreservingComments = (
text: string,
path: readonly string[],
): string | undefined => {
@@ -1286,26 +1301,54 @@ const deleteJsonProperty = (
)
return undefined;
const propertyIndex = parent.children.indexOf(property);
if (propertyIndex < 0) return undefined;
const previous = parent.children[propertyIndex - 1];
const next = parent.children[propertyIndex + 1];
let commaOffset: number | undefined;
if (previous !== undefined) {
const comma = nextJsoncToken(text, previous.offset + previous.length);
if (comma.kind !== JSON_COMMA) return undefined;
commaOffset = comma.offset;
} else {
const token = nextJsoncToken(text, property.offset + property.length);
if (token.kind === JSON_COMMA) commaOffset = token.offset;
else if (next !== undefined) return undefined;
if (propertyIndex <= 0 || previous === undefined) return undefined;
const comments = commentsBetweenProperties(
text.slice(previous.offset + previous.length, property.offset),
);
if (comments === undefined) return undefined;
let removeEnd = property.offset + property.length;
const hasNext = parent.children[propertyIndex + 1] !== undefined;
if (hasNext) {
let comma = removeEnd;
while (comma < text.length) {
const char = text[comma];
if (char === " " || char === "\t" || char === "\r" || char === "\n") {
comma += 1;
continue;
}
if (text.startsWith("/*", comma)) {
const close = text.indexOf("*/", comma + 2);
if (close < 0) return undefined;
comma = close + 2;
continue;
}
if (text.startsWith("//", comma)) {
while (
comma < text.length &&
text[comma] !== "\n" &&
text[comma] !== "\r"
)
comma += 1;
continue;
}
break;
}
if (text[comma] !== ",") return undefined;
removeEnd = comma + 1;
}
const edits = [
{ offset: property.offset, length: property.length, content: "" },
...(commaOffset === undefined
? []
: [{ offset: commaOffset, length: 1, content: "" }]),
];
return applyEdits(text, edits);
// A line comment runs to the next break. Keep the following token outside it.
const remainder = text.slice(removeEnd);
let suffix = `${hasNext ? "," : ""}${comments}`;
if (
suffix.includes("//") &&
!suffix.endsWith("\n") &&
!suffix.endsWith("\r") &&
!remainder.startsWith("\n") &&
!remainder.startsWith("\r")
)
suffix += "\n";
return `${text.slice(0, previous.offset + previous.length)}${suffix}${remainder}`;
};
/** Serialize a validated client document, retaining JSONC and TOML comments. */
@@ -1334,7 +1377,7 @@ export const serializeClientConfiguration = (
return editedPaths.reduce((text, path) => {
const value = valueAt(document, path);
if (value === undefined) {
const preserved = deleteJsonProperty(text, path);
const preserved = deleteJsonPropertyPreservingComments(text, path);
if (preserved !== undefined) return preserved;
}
return applyEdits(
+1 -5
View File
@@ -46,13 +46,10 @@ export const directAnalysisCleanupFailure = (
...primary.data,
code: "cleanup_incomplete",
retryable: false,
remediation: cleanup.remediation,
details: {
// Primary fields describe the original failure; the later close
// assessment, including its resources, belongs to cleanup_error.
resources: error.cleanupResources.slice(),
...primary.data.details,
cleanup: "incomplete",
resources: error.cleanupResources.slice(),
execution_failure:
primary.data.details !== undefined &&
"execution_failure" in primary.data.details
@@ -89,7 +86,6 @@ export const managedAnalysisCleanupFailure = (
"close_binary",
{
cause: error,
remediationAction: cleanup.remediation.action,
cleanup: { reason: cleanup.message, resources: error.cleanupResources },
diagnostics: {
cleanup_error: cleanup,
+4 -20
View File
@@ -1,7 +1,6 @@
import { constants } from "node:fs";
import { access, readdir, stat } from "node:fs/promises";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { join } from "node:path";
@@ -434,12 +433,9 @@ export interface SystemDoctorHostOptions {
readonly architecture?: NodeJS.Architecture;
readonly environment?: NodeJS.ProcessEnv;
readonly execFileOutput?: typeof execFileOutput;
readonly providerInspections?: (
adoptFileHandle: (
resource: string,
handle: OwnedFileHandle,
) => Promise<boolean>,
) => Promise<readonly DoctorProviderInspection[]>;
readonly providerInspections?: () => Promise<
readonly DoctorProviderInspection[]
>;
readonly providerRegistrationEnvironment?: () => Readonly<
Record<string, string>
>;
@@ -553,19 +549,7 @@ export const systemDoctorHost = (
manualHopperPaths: () => manualHopperPaths(homeDirectory),
...(options.providerInspections === undefined
? {}
: {
providerInspections: () =>
options.providerInspections!(
async (resource, handle) =>
(
await resources.release({
kind: "file-handle",
resource,
handle,
})
).kind === "released",
),
}),
: { providerInspections: options.providerInspections }),
...(options.providerRegistrationEnvironment === undefined
? {}
: {
+2 -6
View File
@@ -5,6 +5,7 @@ import { join } from "node:path";
import { execFileBoundedOutput } from "../process/ExecFileBoundedOutput.js";
import { execFileOutput } from "../process/ExecFileOutput.js";
import { downloadHopperPackage } from "./HopperDownload.js";
import { hashStableFile } from "../filesystem/StableFileHash.js";
import { err, type Result } from "../domain/result.js";
import {
HOPPER_RELEASES_URL,
@@ -431,15 +432,10 @@ export const systemLinuxHopperInstallHost = (
!linuxSharedLibrariesAvailable(`${linked.stdout}\n${linked.stderr}`)
)
return "runtime_dependencies";
return (await publisher.hashLauncher(path)).sha256 === expectedSha256
return (await hashStableFile(path)).sha256 === expectedSha256
? "ready"
: "unsupported_hopper_build";
} catch (cause: unknown) {
if (
cause instanceof HopperReleaseError &&
cause.reason === "cleanup_failed"
)
throw cause;
// best-effort cleanup: optional launcher probing; unreadable means missing.
void cause;
return "missing";
+6 -53
View File
@@ -11,11 +11,6 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { err, ok, type Result } from "../domain/result.js";
import { hashStableFile } from "../filesystem/StableFileHash.js";
import {
StableFileHashCleanupFailure,
type StableFileHashResult,
} from "../filesystem/StableFileHash.js";
import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js";
import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
import { verifyDetachedPgpSignature } from "../process/DetachedPgpSignature.js";
import { execFileOutput } from "../process/ExecFileOutput.js";
@@ -129,46 +124,6 @@ export class LinuxHopperPublisher {
readonly packageFamily: () => Promise<LinuxPackageFamily | undefined>,
) {}
/** Hash a selected launcher while retaining any descriptor cleanup owner. */
hashLauncher(path: string): Promise<StableFileHashResult> {
return this.#hashStableFile(path);
}
async #hashStableFile(
path: string,
signal?: AbortSignal,
): Promise<StableFileHashResult> {
try {
return await hashStableFile(path, signal);
} catch (cause: unknown) {
if (!(cause instanceof OwnedRegularFileCleanupFailure)) throw cause;
const cleanup = await this.#resources.release({
kind: "file-handle",
resource: path,
handle: cause.owner,
});
if (cleanup.kind === "failed") {
pendingCleanup.add(this.#resources);
throw new HopperReleaseError(
"cleanup_failed",
`Hopper publisher retained file descriptor cleanup for ${path}.`,
{
cause: new AggregateError(
[cause, cleanup.cause],
"Stable file hash and descriptor cleanup failed",
{ cause },
),
},
);
}
if (cause instanceof StableFileHashCleanupFailure) {
if (cause.hasPrimaryCause) throw cause.cause;
if (cause.completedHash !== undefined) return cause.completedHash;
}
throw cause;
}
}
get cacheRoot(): string {
const location = linuxHopperReleaseCacheDirectory(this.environment);
if (!location.ok) throw location.error;
@@ -338,8 +293,8 @@ export class LinuxHopperPublisher {
await this.#withTemporary(async (stage) => {
await copyFile(archive, join(stage, "archive"));
if (
(await this.#hashStableFile(join(stage, "archive"), signal))
.sha256 !== verified.packageSha256
(await hashStableFile(join(stage, "archive"), signal)).sha256 !==
verified.packageSha256
)
throw new HopperReleaseError(
"integrity",
@@ -480,12 +435,10 @@ export class LinuxHopperPublisher {
signal?: AbortSignal,
): Promise<Result<VerifiedLinuxHopperLauncher, HopperReleaseError>> {
try {
const observed = await this.#hashStableFile(path, signal);
const observed = await hashStableFile(path, signal);
const cached = await this.#cachedPackage(observed.sha256, signal);
if (cached.ok) {
if (
(await this.#hashStableFile(path, signal)).sha256 !== observed.sha256
)
if ((await hashStableFile(path, signal)).sha256 !== observed.sha256)
return err(
new HopperReleaseError(
"integrity",
@@ -537,7 +490,7 @@ export class LinuxHopperPublisher {
if (!verified.ok) throw verified.error;
return verified.value;
});
if ((await this.#hashStableFile(path, signal)).sha256 !== observed.sha256)
if ((await hashStableFile(path, signal)).sha256 !== observed.sha256)
return err(
new HopperReleaseError(
"integrity",
@@ -730,7 +683,7 @@ export class LinuxHopperPublisher {
commandOptions,
);
return (
await this.#hashStableFile(
await hashStableFile(
join(extracted, "opt", "hopper", "bin", "Hopper"),
signal,
)
+6 -23
View File
@@ -1,8 +1,5 @@
import { z } from "zod";
import type {
AnalysisError,
AnalysisErrorOptions,
} from "../domain/analysisErrorBase.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
@@ -30,24 +27,6 @@ export interface ProviderOperationPort<
): Promise<Result<AnalysisExecution, AnalysisError>>;
}
/** Preserve valid completed Evidence when cancellation arrives at completion. */
export const completeProviderEvidence = (
completed: Result<Evidence, AnalysisError>,
operation: string,
options?: ExecutionOptions,
errorOptions?: Omit<AnalysisErrorOptions, "partialObservation">,
): Result<Evidence, AnalysisError> => {
if (!completed.ok) return completed;
return options?.signal?.aborted
? err(
new AnalysisCancelledError(operation, {
...errorOptions,
partialObservation: completed.value,
}),
)
: completed;
};
/**
* Shared CLI/MCP admission, execution, and Evidence composition for
* single-request providers. The caller retains its operation vocabulary and
@@ -90,5 +69,9 @@ export const executeProviderOperation = async <
// the caller no longer wants. The observation it completed is still this
// provider's Evidence, so the cancellation carries it as the partial
// observation instead of discarding a finished result.
return completeProviderEvidence(ok(evidence), operation, options);
if (options?.signal?.aborted)
return err(
new AnalysisCancelledError(operation, { partialObservation: evidence }),
);
return ok(evidence);
};
@@ -116,140 +116,3 @@ it("retains Kotlin source identity and language across the inventory projection"
expect(parsed.parseFailures).toEqual([]);
expect(parsed.relationships).toEqual([]);
});
it("resolves runtime require package mains without an index and before typed indexes", () => {
const files = [
["main.cjs", 'require("./pkg");\nrequire("./dep");\n'],
[
"main.ts",
'import dep = require("./typed");\n' +
'require("./typed");\n' +
'import data = require("./json-package");\n' +
'require("./emitted.js");\n',
],
["pkg/package.json", '{"main":"actual.js"}\n'],
["pkg/actual.js", 'module.exports = "package main";\n'],
["dep.js", 'module.exports = "file";\n'],
["dep/index.ts", 'export const value = "index";\n'],
["typed.js", 'module.exports = "runtime file";\n'],
["typed/index.ts", 'export const value = "source index";\n'],
["json-package/package.json", '{"main":"data"}\n'],
["json-package/data.json", '{"value":"package main"}\n'],
["json-package/index.js", 'module.exports = "fallback index";\n'],
["emitted.ts", 'export const value = "source counterpart";\n'],
] as const;
const read: ReferenceSourceRead = {
root: "/selected/source",
entries: files.map(([path, content]) => ({
status: "read",
kind: "file",
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
})),
bytesRead: files.reduce(
(sum, [, content]) => sum + Buffer.byteLength(content),
0,
),
limitations: [],
};
const parsed = parseReferenceSourceEntries(
read,
new Set(files.map(([path]) => path)),
);
expect(parsed.relationships).toContainEqual({
from_path: "main.cjs",
to: "pkg/actual.js",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
expect(parsed.relationships).toContainEqual({
from_path: "main.cjs",
to: "dep.js",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
expect(parsed.relationships).toContainEqual({
from_path: "main.ts",
to: "typed/index.ts",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
expect(parsed.relationships).toContainEqual({
from_path: "main.ts",
to: "typed.js",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
expect(parsed.relationships).toContainEqual({
from_path: "main.ts",
to: "json-package/data.json",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
expect(parsed.relationships).toContainEqual({
from_path: "main.ts",
to: "emitted.ts",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
});
});
it("matches package manifests by exact or unique canonical directory identity", () => {
const composed = "caf\u00e9";
const decomposed = "cafe\u0301";
const composedTwin = "r\u00e9sum\u00e9";
const decomposedTwin = "re\u0301sume\u0301";
const files = [
[
"main.cjs",
[composed, composedTwin, decomposedTwin]
.map((name) => `require(${JSON.stringify(`./${name}`)});`)
.join("\n"),
],
[`${decomposed}/package.json`, '{"main":"unique.js"}\n'],
[`${decomposed}/unique.js`, 'module.exports = "unique";\n'],
[`${composed}/unique.js`, 'module.exports = "unrelated exact spelling";\n'],
[`${composedTwin}/package.json`, '{"main":"exact-composed.js"}\n'],
[`${composedTwin}/exact-composed.js`, 'module.exports = "composed";\n'],
[`${decomposedTwin}/package.json`, '{"main":"exact-decomposed.js"}\n'],
[
`${decomposedTwin}/exact-decomposed.js`,
'module.exports = "decomposed";\n',
],
] as const;
const read: ReferenceSourceRead = {
root: "/selected/source",
entries: files.map(([path, content]) => ({
status: "read",
kind: "file",
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
})),
bytesRead: files.reduce(
(sum, [, content]) => sum + Buffer.byteLength(content),
0,
),
limitations: [],
};
const parsed = parseReferenceSourceEntries(
read,
new Set(files.map(([path]) => path)),
);
expect(parsed.relationships.map(({ to }) => to)).toEqual([
`${decomposed}/unique.js`,
`${composedTwin}/exact-composed.js`,
`${decomposedTwin}/exact-decomposed.js`,
]);
});
+33 -143
View File
@@ -107,20 +107,11 @@ type PackageMain =
*/
const readPackageMains = (
read: ReferenceSourceRead,
): {
readonly mains: ReadonlyMap<string, PackageMain>;
readonly pathLookup: ReferenceSourcePathLookup;
} => {
): ReadonlyMap<string, PackageMain> => {
const mains = new Map<string, PackageMain>();
const manifestPaths: string[] = [];
for (const entry of read.entries) {
if (posix.basename(entry.path) !== "package.json") continue;
manifestPaths.push(entry.path);
const directory = posix.dirname(entry.path);
if (mains.has(directory)) {
mains.set(directory, { state: "unknown" });
continue;
}
if (entry.status !== "read" || entry.kind !== "file") {
mains.set(directory, { state: "unknown" });
continue;
@@ -143,132 +134,55 @@ const readPackageMains = (
mains.set(directory, { state: "unknown" });
}
}
return {
mains,
pathLookup: createReferenceSourcePathLookup(manifestPaths),
};
return mains;
};
/**
* Extension slots in probe order. Within a slot, files precede directory
* indexes. The CommonJS loader tries `X.json` after `X.js` and before any
* directory index; ESM and TypeScript require an explicit `.json`.
*/
const RESOLUTION_SUFFIXES: readonly (readonly string[])[] = [
[""],
[".ts"],
[".js"],
[".mjs"],
[".cjs"],
[".tsx"],
[".jsx"],
];
const REQUIRE_RESOLUTION_SUFFIXES = RESOLUTION_SUFFIXES.map((slot) =>
slot.includes(".js") ? [...slot, ".json"] : slot,
);
const NODE_REQUIRE_SUFFIXES = ["", ".js", ".json", ".node"] as const;
type InternalSpecifierResolutionMode =
| "source-import"
| "source-require"
| "runtime-require";
const resolutionSuffixes = (
mode: InternalSpecifierResolutionMode,
): readonly (readonly string[])[] =>
mode === "source-require" ? REQUIRE_RESOLUTION_SUFFIXES : RESOLUTION_SUFFIXES;
const nodeRequireFileCandidates = (path: string): string[] =>
NODE_REQUIRE_SUFFIXES.map((suffix) => `${path}${suffix}`);
const nodeRequireIndexCandidates = (directory: string): string[] =>
NODE_REQUIRE_SUFFIXES.filter((suffix) => suffix !== "").map((suffix) =>
posix.join(directory, `index${suffix}`),
);
const RESOLUTION_SUFFIXES = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"];
/** Inventory facts an internal specifier resolves against. */
interface SpecifierContext {
readonly lookupPath: ReferenceSourcePathLookup;
readonly packageMains: ReadonlyMap<string, PackageMain>;
readonly packageManifestPathLookup: ReferenceSourcePathLookup;
}
const packageMainForDirectory = (
directory: string,
context: SpecifierContext,
): { readonly directory: string; readonly main: PackageMain } | undefined => {
const manifestPath = context.packageManifestPathLookup.resolve(
posix.join(directory, "package.json"),
);
if (manifestPath !== undefined) {
const manifestDirectory = posix.dirname(manifestPath);
const main = context.packageMains.get(manifestDirectory);
return main === undefined
? undefined
: { directory: manifestDirectory, main };
}
return context.packageManifestPathLookup.has(
posix.join(directory, "package.json"),
)
? { directory, main: { state: "unknown" } }
: undefined;
};
/** Apply a package main, retaining an existing index as the fallback target. */
/**
* Apply a package `main` to a directory resolved through its index. Node loads
* the declared entry as a file, then as a directory index, and falls back to
* the package's own index only when neither exists.
*/
const selectPackageMain = (
directory: string,
indexFallback: string | undefined,
index: string,
context: SpecifierContext,
mode: InternalSpecifierResolutionMode,
): { to: string; resolution: "internal" | "unresolved" } => {
const packageSelection = packageMainForDirectory(directory, context);
const packageDirectory = packageSelection?.directory ?? directory;
const manifest = packageSelection?.main;
const directory = posix.dirname(index);
const manifest = context.packageMains.get(directory);
if (manifest === undefined || manifest.state === "none")
return indexFallback === undefined
? { to: directory, resolution: "unresolved" }
: { to: indexFallback, resolution: "internal" };
return { to: index, resolution: "internal" };
if (manifest.state === "unknown")
return { to: packageDirectory, resolution: "unresolved" };
return { to: directory, resolution: "unresolved" };
// Node resolves `main` with path.resolve: an absolute main leaves the
// package, and a trailing separator does not make the entry a directory.
if (posix.isAbsolute(manifest.main))
return { to: posix.normalize(manifest.main), resolution: "unresolved" };
const target = posix
.normalize(posix.join(packageDirectory, manifest.main))
.normalize(posix.join(directory, manifest.main))
.replace(/(?<=.)\/+$/u, "");
// Node would load a declared entry outside the imported root.
if (logicalPathEscapesRoot(target))
return { to: target, resolution: "unresolved" };
const slots = resolutionSuffixes(mode);
const candidates =
mode === "runtime-require"
? [
...nodeRequireFileCandidates(target),
...nodeRequireIndexCandidates(target),
]
: [
...slots.flatMap((slot) =>
slot.map((suffix) => `${target}${suffix}`),
),
...slots
.flat()
.filter((suffix) => suffix !== "")
.map((suffix) => posix.join(target, `index${suffix}`)),
];
const selected = context.lookupPath.resolveCandidates(candidates);
return selected === undefined
? indexFallback === undefined
? { to: packageDirectory, resolution: "unresolved" }
: { to: indexFallback, resolution: "internal" }
: { to: selected, resolution: "internal" };
const selected = context.lookupPath.resolveCandidates([
...RESOLUTION_SUFFIXES.map((suffix) => `${target}${suffix}`),
...RESOLUTION_SUFFIXES.filter((suffix) => suffix !== "").map((suffix) =>
posix.join(target, `index${suffix}`),
),
]);
return { to: selected ?? index, resolution: "internal" };
};
const resolveInternalSpecifier = (
fromPath: string,
specifier: string,
context: SpecifierContext,
mode: InternalSpecifierResolutionMode,
): { to: string; resolution: "internal" | "unresolved" } => {
if (posix.isAbsolute(specifier))
return { to: specifier, resolution: "unresolved" };
@@ -282,21 +196,10 @@ const resolveInternalSpecifier = (
specifier.endsWith("/") || [".", ".."].includes(posix.basename(specifier));
// A directory resolves only to an index with an extension; neither Node nor
// TypeScript selects an extensionless `index` file.
const slots = resolutionSuffixes(mode);
const candidates =
mode === "runtime-require"
? [
...(directoryReference ? [] : nodeRequireFileCandidates(normalized)),
...nodeRequireIndexCandidates(normalized),
]
: slots.flatMap((slot) => [
...(directoryReference
? []
: slot.map((suffix) => `${normalized}${suffix}`)),
...slot
.filter((suffix) => suffix !== "")
.map((suffix) => posix.join(normalized, `index${suffix}`)),
]);
const candidates = RESOLUTION_SUFFIXES.flatMap((suffix) => [
...(directoryReference ? [] : [`${normalized}${suffix}`]),
...(suffix === "" ? [] : [posix.join(normalized, `index${suffix}`)]),
]);
// Retain present runtime files first; substitute only a missing counterpart.
// Counterparts cover explicit filenames, not directory/index resolution.
if (!directoryReference)
@@ -306,15 +209,12 @@ const resolveInternalSpecifier = (
),
);
const match = context.lookupPath.resolveCandidates(candidates);
if (match === undefined)
return packageMainForDirectory(normalized, context) !== undefined
? selectPackageMain(normalized, undefined, context, mode)
: { to: normalized, resolution: "unresolved" };
if (match === undefined) return { to: normalized, resolution: "unresolved" };
const resolvedThroughIndex =
posix.basename(match).startsWith("index.") &&
posix.dirname(match).normalize("NFC") === normalized.normalize("NFC");
return resolvedThroughIndex
? selectPackageMain(posix.dirname(match), match, context, mode)
? selectPackageMain(match, context)
: { to: match, resolution: "internal" };
};
@@ -342,25 +242,17 @@ const appendFile = (
});
if (parsed === undefined) return;
for (const relationship of parsed.relationships) {
const { resolutionStrategy, ...graphRelationship } = relationship;
if (graphRelationship.resolution !== "internal") {
output.relationships.push(graphRelationship);
if (relationship.resolution !== "internal") {
output.relationships.push(relationship);
continue;
}
const mode: InternalSpecifierResolutionMode =
resolutionStrategy === "runtime-require"
? "runtime-require"
: graphRelationship.kind === "requires"
? "source-require"
: "source-import";
const resolved = resolveInternalSpecifier(
graphRelationship.from_path,
graphRelationship.to,
relationship.from_path,
relationship.to,
context,
mode,
);
output.relationships.push({
...graphRelationship,
...relationship,
to: resolved.to,
resolution: resolved.resolution,
});
@@ -417,11 +309,9 @@ export const parseReferenceSourceEntries = (
parseFailures: [],
limitations: [...read.limitations],
};
const packageMainInventory = readPackageMains(read);
const context: SpecifierContext = {
lookupPath: createReferenceSourcePathLookup(filePaths),
packageMains: packageMainInventory.mains,
packageManifestPathLookup: packageMainInventory.pathLookup,
packageMains: readPackageMains(read),
};
for (const entry of read.entries) {
if (signal?.aborted === true) break;
+4 -4
View File
@@ -159,10 +159,10 @@ const claudeDesktopLocation = (
};
const claudeCodeConfigDirectory = ({ home, env }: ClientPathContext): string =>
env.CLAUDE_CONFIG_DIR || home;
env.CLAUDE_CONFIG_DIR ?? home;
const claudeCodeMarkerDirectory = ({ home, env }: ClientPathContext): string =>
env.CLAUDE_CONFIG_DIR || join(home, ".claude");
env.CLAUDE_CONFIG_DIR ?? join(home, ".claude");
const codexDirectory = ({ home, env }: ClientPathContext): string =>
env.CODEX_HOME || join(home, ".codex");
@@ -486,8 +486,8 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
{
name: "claude_code",
displayName: "Claude Code",
skillPath: (context: ClientPathContext) =>
join(claudeCodeMarkerDirectory(context), "skills"),
skillPath: ({ home, env }: ClientPathContext) =>
join(env.CLAUDE_CONFIG_DIR ?? join(home, ".claude"), "skills"),
configPath: (context: ClientPathContext) =>
join(claudeCodeConfigDirectory(context), ".claude.json"),
markerPath: claudeCodeMarkerDirectory,
Loaded 100 of 478 files, more files were not shown because too many files have changed in this diff. Show more