Compare commits

...
Author SHA1 Message Date
Kaoru0822andClaude Opus 5.5 c6cda443c1 docs(javascript): state that duplicate unmapped content stays ambiguous
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 03:33:36 +00:00
Kaoru0822andClaude Opus 5.5 e308f35c43 fix(javascript): keep unmapped captured runtime scripts unknown
A captured script whose bytes matched no static digest was reported as
high-confidence unmatched even when no automatic or caller-declared
mapping applied to its location in any static layer. The matcher now
computes mapping presence once while locating candidates and shares one
no-candidate classification between the captured-source and source-free
branches, so an unmapped location stays unknown with
no-authorized-location-mapping and is not counted as unmatched.

Fixes #6061.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 02:20:21 +00:00
morluto 719d56ebfe fix(evidence): preserve provider execution provenance across workflows 2026-10-22 10:02:31 +08:00
morluto 87c8086fc1 refactor(managed): reuse canonical JSON digest owner 2026-10-22 09:52:58 +08:00
morluto eab4e47916 refactor(ghidra): share session startup observation checks 2026-10-22 09:52:49 +08:00
morluto db87917daa refactor(native): use canonical deadline defaults 2026-10-22 09:52:40 +08:00
morluto 183b4d4e7e refactor(managed): share metadata token schema 2026-10-22 09:33:58 +08:00
morluto 00819258a7 refactor(browser): separate bundle state from AST inspection 2026-10-22 09:33:58 +08:00
morluto c465ff5a81 refactor(evidence): share canonical ASCII ordering rules 2026-10-22 09:21:54 +08:00
morluto 1ac7ce3f3b refactor(native): share call observation outcome schema 2026-10-22 09:21:54 +08:00
morluto 55f5c8db69 refactor(kuna): centralize installation cancellation projection 2026-10-22 09:21:54 +08:00
morluto 5b0373db11 refactor(browser): map artifact failures at one boundary 2026-10-22 09:21:54 +08:00
morluto 73be86c39e fix(android): decode guardian output as UTF-8 streams 2026-10-22 09:11:44 +08:00
morluto 8dbe92a4a3 refactor(reference): centralize source inventory rules 2026-10-22 09:11:44 +08:00
morluto 5a28a147f1 fix(hopper): preserve streamed Unicode lease owner identities 2026-10-22 09:11:44 +08:00
morluto 13a8640204 fix(browser): observe fetch settlement after synchronous abort 2026-10-22 09:03:51 +08:00
morluto 09b29436f4 refactor(snapshot): share persisted execution projection 2026-10-22 09:03:51 +08:00
morluto 421d527bfe Merge remote-tracking branch 'origin/main' into codex/legacy-cleanup-followup-20261022 2026-10-22 08:52:56 +08:00
morluto 24bd2b989a refactor(browser): derive comparability once per capture pair 2026-10-22 08:52:56 +08:00
morluto b2fad362f2 fix(browser): stop expired work and release deadline resources 2026-10-22 08:52:56 +08:00
morluto 247c8f4eab refactor(javascript): own property capture translation 2026-10-22 08:52:56 +08:00
morluto adfde012ec refactor(readiness): share canonical stage validation 2026-10-22 08:52:56 +08:00
Jay-Alexander Elliot 57fd3ea5d8 feat(cli): summarize managed members with durable Evidence paging (#5167)
Fixes #5148.

Add opt-in durable managed member summaries and authenticated paging while preserving complete default output, original Evidence, cleanup ownership and queued cancellation.

Verified with bounded real Newtonsoft/Redux/Rust tasks, independent oracles, 556 focused tests and the expanded domain gate; paired token estimates and latency tradeoffs are recorded in experiments/analysis-efficiency/managed-cli.md.
2026-10-22 00:43:30 +00:00
morluto d8948efb81 refactor(mcp): bind Electron provider failures at registration 2026-10-22 08:40:36 +08:00
morluto 3966c92c71 refactor(ghidra): derive reference facts from shared schema 2026-10-22 08:40:36 +08:00
morluto 1fbfbf74ef refactor(gdb): own command record snapshots and stream groups 2026-10-22 08:40:36 +08:00
morluto 2604059961 refactor(process): share decoded NDJSON framing 2026-10-22 08:35:20 +08:00
morluto 6b44e35bec refactor(managed): own successful PE inspection identity 2026-10-22 08:35:20 +08:00
morluto 20bf7d957f refactor(setup): reuse desired registration per document snapshot 2026-10-22 08:35:20 +08:00
morluto c9f95eaf90 refactor(javascript): own graph Evidence envelope validation 2026-10-22 08:27:39 +08:00
morluto a2805bc934 refactor(managed): share artifact and token identity schemas 2026-10-22 08:27:39 +08:00
morluto b7529304f0 refactor(browser): centralize shared CDP observation facts 2026-10-22 08:27:39 +08:00
morluto 22c4b528d2 Merge remote-tracking branch 'origin/main' into codex/legacy-cleanup-followup-20261022 2026-10-22 08:20:30 +08:00
morluto 1158dc2a94 refactor(process): own worker frame wait lifecycle 2026-10-22 08:20:30 +08:00
morluto 2765fc12bb refactor(browser): share composed artifact cleanup ownership 2026-10-22 08:20:30 +08:00
gensokyo-dev 18669641e9 test(browser): select a bounded service-worker setup deadline (#6186) 2026-10-22 08:16:34 +08:00
morluto cdd718ffb0 refactor(inputs): own absolute path validation in shared contracts 2026-10-22 08:13:46 +08:00
morluto c5c5ff45aa refactor: import shared limits and projection helpers from their owners 2026-10-22 08:13:46 +08:00
morluto 5da3b7c393 refactor(asar): centralize read failure and cancellation semantics 2026-10-22 08:13:46 +08:00
morluto 2451e187be test(prompts): isolate capture admission regression 2026-10-22 07:51:42 +08:00
morluto 5e6529521c Merge remote-tracking branch 'origin/main' into codex/legacy-cleanup-followup-20261022 2026-10-22 07:49:01 +08:00
morluto 7cec547ecc fix(prompts): suggest only admitted process capture evidence 2026-10-22 07:49:00 +08:00
morluto c1a0561bf3 refactor(mobile): retain inventory sources at the projection boundary 2026-10-22 07:48:39 +08:00
morluto fafc1881b5 refactor(managed): return singular admitted evidence records 2026-10-22 07:48:39 +08:00
morluto a59abe932e refactor(evidence): centralize comparison identities and source recording 2026-10-22 07:45:47 +08:00
morluto 452e3ccad1 refactor(javascript): own source range validation once 2026-10-22 07:45:47 +08:00
8413ec08f7 fix(process): report the effective regex heap and uncertain signal retry (#6099)
* fix(process): report the effective regex worker heap

Node gives --max-old-space-size, including NODE_OPTIONS, precedence over
the worker's 128 MiB resourceLimits request. Name that effective
old-generation heap and stop saying the request has no runtime override.
The matching deadline stays a fixed cap.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): keep SIGXCPU and SIGXFSZ retries uncertain

The signal does not prove the retained rlimit was exhausted, so an
unchanged retry may succeed. Project that retry as possible instead of
saying it fails the same way.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): leave percentage heap overrides unestablished

--max-old-space-size-percentage takes precedence over the absolute flag
in either order. Node computes that MiB from startup memory, which a
later read does not recover, so the effective heap stays unknown instead
of the 128 MiB request or an absolute flag.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): read launch NODE_OPTIONS, not process.env

A preload can rewrite process.env.NODE_OPTIONS after V8 has applied the
original flags, and those flags are not copied into execArgv. Use the
initial environment block when it can be read, and leave the effective
heap unestablished when that provenance cannot be recovered.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): leave env-file heap overrides unestablished

--env-file and --env-file-if-exists can supply NODE_OPTIONS when that
variable was absent from the initial environment. The file is not in
/proc/self/environ, so a missing entry is not the 128 MiB request.
An initial NODE_OPTIONS is still parsed, because Node does not let the
file replace it.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): leave --max-heap-size old generation unestablished

A positive --max-heap-size or --max_heap_size sets the total heap, not the
old generation, and can stay below the 128 MiB worker request when an
old-space flag is also present. The effective old generation stays
unestablished while that value is in effect. Zero or empty clears it.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): leave config-file heap options unestablished

--experimental-config-file and --experimental-default-config-file can
apply a heap percentage from a file that is not launch provenance. The
effective old generation stays unestablished when either flag is present,
including beside an absolute --max-old-space-size.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): read launch exec argv from the command line

process.execArgv can be cleared after V8 applies --max-old-space-size.
Linux recovers those tokens from /proc/self/cmdline and leaves the heap
unestablished when that vector cannot be separated from the program entry.
Heap probes compare pinned absolute limits instead of a percentage of host memory.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(process): leave a title-overwritten command line unestablished

process.title and --title replace /proc/self/cmdline with the title.
A buffer without a program entry is not an empty option list, so the
heap stays unknown instead of the 128 MiB request. The command-line
probe does not read /proc off Linux.

Co-authored-by: morluto <morluto@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 07:40:14 +08:00
morluto 603a7d022c refactor(process): share capture evidence admission 2026-10-22 07:37:21 +08:00
morluto b2dc45006d refactor(cli): share registered command traversal 2026-10-22 07:37:21 +08:00
morluto 1828cf87a0 refactor: remove redundant contract and syntax definitions 2026-10-22 07:37:21 +08:00
morluto a73eac190b refactor: remove obsolete forwarding APIs and unify CI contracts (#6181)
* refactor: remove obsolete internal forwarding APIs

* refactor(ci): centralize failure classification contracts
2026-10-22 07:30:54 +08:00
0e5c511c01 fix(electron): follow absolute symlinks that stay inside the framework (#3587)
* fix(electron): follow absolute symlinks that stay inside the framework

An absolute symlink target is now passed to relativeInside(root,
target), and a null result still ends the walk as an escape. When the
target lies inside the root, resolved is cleared and the root-relative
pieces are queued on pending, which replaces the path walked so far with
that absolute target. Later hops stay under the hop limit and the
existing lstat containment checks. Provider tests now cover an absolute
Versions/Current that stays on in-framework version B and matches the
relative bundle's resolved path and SHA-256, an absolute Current outside
the framework, and an absolute in-framework link whose next hop leaves
the framework; those last two still surface an escape error.

Fixes #3380

* test(electron): cover absolute framework symlink cycles

---------

Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
2026-10-22 07:21:37 +08:00
2e6b38cef0 feat(environment): render more read tools from the state and describe responses (#5352)
* feat(environment): render more read tools from the state and describe responses

Keep main's environment serving and fold in coverage for non-entity reads,
response summaries, and MCP tool annotations.

Squashed rebase of #5352 onto main.

* fix(environment): keep rendered documents and quoted samples inside their budgets

A recorded body of exponential numbers can render past 256 KiB and turn a
working read into a state size error, so that document stays a recorded replay.
Description samples are measured after JSON quoting, and the replay-fallback
catalog check uses a tool count the longer response descriptions still separate.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* test(environment): pin the replay card this branch serves

The replay-only card test pinned main's 11,131-byte card. This branch's descriptions name each tool's recorded responses, so the card it serves is 12,818 bytes; the pin and its name follow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 07:08:44 +08:00
d7ca93cc2c fix(artifacts): name the ASAR header admission budget (#6172)
Keep the 16 MiB ASAR header/index ceiling fixed and report that budget,
the lack of a runtime override, and that source selection does not bypass
header admission.

Refs #5910

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 06:33:09 +08:00
Afloat 82e4e032df fix(artifacts): complete positional ZIP reads (#4414)
fix(artifacts): complete ZIP positional reads
2026-10-22 06:13:24 +08:00
Abdul Samad Siddiquiandmorluto ca2ca16146 fix(dotnet): reject unfinished CIL prefix chains (#4416)
* fix(dotnet): reject unfinished CIL prefix chains

* test(dotnet): cover unfinished prefixes in fat method bodies

---------

Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
2026-10-22 06:13:10 +08:00
xjh 14c4c2a1d7 fix(managed): preserve observed method header differences (#3814) 2026-10-22 06:12:55 +08:00
Hemang VarshneyandHemang Varshney 4842f23765 fix(source): retain CommonJS requires spelled as exact no-substitution templates (#4511)
* fix(source): retain CommonJS requires spelled as exact no-substitution templates

Admit expression-free template arguments at the CommonJS require gate and
read their single cooked quasi through the shared exact-literal reader.
Scope is the require path only: static ESM template declarations keep their
parse failures and dynamic import() keeps the explicit unknown/partial
<dynamic-import> relationship with no invented target.

Refs: #4382

* fix(source): admit TypeScript-wrapped exact require arguments via the shared literal reader

Address review: the require gate matched the outer AST node, so
runtime-transparent TS wrappers (as const, !, satisfies) around exact
strings or expression-free templates were still silently omitted even
though readExactJavaScriptLiteral unwraps them. Remove the gate and let
the shared exact-literal reader own argument exactness; computed
arguments remain silent and dynamic import() behavior is unchanged.

Refs: #4382

---------

Co-authored-by: Hemang Varshney <218273197+hemv-857@users.noreply.github.com>
2026-10-22 06:12:42 +08:00
7721d59e1a fix(adb): normalize absent installer package sentinels (#4073)
* fix(adb): report a null installer declaration as an absent installer

Package Manager prints its String fields by concatenation, so a package
without a recorded installer shows `installerPackageName=null` in
`dumpsys package`. The projection treated only an empty value as absent
and forwarded the text "null" as a present installer identity, which a
consumer cannot tell apart from a package literally named "null" (#3943).
`versionName` is printed the same way and had the same gap.

The parser now maps the empty value and the `null` sentinel of both
String fields to the nullable result's null at the producing boundary;
a reported installer such as com.android.vending is kept unchanged.

Validation: parser unit tests for the sentinel, empty and missing
installer lines and the absent version name, plus a provider-level
regression through the inspect_adb_package operation; three of them fail
on main and pass here. The issue's four CLI controls through
`rea inspect-adb-package` against a stub adb return JSON null for the
sentinel, empty and missing cases and keep com.android.vending. Typecheck,
lint and module boundaries pass.

Closes #3943

* fix(adb): keep the null sentinel handling to the installer field

Review feedback on #4073: a manifest can declare the literal string
"null" as android:versionName, and dumpsys prints it exactly like an
absent value, so mapping versionName to null would turn a real version
into absence. Only installerPackageName, where no package can carry that
name, treats the printed null as the absent installer. The test for the
version name now pins the retained text.

* test(adb): establish permission inventory in null-installer fixture

Main now reports partial coverage when the requested-permission
inventory is not established (9534e42ec). Give the null-installer
fixture a requested permissions section so it still checks a
complete projection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 06:11:49 +08:00
76dbc32f39 fix(environment): bound state reads and retain cleanup ownership (#6158)
* feat(environment): extract the initial state a capture's reads observed

Add the environment-state analysis view. From the parents the environment
manifest accepts, it extracts the data the capture's reads observed before
the first related write: entity types with an identity field (proved by a
path parameter or inferred from an identity name), entities with their
recorded values and provenance, documents for written non-entity paths, and
a render plan per read tool that is kept only when it reproduces the tool's
latest initial recordings from the state; every other read tool is a
recorded view with the reason. The Evidence is always inferred.

Response-body decoding moves to a domain owner shared with replay, exact
JSON parsing moves to the owner of the exact-number policy, the manifest and
state views share parent admission and record grouping, and
projectAnalysisView takes its accumulators as one object.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): bound state at redacted writes; infer identity only from items

A scenario write whose URL was redacted has no request line, so it was
dropped and reads after it counted as initial state. Such a successful
write now joins the timeline with an empty path, whose root relates to
every type, so it ends every type's state; a limitation counts them.

A list whose recordings were all empty passed the "every listed item
carries it" check vacuously and published an identity field nothing
showed; it now needs at least one item, else the view stays recorded
as no-identity.

A value that differed between two request lines of one item tool, such
as a query variant, was called volatile, and the gate then substituted
each recording's own value; only repeats of one request line now show
volatility, so differing lines go through fields-differ.

List membership is compared only while a list can still be the whole
collection, and merged entities are indexed by type and collection once,
so paged captures no longer cost pages times entities (8,000 pages of
40,000 entities: 5.7 s to 0.23 s).

Docs and the tool contract say scenario state keeps the capture's
redaction markers and how redacted writes bound it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep first sightings, one list pointer and linear field unions

Entities took the order of the first plan observed, and item plans run
before list plans, so a complete list read before its detail reads came
out reordered and lost "all" membership; each entity now keeps its
earliest recorded sighting.

A list plan reused the first recording's pointer for later recordings,
so a later response that was itself an entity wrapping a list was read
as that list; every recording must now locate the same list.

The field union scanned its ordered list for every key, quadratic in an
object's width; a set now answers membership.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep post-write values and partial lists out of the initial state

A value that changed between repeats of one request line was named
volatile whenever no write under the type's own root lay between, so a
write under another root (for example POST /api/dashboards/db before a
second GET /api/search) left its effect in the initial state. Any
recorded write between the two observations now leaves every tool of the
type recorded with reason fields-differ instead.

A list was published with membership "all" even when a total or count
envelope member exceeded the number it listed. Such a counter now makes
the membership "recorded".

Also drop a stale doc comment on ObservedEntity.order.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(environment): scope item-shaped and deleted written paths out of documents

State documents are held only for list-shaped paths a POST, PUT or PATCH
tool writes. Reads of item-shaped written paths and of paths that are only
deleted stay recorded views; the guide now says so and the spec defers them
to a later increment of #4933.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): count writes in flight when a value changes between reads

The volatility check compared write start positions with the two reads,
so a write already in flight when the earlier read started was missed and
its effect was named volatile. A write now counts when its interval
overlaps the span from the earlier read's start to the later read's end.

The check also ran once per changed field and scanned every write. It now
runs once per pair of observations, answered by a binary search over the
writes in timeline order and the latest end among each prefix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): catch cross-tool changes across writes; read only item totals

A value that changed across a write was caught only between repeats of
one tool's request line, so a list read before a write and an item read
after it put the post-write value into the initial state. Changes are
now classified per entity in timeline order after observation: across
any recorded write, a change between repeats of a line or between two
lines of the type rewrites the type (every tool recorded as
fields-differ); with no write between, a repeat's change is volatile.
The classification lives in environmentStateMerge.ts.

Any numeric member whose name contained "total" or "count" counted as
an item total, so total_pages, account_id or page_count made complete
lists partial. Item totals are now members whose name tokens include
total or count and none of page, pages or per; a string of digits counts
as a number; and a set has_more, next or next-page token member also
marks the list partial.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep post-write observations out of initial state

* fix(environment): retain field provenance and bound state inference work

* feat(environment): serve tool calls from the environment state

`environment-serve --state <state.json>` serves an environment from the
initial state an environment-state view extracted. Read tools whose views
are state-backed answer from a live state store: item views render the held
entity into the recorded envelope, list views render the collection or the
recorded members, and document views return the held value; a missing entity
or one lacking a required field is refused, and requests the state cannot
answer fall back to their recordings with the reason. The /agentenv data
plane implements data/reset, data/add (a rea-state/v1 document as a data
part, base64 bytes, a file:// path or an https URL, validated and replaced
atomically) and data/get, and the card advertises those operations and an
intake declaration. Stage 1 is read-only: write tools are not served in
state mode. Without --state nothing changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(environment): record verification after merging the state view's review fixes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* feat(environment)!: serve the environment state by default

environment-serve derives the environment state at startup by running the
environment-state view in-process on the capture with the manifest's
template rules, so an environment comes with its state and a user supplies
nothing extra. The derived Evidence is the same one inspect-analysis-view
produces by hand. --state is removed; --replay-only serves exactly the
stateless replay of main. When the state cannot be derived, or its catalog
does not fit the discovery budget, the server starts in replay mode, logs
the reason and reports it in the stop summary's `state`. Exported bundles
serve the state by default too: export derives it the same way, so the
verifier's tool count and checked answer match what is served, and the
README describes the served mode.

BREAKING CHANGE: environment-serve answers read tools from the derived
state and does not serve write tools unless --replay-only is given; the
stop summary reports `state` instead of `state_evidence_id`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): bound data-plane replies, keep batch order, cancel intake on stop

- One /agentenv response is held to 64 MiB: replies are measured as they
  are produced and one past the budget becomes a -32000 error stating it.
- Every message of a batch enters the arrival-order queue on arrival, so
  another request's reset never lands between them.
- Stopping the server, or a caller leaving, cancels the active download
  and answers queued calls `cancelled` without running them.
- file:// URIs name only this host: another host or a network share is
  refused before anything is opened.
- A loaded view value may not give an entity another identity.
- List answers refuse a member lacking a field every recording carried,
  as item answers do.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep bodies too deep for the state's nesting out of the state

A recorded body at the JSON depth limit was accepted by extraction, then
nested a few levels deeper as a document or entity value, so sealing the
state Evidence threw and environment-serve and environment-export stopped
instead of falling back. stateBodyJson, the one rule extraction and
serving share, now treats a body nesting deeper than the limit minus 16
levels as unusable, so its tool stays a recorded view.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): measure data-plane replies before building them

83fcc739 bounded one /agentenv response to 64 MiB, but it still
serialized each reply whole before measuring it, so a batch of data/get
calls on a large state built every refused reply in full only to drop
it. Each reply is now measured in bounded JSON parts (the shared
jsonParts encoder), its parts kept only while it fits; a reply past the
budget is counted for its exact reply_bytes but never assembled. A test
checks that a refused read is never handed to JSON.stringify whole and
that its reply_bytes is exact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): release batch replies as they are encoded; judge exact numbers by value

answerBatch kept every settled reply in its pending array until the
batch ended, so a batch alternating data/add and data/get held each
loaded state alive; each reply now leaves the queue as it is encoded.

The exact-number policy judged a number by its recorded spelling, so a
state loaded with 1e20 came back from data/get as
100000000000000000000, which data/add refused, breaking the promise
that data/get output loads back unchanged. A number is exact only when
both its recorded spelling and the one serialization writes are safe,
so integers from 2^53 below 1e21 are refused in any spelling; replay
keeps such bodies as recorded text. A domain test checks that every
accepted number is accepted again once written back.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep room for refusals and framing in the reply budget

The response budget counted accepted replies only: a budget error that
replaced a reply, and a batch's brackets and commas, were not charged,
so near 64 MiB later refusals pushed the body past the advertised
limit. Every reply still to come now keeps room for its own budget
error, and a batch keeps room for its framing, so a refusal always fits
and the body never passes the budget. Reply ids and which messages are
answered come from one place shared by the replies and the reserve.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): refuse a loaded state that data/get would write past the intake limit

data/add measured only the document it was given, while data/get
writes the state back with the members a document may omit (types, the
state Evidence id, view fields and record citations). A document just
under 32 MiB could therefore load and be reported larger than any intake
accepts, breaking the data/get then data/add round trip. replace now
measures the state as data/get writes it, part by part, and refuses it
past the limit, leaving the state unchanged. The limit moves to the
state document module, so the application store can read it; the
intake re-exports it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): keep merged export checks on the shared test graph

Main's export tests installed module mocks for the whole services
project, which reuses one module graph. A tarball read failure then
missed its importer, and a one-shot heap mock could be taken by another
file. Read those through bindings the tests replace for one case.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(environment): keep a state file whose close failed owned until it closes

A file:// data/add that could not close its descriptor dropped the handle,
and an admission failure whose cleanup close failed dropped its owner, so
nothing could retry the close or report it.

The read now runs through withOwnedRegularFile, and a failed close is
retained in a server-lifetime ArtifactResourceScope. The next file read
retries it first and answers -32000 while it is still open; inline and
https documents still load. Closing the served environment retries it again
and rejects with the unfinished close.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(environment): attempt all state cleanup on close

* fix(environment): bound wide state JSON traversal

* fix(environment): scope cached state reply sizes to requests

* fix(domain): preserve JSON depth diagnostic guidance

---------

Co-authored-by: Edgar Arakelyan <edgar.arakelyan@scale.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
Co-authored-by: gensokyo-dev <gensokyo-dev@users.noreply.github.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 06:06:36 +08:00
Mohith DK f23066f0da fix(reconstruction): fall back to a kind-based title for blank graph labels (#6137)
Fall back to a kind-based obligation title when a graph label is blank

A whitespace-only application observation or semantic node label passed through ?? as the obligation title, and the ledger schema (trimmed, min length 1) then rejected the whole ledger. Use trim() || so a blank label takes the generated title.

Closes #6126
2026-10-22 05:53:42 +08:00
Abel YagubyanandAbelo9996 cbbd317fb5 fix(process): index late scenario events in validation issues (#6160)
Co-authored-by: Abelo9996 <Abelo9996@users.noreply.github.com>
2026-10-22 05:53:22 +08:00
Abel YagubyanandAbelo9996 d54c4345d3 fix(browser): report zero decoded bytes for undecodable gRPC-web text (#6159)
Co-authored-by: Abelo9996 <Abelo9996@users.noreply.github.com>
2026-10-22 05:53:13 +08:00
bdf0be87c6 fix(mcp): name response-byte and startup ceilings in recovery (#6005)
* fix(mcp): name response-byte and startup ceilings in recovery

Name REA_MCP_MAX_RESPONSE_BYTES, its effective value, the 10485760 byte
floor, and process restart when the stdio response budget is hit, and
state that the setting does not raise Node's string ceiling. Name fixed
MCP startup phase budgets when those deadlines fire, and keep Codex's
30 second startup figure as client guidance.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(mcp): distinguish string ceilings and keep dossier recovery

Name REA's reserved string limit separately from Node's ceiling, restore
detail:summary discovery for an oversized complete evidence bundle, and
apply the doctor deadline sentence when binary_session itself times out.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(mcp): keep retained SDK errors inside the text budget

Quoted SDK validation diagnostics expand when recovery JSON is escaped
into MCP text. Shorten only that preview so the retained-evidence
reference still arrives as parseable error content.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(mcp): keep export advice when a summary exceeds the response budget

An oversized get_evidence_bundle summary no longer repeats detail: summary.
export_evidence_bundle still writes the retained bundle outside one MCP frame,
including when a larger byte budget cannot raise the string ceiling.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* style(mcp): format the SDK diagnostic preview return

oxfmt rejected the fitted-preview ternary in EvidenceMcpServer.ts, which failed Static checks and the CI required gate.

Co-authored-by: morluto <morluto@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 05:48:56 +08:00
0155b89953 fix(browser): name web-bundle budgets and recover web deadlines (#6036)
* fix(browser): name web-bundle budgets and recover web deadlines

Name the existing heap, per-script, and total web-bundle phases, and reject a per-script deadline above the total. Promote capture, CDP command, source-map decode, and source-map fetch deadlines at today's defaults. Name DevTools timeoutMs, and state that discovery, cleanup grace, byte budgets, and Electron hook retention have no runtime override.

Refs #5910

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): apply selected web deadlines to the owning phase

Keep HTTP and browser-socket discovery on the fixed discovery deadline so a
short command_timeout_ms starts at the handshake. Apply scenario
command_timeout_ms to launch startup and the primary connect session, and
keep service-worker restoration on the fixed 5000 ms grace. Omit
command_timeout_ms from recovery text when the operation does not accept it.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): retain scenario commands that finish after their deadline

A timed-out scenario command stays owned so cleanup can close a late
browser or context and restore a mutation that lands after the deadline.
Child storage replies use that deadline, scenario recovery names
command_timeout_ms in the capture JSON, and the session test connection
implements beginCleanupGrace.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): keep child command timeouts as the public failure

A nested command such as Page.enable that is delivered but never answered
was stored as protocol_error, hiding the command name, selected deadline,
and recovery. Store the typed timeout and assert that public error.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): keep runtime cleanup on the fixed grace

Switch runtime restoration to the cleanup grace before the first send so a
short command_timeout_ms cannot fail an otherwise successful observation.
Stop advertising command_timeout_ms on list_browser_targets, which never
bounds a post-discovery command.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* test(browser): prove a short command deadline cannot fail runtime cleanup

Drive restoration through the runtime fixture. A cleanup command that
outlives command_timeout_ms still succeeds, and a stalled restoration
names the fixed 5000 ms grace.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): name the effective source-map decode deadline

codecBudgetLimitation reported a fixed 20-second deadline after a
caller selected decode_timeout_ms. Result and Evidence limitations
now name that effective millisecond deadline.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): separate handshake and target-attach deadlines

A browser-level Target.attachToTarget no longer shares the WebSocket
handshake budget. Blocked service-worker attachment keeps discovery on
its fixed deadline and applies command_timeout_ms to the later handshake
and attach.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): name the web-bundle resource each runtime exhausts

Bun ignores the V8 old-space flag. A Windows Job reports the process
commit limit, and other Bun hosts no longer suggest raising max_heap_mb.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): cite the browser-socket discovery message budget

WebSocket discovery enforces the 64 MiB protocol budget. Timeout
recovery no longer advertises the 8 MiB HTTP document budget.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): bound the blocked service-worker check before discovery

The first registration check has its own fixed 10s deadline. Discovery,
handshake, and Target.attachToTarget stay on their separate deadlines.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(browser): name the operand that stopped web-bundle admission

Host heap pressure no longer claims the fixed result budget is exhausted.
A Windows Bun allocation before the commit limit still names that limit.

Co-authored-by: morluto <morluto@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 05:47:38 +08:00
441 changed files with 16670 additions and 3652 deletions

No files matched your search

@@ -22,6 +22,11 @@ requires complete application analysis; `source_paths: []` is invalid, not an
unfiltered request. `detail: "summary"` reduces delivery
and does not by itself reduce analysis work.
ASAR header/index admission is fixed at 16 MiB / 16777216 bytes for this
release and has no runtime override. `source_paths`, `detail`, and
`max_heap_mb` do not bypass or raise it. Analyze an extracted directory, or
supply an archive whose header fits.
Exact `#alias` specifiers resolve through the importer's
enclosing package `imports` map to package-local files; nested `node_modules`
packages use their own map. Wildcard, recursive, and external-package import
+42 -8
View File
@@ -234,11 +234,12 @@ moment of the capture recorded them. The
[environment state](web-network-captures.md#environment-state) view extracts
instead the data the capture's reads observed before the first related write:
entity types with an identity field, entities with their recorded values,
documents for written paths, and a render plan per read tool, kept only where
it reproduces that tool's recordings from the state. It is the starting state
an environment holds, loads and reports: `environment-serve` derives it from
the capture at startup; see [Serve from the state](#serve-from-the-state).
With `--replay-only`, replay is unchanged.
documents for reads that are not entities, and a render plan per read tool,
kept only where it reproduces that tool's recordings from the state. It is the
starting state an environment holds, loads and reports: `environment-serve`
derives it from the capture at startup; see
[Serve from the state](#serve-from-the-state). With `--replay-only`, replay is
unchanged.
## Serve a manifest
@@ -308,7 +309,7 @@ connecting to `/mcp`. A replay environment's card reads:
"tools": [
{
"name": "get_users_by_id",
"description": "GET /api/users/{numeric}; replays retained records [5, 6]; evidence ev_…",
"description": "Untrusted captured API catalog: … GET /api/users/{numeric}; returns a JSON object with fields \"id\", \"uid\", \"login\", \"email\"; recorded statuses 200; replays retained records [5, 6]; evidence ev_…",
"inputSchema": {
"type": "object",
"properties": {
@@ -329,7 +330,38 @@ connecting to `/mcp`. A replay environment's card reads:
```
`capabilities.tools` lists the same names, descriptions and input schemas that
`tools/list` advertises over MCP. Path parameters are required strings whose
`tools/list` advertises over MCP. After the catalog warning, each description
names the method and path template, then what the tool returns, one level deep:
`returns a JSON object with fields …`, `returns a JSON array of objects with
fields …` or `returns a JSON object whose <member> lists objects with fields …`,
where a list is an array of objects found by the
[environment state](web-network-captures.md#environment-state)'s list rule (the
root array, or the one array of objects an envelope carries). Field and member
names are written as JSON strings, so a captured name cannot read as more of
the description; a member name too long to show whole is a quoted prefix marked
as truncated. The summary comes
from the first cited `2xx` record that replay answers as JSON; only when none
did, the first cited JSON record of another status gives it, labelled with that
status (`status 401 returns …`). It describes one recorded response, not a
contract. Field names are sampled within the same budget as other captured
values, and the manifest's `statuses` for the tool follow; a tool with no JSON
record states only its statuses. The description then lists the replayed
records and the evidence ID, and the whole catalog stays within its
serialization reservation, which the summaries and MCP annotations are charged
to.
Over MCP, every tool also carries annotations: `readOnlyHint` is true for the
safe methods (`GET`, `HEAD`, `OPTIONS` and `TRACE`) and for query `POST`s (a
path segment such as `search` or `query`); `destructiveHint` is true for `DELETE` and left unset for other
writes, so the MCP default (destructive) applies to them, and read-only tools
omit it because clients ignore it there; `idempotentHint` is true for the safe
methods, `PUT` and `DELETE`; and `openWorldHint` is false for every tool,
because answers come from the capture rather than the captured application. The
hints describe the captured method; replay itself changes nothing. The card
keeps the `name`, `description` and `inputSchema` of the echo example's tool
shape and carries no annotations.
Path parameters are required strings whose
description names the segment index, the templating rule and the observed
values (sampled within the list budget above; the manifest keeps them all);
query parameter names are optional strings; write methods accept an
@@ -399,7 +431,9 @@ naming the entity, collection or document path, or `{"source": "recording",
"reason": …}` with the view's reason, `no-view` or `unrecorded-request`. A
state answer carries the base record's status, content type and `replay`
provenance with `truncated: false`; one whose JSON would exceed 256 KiB is
refused as a `resource_constraint` tool error rather than cut. Serving is
refused as a `resource_constraint` tool error rather than cut. Extraction does
not state-back a document whose rendered JSON exceeds 256 KiB; that read stays
recorded (`rendered-oversize`) and replays the captured body. Serving is
read-only: `POST`, `PUT`, `PATCH` and `DELETE` tools, other than query `POST`s,
are not served, so nothing a call does changes the state. Query `POST`s stay
served and answer from their recordings with the reason `query-body`.
+40 -19
View File
@@ -385,7 +385,8 @@ section explicitly truncated.
Disallowed-origin entries are filtered before retention.
Selected script-source collection shares a 32 MiB aggregate UTF-8 retention
budget and a 20-second deadline across the capture. An oversized source is
budget and the `command_timeout_ms` deadline (default 20000 ms) across the
capture. The byte budget has no runtime override. An oversized source is
omitted while later smaller sources may still fit. When the deadline expires,
remaining source reads are skipped. Script metadata, per-item omission reasons,
and `script_sources` completeness remain available. A single CDP response can
@@ -393,19 +394,25 @@ still be materialized up to the separate 64 MiB transport limit before source
admission; the retention budget does not bound that transient allocation.
Bundle parsing and semantic inspection run sequentially in an owned child,
without executing captured code. `max_heap_mb` defaults to 1024 MiB V8
old-space; `analysis_timeout_ms` defaults to 30000 per script, including startup
and transfer. A failed or timed-out script becomes an explicit unknown while
completed findings and later scripts remain available. `total_analysis_timeout_ms`
defaults to 300000 across script analysis; expiry returns a typed timeout with
partial Evidence and identifies unstarted scripts. These controls are also
available as CLI flags with hyphenated names. Capture and optional source-map
fetching have separate budgets.
without executing captured code. These are separate phases. `max_heap_mb`
defaults to 1024 MiB V8 old-space (range 128..16384). `analysis_timeout_ms`
defaults to 30000 per script, including startup and transfer.
`total_analysis_timeout_ms` defaults to 300000 across script analysis. A
per-script deadline above the effective total is rejected. A failed or
timed-out script becomes an explicit unknown while completed findings and later
scripts remain available. Total-deadline expiry returns a typed timeout with
partial Evidence and identifies unstarted scripts. `source_map_fetch_timeout_ms`
defaults to 30000 and is an item limitation, not that analysis timeout. These
controls are also available as CLI flags with hyphenated names. Worker protocol
and result byte budgets have no runtime override.
One web-bundle capture/analysis is admitted per REA process at a time. Completed
script results share an 8 MiB serialized transfer budget, with host heap
reservation checked before decoding and validating another result.
Exhausted result capacity stops further script launches and returns a typed
When the host reservation is what reaches zero, the failure names that heap
headroom and the remaining fixed budget. When the 8 MiB allowance itself is
exhausted, the failure names that fixed budget and says it has no runtime
override. Either stop ends further script launches and returns a typed
resource failure with completed facts and explicit unstarted-script unknowns.
Worker control frames are limited to 64 KiB. Cleanup must confirm child termination
before releasing its private files; failed cleanup retains its exact owner and
@@ -415,8 +422,10 @@ facts, including cancellation during final cleanup.
The worker heap setting is not a process-family RSS limit. Node enforces V8
old-space flags; Bun ignores them. Windows Bun instead uses an owned process
commit limit with a 512 MiB runtime reserve; other Bun hosts have no enforced
memory limit. These bounds do not establish an aggregate host-memory limit
across other REA operations or external processes.
memory limit. An allocation failure before that Windows limit is reported as
reached still names the configured limit and says it was not reached. These
bounds do not establish an aggregate host-memory limit across other REA
operations or external processes.
Resource-tree inventories distinguish a valid empty `resources` array from a
missing or malformed collection. Invalid resource entries are omitted with
@@ -430,12 +439,24 @@ an invalid supplied size remains unknown with incomplete resource coverage while
the resource's other facts are retained. Zero and nonnegative fractional sizes
are preserved.
CDP discovery has a 10-second deadline. Each WebSocket handshake and command
response has its own 20-second deadline, separate from the caller-selected
observation window. A stalled handshake or command reports `provider_timeout`;
caller cancellation still reports cancellation. WebSocket messages remain subject
to the [64 MiB limit](#request-boundary). REA does not impose a message-count or
pending-command quota. On cancellation, it closes only its own WebSocket,
CDP discovery has a 10-second response deadline. HTTP discovery documents
use an 8 MiB budget. Browser-socket discovery (`Browser.getVersion`,
`Target.getTargets`) uses the 64 MiB protocol message budget. Neither budget
nor the discovery deadline has a runtime override. `list_browser_targets` accepts no
`command_timeout_ms`: HTTP discovery and browser-socket discovery
(`Browser.getVersion`, `Target.getTargets`) stay on that fixed deadline, and
the socket closes without a later command. Other CDP tools use
`command_timeout_ms` (default 20000) for each WebSocket handshake and command
response, separate from the caller-selected observation window, discovery, and
the 5000 ms cleanup grace. A browser-level socket's handshake and the following
`Target.attachToTarget` each receive that full deadline; they do not share one
budget. A stalled handshake or command reports
`provider_timeout` and names `command_timeout_ms` only when that operation
accepts it; caller cancellation still reports cancellation. WebSocket messages
remain subject to the [64 MiB limit](#request-boundary), which has no runtime
override. REA does
not impose a message-count or pending-command quota. On cancellation, it closes
only its own WebSocket,
waiting up to one second before terminating it. Malformed unsolicited events
and malformed replies to pending commands poison the connection so later
commands fail closed. A correlated reply's session ID must match the command's
@@ -444,7 +465,7 @@ rejects it as unknown; other command rejections retain their reported failure.
Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete.
Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.
Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation uses `source_map_fetch_timeout_ms` (default 30000) and a 64 MiB aggregate streamed-response budget across redirects. The byte budget has no runtime override. A fetch overrun is an item limitation, not a thrown analysis timeout. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.
source-map items retain the script's `declared_url` separately from the observed
`effective_url`. successful maps use the latter as their source-resolution base.
+15 -3
View File
@@ -23,9 +23,21 @@ that rejection does not roll back browser effects. REA removes its owned
future-document scripts, restores live frame methods, then disconnects.
Add secret declarations, storage seeds, and additional artifact or event
capture only when needed. Action, secret, and storage counts are not capped.
Duration, action, and navigation timeouts remain fixed provider-owned liveness
limits. `capture.limits` controls retained event and network-content budgets;
it does not change those liveness deadlines.
Duration, action, and navigation timeouts remain separate from CDP command
deadlines. `command_timeout_ms` (default 20000; omit it to keep that default
and leave it out of Evidence) bounds launch startup, the connect-over-CDP
handshake, target lookup, environment commands, and storage CDP commands,
including replies from attached child targets. Set it in the scenario JSON.
`capture-browser-scenario` has no `--command-timeout-ms` option.
It does not change action `timeout_ms`, `wait_for_timeout.duration_ms`, or
navigation. Service-worker restoration keeps a fixed 5000 ms grace and does
not inherit a raised `command_timeout_ms`. CDP discovery stays on its own
fixed deadline. When service workers are blocked, the registration check
before discovery has its own fixed 10000 ms deadline; `command_timeout_ms`
does not apply because discovery has not started. The discovery deadline
covers endpoint discovery only. The following WebSocket handshake and
`Target.attachToTarget` each use `command_timeout_ms`. `capture.limits` controls retained event and network-content
budgets; it does not change those liveness deadlines.
## Minimal launch example
+16 -1
View File
@@ -63,6 +63,16 @@ or repeat it to select several types.
Selection limits decoding to the selected types; member references remain an
assembly-wide inventory, so the output can still be large.
Select `--detail summary --evidence-bundle /absolute/path/members.json` to
receive identity, selection, coverage, limitations and collection counts while
saving the complete original Evidence in a new bundle. Summary requires that
explicit destination; existing files are preserved. Later
`inspect-analysis-view` calls can page the exact parent from the saved bundle
without reopening the assembly. See
[CLI summary and saved member rows](managed-code-analysis.md#cli-summary-and-saved-member-rows)
for the complete command sequence. Omitted or `--detail complete` retains the
full inline response.
Choose the workflow from the artifact's contents rather than its `.exe` or
`.dll` extension. The managed parser requires PE/CLI metadata; it does not
unpack .NET single-file hosts or decode IL2CPP. For the image, type and method
@@ -279,7 +289,12 @@ Exports preserve an existing destination unless `--overwrite` is explicit.
For application traces and version comparisons, see
[JavaScript application workflows](javascript-application-workflows.md).
MCP [retained Evidence references](mcp-contracts.md#retained-application-evidence-inputs)
belong to one connection. Separate CLI calls consume full saved records.
belong to one connection. Separate CLI calls can consume full saved records.
`inspect-analysis-view REQUEST.json --evidence-bundle BUNDLE.json` resolves
the request's `retained-evidence` source ID from that authenticated bundle,
including bundles exported from an MCP session. The same view JSON works for
every supported parent kind; no target is reopened and no record is selected
by file order. Omit `--evidence-bundle` when the request supplies inline Evidence.
## Import historical source
+7 -1
View File
@@ -245,4 +245,10 @@ application path and capture outcome to the static graph, while frames, scripts,
workers, and execution claims remain unavailable until a separate passive runtime
capture provides them.
ASAR observation reads the header through one open file handle, validates its declared lengths against the archive size, and limits encoded JSON headers to 16 MiB before parsing. The header cache also retains at most 16 MiB of encoded headers across eight archives. Oversized headers produce a scope refusal.
ASAR observation reads the header through one open file handle and validates its
declared lengths against the archive size. Header/index admission is the same
fixed 16 MiB / 16777216 byte budget used by static ASAR reads. This release has
no runtime override. An oversized header is not parsed; selecting a member path
does not bypass admission, and the page stays unauthorized. The header cache
separately retains at most 16 MiB of already-admitted encoded headers across
eight archives. Evicting that cache does not raise admission.
+11 -1
View File
@@ -106,6 +106,15 @@ only reduces delivery; source selection reduces text acquisition and analysis.
Explicit native provider or snapshot routing rejects `--source-path` rather
than ignoring it.
ASAR header/index admission is a fixed 16 MiB / 16777216 bytes for this
release. There is no runtime override. The archive index is read before any
member can be selected, so `source_paths`, `--source-path`, `detail:
"summary"`, `max_heap_mb`, and available host memory do not bypass or raise
it. When the header exceeds that allowance, the failure names this budget.
Analyze an extracted application directory, which does not admit an ASAR
header, or supply an archive whose header fits. Repeating the unchanged
request fails the same way.
## Read the analysis
`analyze_javascript_application` returns the complete analysis Evidence by
@@ -172,7 +181,8 @@ for the tool-result contract.
Use MCP `detail: "summary"` for a large application, then read selected views of
the retained analysis. Summary changes delivery only. To reduce source parsing,
select exact [source paths](#select-sources-before-analysis).
select exact [source paths](#select-sources-before-analysis). Source selection
does not bypass ASAR header/index admission.
If a complete response exceeds the transport budget, use its retained Evidence
ID for a summary, module page, or item, or export the bundle. See
+6 -1
View File
@@ -106,7 +106,12 @@ nodes. REA evaluates compatible static candidates conservatively:
that disagreement. If a mapped candidate's digest is unavailable and no
mapped content match is known, the comparison remains `unknown`.
6. Missing authorization mappings and incomplete or truncated input coverage
remain unknown or truncated rather than becoming absence claims.
remain unknown or truncated rather than becoming absence claims. A runtime
location with no automatic artifact-root or caller-declared mapping into
any layer, and no content match, is `unknown` with
`no-authorized-location-mapping`, with or without captured bytes; several
content matches remain `ambiguous` under rule 4. Only a mapped location can
be `unmatched` with `no-static-content-match`.
Every successful cross-layer `observed_as` edge has
`authority: cross-layer-reconciliation` and `state: inferred`. The underlying
+1 -1
View File
@@ -68,7 +68,7 @@ The background export belongs to the session. `close_binary`, or the end of a CL
The assembly label index is scanned once per export reader. Cancelling a request stops its wait without interrupting other readers; closing the session aborts and drains the shared scan before releasing its files.
Regex procedure and string search uses ECMAScript Unicode matching in a cancellable worker with a five-second matching deadline and a 128 MiB worker heap. Literal matching remains the default. Cancellation or a matching deadline leaves the session available. This release has no runtime override for that deadline or heap.
Regex procedure and string search uses ECMAScript Unicode matching in a cancellable worker with a five-second matching deadline and a worker old-generation heap. Literal matching remains the default. Cancellation or a matching deadline leaves the session available. This release has no runtime override for that deadline. The worker requests 128 MiB. A positive Node.js `--max-old-space-size` on the process that starts REA, including `NODE_OPTIONS`, overrides that request and sets the effective heap. `--max-old-space-size-percentage` takes precedence over that flag in either order. REA does not turn the percentage into a MiB, so the effective heap stays unestablished instead of the 128 MiB request or the absolute flag. The reported `NODE_OPTIONS` value is the process initial environment when that block can be read. A later edit to the environment is not the value Node applied, and an unreadable initial value leaves the effective heap unestablished. The launch Node options are the initial command line on Linux when that buffer still has a program entry, not a later edit to `process.execArgv`. A preload can clear that array after V8 has applied the flags. Setting `process.title` or `--title` overwrites the command line, so a buffer without a program entry leaves the effective heap unestablished. `--env-file` or `--env-file-if-exists` can supply `NODE_OPTIONS` when the initial environment has none. That file is not part of the initial environment block, so the effective heap stays unestablished instead of the 128 MiB request. An initial `NODE_OPTIONS` is still used, because Node does not let the file replace it. A positive `--max-heap-size`, including the underscore form and launch `NODE_OPTIONS` on Node versions that allow it, sets the total heap rather than the old generation and can be lower than the 128 MiB request. The effective old generation stays unestablished while that value is in effect. A zero or empty value clears it. `--experimental-config-file` or `--experimental-default-config-file` can set `nodeOptions`, including a heap percentage, from a file that is not read back. The effective heap stays unestablished when either flag is present, including when an absolute `--max-old-space-size` is also on the command line or in the recovered `NODE_OPTIONS`. Node accepts the config path as `=` or the next argument.
Single-function subprocess output and each cached source facet share a 16 MiB buffered-text capacity. Cached spans are checked before allocation; larger text remains in the local export and the error identifies the file and range. `analyze_function` initially returns the complete dossier, which can still be large; later retained views do not remove that first transfer. Prefer the individual text tools for a focused question.
+50 -4
View File
@@ -85,7 +85,7 @@ rejects selected summary before analysis; complete delivery remains available.
Complete `inspect_managed_members` delivery always includes the assembly-wide
`member_refs` and `exported_types`, which `type_names` does not narrow, so even
one selected or unmatched type can return megabytes. Select
one selected or unmatched type can return megabytes. In MCP, select
`{"path":"/absolute/path/Assembly.dll","type_names":["Namespace.Type"],"detail":"summary"}`
to keep the complete source in the session and receive identity, metadata
status, the selection with its unmatched names, coverage/issues, the separate
@@ -115,6 +115,49 @@ literal pool's own coverage. Older Evidence that never examined exported types
or user strings reports a null count and refuses those pages rather than
returning an empty one.
#### CLI summary and saved member rows
The CLI accepts the same `complete` and `summary` detail choices. Omitted or
`--detail complete` returns complete Evidence. To receive a summary, choose a
new Evidence bundle destination so its original complete parent remains
available after the command exits:
```bash
rea inspect-managed-members /absolute/path/Assembly.dll --type Namespace.Type \
--detail summary --evidence-bundle /absolute/path/evidence/members.json --json
```
`--evidence-bundle` writes the complete original Evidence in REA's canonical
bundle format. Its parent directory must exist, and it never replaces an
existing path. Relative paths resolve against the command's working directory.
The option also works with complete delivery when a saved bundle is useful.
Summary without a destination is rejected before analysis. A publication failure
returns a typed error with the completed Evidence under
`details.partial_observation`; it does not report successful summary delivery.
Cancellation before publication prevents the new output. Once atomic
publication has started, it can complete despite cancellation; a cancelled
delivery then retains the committed path and byte count under
`details.diagnostics.evidence_bundle`.
Copy `normalized_result.parent_evidence_id` from the summary into the
`source.evidence_id` field of the `managed-members` request above, then save that
small request as `members-page.json` and run:
```bash
rea inspect-analysis-view /absolute/path/evidence/members-page.json \
--evidence-bundle /absolute/path/evidence/members.json --json
```
The view command authenticates the selected bundle and resolves that exact
parent ID. It does not reopen the assembly, so the original target may have
been moved or removed. Subsequent calls can select another collection or use
the returned `coverage.next_offset` to continue the same collection. An inline
source already includes its parent and cannot also select `--evidence-bundle`.
Summary changes delivery, while type selection still controls decoding and the
same inspection resource limits apply. Each separate CLI page reads and
authenticates the saved bundle; the full source is retained on disk.
## Shipped scope
The canonical parser admits PE/CLI bytes and their metadata/CIL without loading
@@ -622,6 +665,9 @@ which managed methods executed or how the CLR resolved them.
Managed member comparisons report a `metadata` dimension when an observed
MethodDef flags or implementation flags value differs, including accessibility
or synchronization changes. This is separate from the CIL/signature matching
tiers: identical instructions do not establish unchanged method metadata. The
reported difference is static metadata evidence, not proof of runtime behavior.
or synchronization changes. Observed method-body `init_locals` and `max_stack`
differences also contribute to `metadata`, even when CIL decoding is incomplete.
An unavailable header value keeps body coverage unknown rather than establishing
equality or a difference. These comparisons are separate from the CIL/signature
matching tiers: identical instructions do not establish unchanged method metadata.
The reported difference is static metadata evidence, not proof of runtime behavior.
+9 -6
View File
@@ -234,11 +234,14 @@ The destination changes only when a complete export is atomically published.
Clients that explicitly configure a larger receive buffer can set the REA
server's `REA_MCP_MAX_RESPONSE_BYTES` environment variable to the same byte
count. This setting must be a safe decimal integer at least 10485760; REA
reserves 1024 bytes for the envelope. Raising it restores complete inline
delivery for responses that fit that buffer and Node's single-string limit.
It does not change the client's buffer, analysis coverage, or retained content.
Ordinary responses keep their existing complete result contract.
count. This setting must be a safe decimal integer at least 10485760 (10 MiB);
REA reserves 1024 bytes for the envelope. The value is read when the REA
process starts, so a change requires a restart and a matching client receive
buffer. Raising it restores complete inline delivery for responses that fit
that buffer. It does not raise REA's reserved single-string limit, which is
1024 UTF-16 code units below Node's single-string ceiling, and it does not
raise that Node ceiling. It does not change analysis coverage or retained
content. Ordinary responses keep their existing complete result contract.
### Error recovery and deadline ownership
@@ -329,7 +332,7 @@ See [environment manifests](web-network-captures.md#environment-manifest).
`inspect_analysis_view` also accepts `environment-state` for the same parents,
extracting the initial state the recorded reads observed: entity types with an
identity field, entities, documents for written non-entity paths, and a render
identity field, entities, documents for non-entity reads, and a render
plan per read tool that reproduces its recordings from the state or names why it
cannot. Types and identity are inferences, so the Evidence is always inferred.
See [environment state](web-network-captures.md#environment-state).
+31 -3
View File
@@ -202,10 +202,38 @@ Function dossiers retain this same string evidence. Native call edges retain
Hopper's partial `CallReference` classification and exact endpoints; detailed
reference flags remain unavailable rather than being invented.
Regex searches use ECMAScript Unicode syntax in a cancellable worker with a
five-second matching deadline and a 128 MiB worker heap. Deadline or
five-second matching deadline and a worker old-generation heap. Deadline or
cancellation stops matching while leaving the Hopper API available. This
release has no runtime override for that deadline or heap. Literal mode
retains Unicode casefold matching.
release has no runtime override for that deadline. The worker requests 128
MiB. A positive Node.js `--max-old-space-size` on the process that starts
REA, including `NODE_OPTIONS`, overrides that request and sets the effective
heap. `--max-old-space-size-percentage` takes precedence over that flag in
either order. REA does not turn the percentage into a MiB, so the effective
heap stays unestablished instead of the 128 MiB request or the absolute flag.
The reported `NODE_OPTIONS` value is the process initial environment when that
block can be read. A later edit to the environment is not the value Node
applied, and an unreadable initial value leaves the effective heap
unestablished. The launch Node options are the initial command line on Linux
when that buffer still has a program entry, not a later edit to
`process.execArgv`. A preload can clear that array after V8 has applied the
flags. Setting `process.title` or `--title` overwrites the command line, so a
buffer without a program entry leaves the effective heap unestablished.
`--env-file` or `--env-file-if-exists` can supply `NODE_OPTIONS`
when the initial environment has none. That file is not part of the initial
environment block, so the effective heap stays unestablished instead of the
128 MiB request. An initial `NODE_OPTIONS` is still used, because Node does
not let the file replace it. A positive `--max-heap-size`, including the
underscore form and launch `NODE_OPTIONS` on Node versions that allow it,
sets the total heap rather than the old generation and can be lower than the
128 MiB request. The effective old generation stays unestablished while that
value is in effect. A zero or empty value clears it.
`--experimental-config-file` or `--experimental-default-config-file` can set
`nodeOptions`, including a heap percentage, from a file that is not read
back. The effective heap stays unestablished when either flag is present,
including when an absolute `--max-old-space-size` is also on the command
line or in the recovered `NODE_OPTIONS`. Node accepts the config path as
`=` or the next argument.
Literal mode retains Unicode casefold matching.
## Bounded static traces
@@ -0,0 +1,32 @@
# Implementation plan
1. In `environmentStateRules.ts`, parse pointers into tokens and let `at` and
`replaceAt` read and replace element 0 of a one-element array at the root
or in one member; extend `locateItem` with those forms after the object
forms, requiring an identity name there; add `identityCandidates`, the
identity names then suffix-named fields, and let `locateList` prefer arrays
identified by name, then arrays of two or more items identified by suffix.
2. In `EnvironmentStateExtraction.ts`, plan lists with the shared candidates,
treat an empty recording as unconstraining, accept a suffix-named field only
once a recording shows it distinct across two or more items, leave reads
without a list or identity to documents, drop the written-path requirement
and the `not-entity` fallback, record `path-varies` apart from
`query-varies`, and keep each entity's earliest sighting;
`environmentStateMerge.ts` cites records in timeline order. Remove
`not-entity` from the reason type and result schema, add `path-varies`, and
describe the new pointer forms and `field-name` basis.
3. Move the classification of retained bodies (none, base64, cut, JSON, text)
into `replayedBody` in `environmentSource.ts`, which owns recorded
exchanges; replay renders from it and tool descriptions read its JSON case.
4. Make `environmentToolDefinitions` take the `EnvironmentSource`; describe the
first `2xx` JSON record, else the first JSON record labelled with its status,
one level deep with the state's list rule and the existing sample budget;
list the manifest tool's statuses; and charge the whole description and the
tool's MCP annotations to the catalog reservation. Update the serve and
bundle callers.
5. Let the tool definitions own the MCP annotations derived from the method and
the query-`POST` rule; `serveEnvironment.ts` registers them and the card
leaves them out.
6. Cover domain, services, adapters and the compiled MCP boundary; document the
rules in the capture guide, descriptions and hints in the agent environments
guide, and the contract text.
@@ -0,0 +1,99 @@
# Wider environment state coverage and tool descriptions
Issue: https://github.com/morluto/rea/issues/4933
## Outcome
The `environment-state` view backs more read tools with state, and a served
environment tells agents more about each tool. Read-only paths whose responses
are not entities become documents, listed items may be identified by an
`…Id` or `…_id` field, an item wrapped in a one-element array is located
inside it, and reads that vary by path get their own recorded reason. Each tool description says what the tool returns and which statuses
were recorded, and every MCP tool carries method hints. This increment adds no
tool, view kind, provider, network activity or write effect.
## Requirements
- A `GET` list tool (its template does not end in a parameter) that yields no
item or list plan holds its latest initial value as a `document` when its
initial reads have one request line, whether or not the capture writes its
path, unless its value changed between two of those reads across any
recorded write (even under another root), which leaves it `fields-differ`. A
value whose rendered JSON exceeds 256 KiB stays `rendered-oversize` and is
replayed from the recorded body, which itself fit. More request lines give
`path-varies` when they differ by path (a template with a parameter before
its end) and `query-varies` when they share one path. With every such read
now a document, `path-varies`, `query-varies`, `body-unusable` or
`rendered-oversize`, the unreachable `not-entity` reason leaves the result
schema, and `no-identity` names item tools only. A document holds the value
of its one `path`.
- When no path parameter proves an identity and no identity name identifies
the listed items, the `field-name` identity is the first key of the first
listed item, in its order, whose name ends in `Id` or `_id`, that every item
carries as a distinct string or number, and that at least one initial
recording shows distinct across two or more items. The last condition goes
beyond "every item carries it distinctly": one item satisfies that
vacuously, and a lone item's suffix-named fields are often foreign keys (a
reaction keyed by its user, a comment by its post), so a list that only ever
held one item stays a document. Empty recordings constrain no field.
Identity names keep precedence, also when choosing among an envelope's
arrays, where a suffix-named field identifies only an array of two or more
items.
- An item response that is a root array of exactly one object, or an object
with one member holding such an array, locates its entity at `/0` or
`/<member>/0` when one of the identity names in that object repeats the path
value. Objects at the root or in one member keep precedence. `at` and
`replaceAt` read and replace exactly these forms besides the root and one
member, so the gate and renderers run unchanged.
- An entity's state order is its earliest sighting on the timeline and it cites
its records in timeline order, whichever plan observed it first.
- Each tool description reads `METHOD template; <returns>; recorded statuses
…; replays retained records […]; evidence ev_…` after the untrusted-catalog
warning. `<returns>` describes one level deep the first cited `2xx` record
replay answers as JSON: an object's fields, an array of objects' fields, or
an envelope whose member lists objects, where a list is an array of objects
found by the state's list rule; an empty field set reads `with no fields`.
Only when no `2xx` record replays as JSON does the first cited JSON record of
another status give it, labelled `status <code> returns …`; with no JSON
record it is omitted. The statuses are the manifest tool's `statuses`, the
one owner of that set (`no recorded status` when it is empty). Field names
and statuses are sampled within the description sample budget and charged
with the whole description to the catalog reservation before it is retained.
One owner in the environment source classifies each retained body as none,
base64, cut, JSON or text; replay answers from that classification and
descriptions read only its JSON case.
- MCP tools carry `readOnlyHint` (true for the safe methods `GET`, `HEAD`,
`OPTIONS` and `TRACE`, and for query `POST`s),
`destructiveHint: true` for `DELETE` (unset for other writes, so the MCP
default applies, and for reads, where clients ignore it), `idempotentHint`
(true for the safe methods, `PUT` and `DELETE`) and `openWorldHint: false`. The
tool definitions own the hints and charge them to the catalog reservation;
the environment card keeps its `name`, `description`, `inputSchema` tool
shape.
## Acceptance and limits
Synthetic captures cover read-only documents, `path-varies` and
`query-varies`, suffix-named identities with their key order, their
two-item proof, empty first recordings and their precedence (including the
choice between an envelope's arrays), a Ghost-like `{"posts":[...]}` item under
a trailing-slash path merging with its list view, a root one-element array, a
one-element filtered list that stays unidentified, and the pointer forms. The
Grafana-shaped fixture covers documents for its read-only paths (its one-member
team list among them) and object, array and enveloped-list descriptions;
synthetic sources cover the `2xx` preference over an earlier error body, the
first of several JSON records, the labelled fallback, the manifest's statuses,
empty, scalar and fieldless bodies, a truncated member name, a name whose
JSON quoting exceeds the sample, a recorded body under 256 KiB whose rendered
JSON exceeds it, non-JSON bodies and the catalog budget; the in-process and
compiled MCP servers advertise the
hints. A filtered list may still answer with one element whose identity name
repeats the requested value; list filters, pagination, lossless big integers
and cross-endpoint entity linking remain out of scope. Documents inline whole
recorded values and enlarge the state result; agent tokens, CPU time and peak
memory of the state and description build were not measured.
## Deferred issue criteria
Serving from the state, the AgentEnv data plane and write effects remain later
increments of #4933. This PR must reference rather than close the issue.
@@ -0,0 +1,16 @@
# Tasks
- [x] Record the coverage and description contract and its limits.
- [x] Extend pointers, item location and identity candidates.
- [x] Require two distinct items before a suffix-named field identifies a list.
- [x] Hold read-only non-entity reads as documents, retire `not-entity` and
separate `path-varies` from `query-varies`.
- [x] Order entities by earliest sighting and cite records in timeline order.
- [x] Classify retained bodies in one owner and describe tool responses,
preferring `2xx` records, with the manifest's statuses.
- [x] Advertise MCP tool annotations, charged to the catalog, without changing
the card.
- [x] Add domain, services, adapters and compiled-boundary coverage.
- [x] Document the rules, descriptions and hints; update the contract text.
- [x] Run static, formatting, duplication, documentation and test gates and the
real-capture measurement.
@@ -0,0 +1,123 @@
# Verification
Verified base: 7a668b73f4783b42eb0e8ce1eeff079f86d53e22 (the environment-state
view branch).
Executed on the implementation worktree with its existing locked installation
under Node.js v22.23.1 (the repository pins 24.18.0, which was not available);
every check below ran as written in this pass, after the last source change.
- `npm run format` then `npm run format:check`: passed.
- `npm run check:fast`: typecheck and lint passed (3959 files, no errors). The
touched files carry two lint warnings, both present on the base: `max-params`
in `environmentReplay.ts` and an unused `_` in `environmentCapacity.test.ts`.
- `npx knip`: passed (exit 0).
- `npm run jscpd`: 3.40% duplicated lines over 3348 files, under the 3.5%
gate. The three clones that touch changed test files pair the same blocks as
on the base; none involves code this change adds.
- `npm run scan:todos`: no markers.
- `npm run docs:check`: the built documentation checks and the README
consistency conformance test (5 tests) passed.
- `npx vitest run --project domain --project services`: 341 files, 5859 tests
passed, 5 pre-existing skips.
- `npm run build:cached`: passed.
- `npm run test:focused --` the environment serve adapters and capacity tests,
the environment serve MCP boundary, the environment serve and export CLI
boundaries, the environment serve shutdown process boundary, the
analysis-view MCP and CLI boundaries, the compact input-schema contract and
MCP boundary, the tool-schema validity boundary, the analysis-detail and
advertised-input-example contracts and the qlog acceptance test: 14 files,
113 tests passed, 1 pre-existing skip.
Real captures (outside the test suite): the built view ran through the
application service on the same 44 recorded captures (39 HAR, 5 browser
scenario) with all three template rules, in 7 to 50 ms each. 42 produced a
state, and the same two browser-scenario captures are refused with
`resource_constraint` as on the base. The base column is the brief's baseline,
which the same script reproduced on the base build in an earlier pass; this
pass did not rebuild the base.
| Measure | Base | This change |
| ---------------------------------- | ----- | ----------- |
| Types | 568 | 576 |
| Entities | 2,213 | 2,246 |
| Documents | 2 | 299 |
| Item views | 211 | 213 |
| List views | 497 | 504 |
| Document views | 2 | 299 |
| State-backed read tools (of 1,258) | 710 | 1,016 |
| Recorded: `not-entity` | 306 | (retired) |
| Recorded: `no-identity` | 129 | 32 |
| Recorded: `path-varies` | (new) | 32 |
| Recorded: `query-varies` | 5 | 69 |
| Recorded: `body-unusable` | 32 | 32 |
| Recorded: `list-differs` | 31 | 31 |
| Recorded: `no-init-recording` | 26 | 26 |
| Recorded: `query-body` | 12 | 12 |
| Recorded: `fields-differ` | 3 | 4 |
| Recorded: `envelope-varies` | 4 | 4 |
Seven list types gained a suffix-named identity (`permissionId`, `product_id`,
`ItemId`, `user_id`, `channel_id`, `spl_id` and `eventId`). Seven more had one
before the two-item proof and lost it, because every initial response listed a
single item: an Airbyte organization, a Mattermost reaction and team
membership, and a Redpanda broker and schema reference became documents; a
NOAA station read is `query-varies` and a Redpanda consumer read
`path-varies`. Two Ghost item tools render from `/members/0` and `/tags/0`. A
third Ghost item tool is located at `/posts/0` but stays `fields-differ`: its
recordings differ by an `include` query.
State result size, as the JSON length of each Evidence summed over the 42
states: 2,677 KiB on the base and 3,196 KiB now (+19%), measured by the same
script. The largest growth per capture is Metabase (133 to 236 KiB), Directus
(21 to 93), Redpanda Console (26 to 68) and Vault (9 to 43). Documents take
492,516 of the 3,274,243 UTF-8 bytes of the results now; the base held two.
Agent tokens, CPU time and peak memory of the state view were not measured.
Tool descriptions on the 44 captures' manifests: all 44 catalogs were admitted
within their reservation. 1,336 of 1,400 tools state what they return: 1,333
from a `2xx` record and three labelled with another status (Miniflux and
NocoDB 404, Shlink 501), because none of their records succeeded as JSON. The
summaries add 261,502 bytes to a 1,185,893-byte catalog of names, descriptions
and input schemas (median 136, 95th percentile 554, maximum 1,133 bytes per
tool); five reached the field sample bound (one Keycloak, one Lychee and three
Sentry tools with 46 to 105 fields). The MCP annotations add 113,182 bytes to
`tools/list` and none to the card. Building a catalog took a median 1.6 ms and
at most 7.6 ms (Metabase), after the source was loaded. Agent tokens, CPU time
and peak memory of the description build were not measured.
After merging the environment state view's review fixes (its branch at
6ad63bcc, which also brings main's evidence-file sources), every gate ran
again on the merged tree: `npm run check:fast`, `npm run format:check`,
`npx knip`, `npm run scan:todos` and `npm run docs:check` passed; `npm run
jscpd`: 3.39% duplicated lines over 3362 files; `npx vitest run --project
domain --project services`: 344 files, 5914 tests passed, 5 pre-existing
skips; `npm run build:cached`, then `npm run test:focused --` the serve and
tool-definition adapters, the serve MCP and CLI boundaries, the export CLI
boundary, the analysis-view MCP and CLI boundaries, the compact input-schema
budget and the analysis-detail and advertised-input-example contracts: 10
files, 98 tests passed.
The real-capture comparison was repeated on the merged code, with the base
measured on the merged state view without this change: all 44 captures now
produce a state, and 1,011 of 1,277 read tools are state-backed against 625
on the base (216 item, 416 list and 379 document views against 214, 409 and
2). Recorded reasons: `no-identity` 216 to 32 and `not-entity` 310 to none;
`query-varies` 5 to 76 and `path-varies` 36 are document reads under more
than one query or path; `list-differs` 35, `body-unusable` 33,
`no-init-recording` 28, `query-body` 12, `envelope-varies` 4 are unchanged,
and `fields-differ` is 9 to 10.
Not covered here: serving from the state, the AgentEnv data plane and write
effects, which are later increments.
## After the squash onto main
The branch was squashed onto main once the state-serve branch merged (166fb082). Against a merge of the previous head
into that main, the squash keeps both sides of every conflict and drops nothing of this branch. Two tests then failed
on it and were refitted with the oversized-document fix in 2a926a0a: the bundle test of a state catalog over the
discovery budget uses 300 read tools, inside the window where the state's catalog is refused and replay's, with its
response summaries, still fits (280–320 by a probe), and the rea-state validation test names the pushed document by
the count of documents the state holds. The replay-only card test pinned main's 11,131-byte card; it now pins this
branch's 12,818-byte card, whose descriptions name each tool's recorded responses. The branch then merged main at
d7ca93cc without conflicts.
+38 -19
View File
@@ -522,9 +522,9 @@ Pass the same parent to `inspect_analysis_view` with
`view: {"kind":"environment-state"}` to extract the initial state the
capture's reads observed, and a render plan per read tool that reproduces those
reads from it. Where the environment manifest names an application's tools, the
state holds its data: entities by type and identity, plus documents for written
paths whose reads are not entities. Use the `template_rules` of the manifest the
state accompanies so tool names match; the view groups records the same way.
state holds its data: entities by type and identity, plus documents for reads
that are not entities. Use the `template_rules` of the manifest the state
accompanies so tool names match; the view groups records the same way.
```json
{
@@ -560,13 +560,27 @@ refused for them, as the manifest is not.
Each `GET` tool whose template ends in a parameter is an item tool, and its
type's `collection_template` is the template without that parameter; any other
`GET` tool is a list tool of its own template. An item tool proves its type's
`GET` tool is a list tool of its own template. Empty segments are ignored, so a
trailing slash names the same collection. An item tool proves its type's
`id_field` when every initial response carries the requested path value in the
same field, at the root or in one wrapper member (`id_basis: "path-parameter"`).
Otherwise the field is the first of `id`, `uid`, `uuid`, `_id`, `ID`, `UID`,
`Id`, `guid`, `key`, `slug` and `name` that every listed item carries as a
distinct string or number (`id_basis: "field-name"`). A list is the response
array, or the one array of objects in an envelope that is not itself an entity.
same field of one entity object (`id_basis: "path-parameter"`). The object is
the response itself or one wrapper member; failing both, it is the only element
of a one-element array at the root (pointer `/0`) or in one member
(`/<member>/0`), as in `{"posts":[{...}]}` or `{"data":[{...}]}`, provided one
of the identity names below repeats the value, since a filtered list may also
answer with one element that repeats its parent's value. Otherwise the field is
the first of `id`, `uid`, `uuid`, `_id`, `ID`, `UID`, `Id`, `guid`, `key`,
`slug` and `name` that every listed item carries as a distinct string or
number; when none does, it is the first key of the first listed item, in its
order, whose name ends in `Id` or `_id` (such as `orgId` or `user_id`), that
every item carries as a distinct string or number, and that at least one
initial response shows distinct across two or more items
(`id_basis: "field-name"`). A single listed item is distinct from nothing, and
its suffix-named fields are often foreign keys, so a list that only ever held
one item is left to a document. Empty initial lists constrain no field. A list
is the response array, or the one array of objects in an envelope that is not
itself an entity, preferring an array whose items an identity name identifies,
then an array of two or more items a suffix-named field identifies.
`entities` merge every initial observation of one identity within one
collection path: an item view's latest values win over a list view's, a view
@@ -583,9 +597,9 @@ Every entity cites its `record_ordinals`.
| Shape | Plan |
| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `item` | The entity object at `pointer` (the root or one member), the ordered `fields` the view shows and the `required_fields` every recording had, and `envelope_ordinal`: the recording whose response, with the entity replaced, every item response shares |
| `item` | The entity object at `pointer` (the root, one member, or `/0` or `/<member>/0`), the ordered `fields` the view shows and the `required_fields` every recording had, and `envelope_ordinal`: the recording whose response, with the entity replaced, every item response shares |
| `list` | The array at `pointer`, its `fields`, `membership` (`all`: every entity of the collection in state order, when each recording listed exactly that, had no query, no item-total member (`total`, `count`, `totalCount`, `item_count`, `totalcount`, `itemcount`, but not `total_pages` or `per_page`; a number or a string of digits) differing from the number listed, and no `has_more`, `next` or next-page token member set; `recorded`: the entities each recording listed) and `counters` that equal the number listed |
| `document` | The path's latest initial value, held in `documents` |
| `document` | The latest initial value of a list tool whose responses hold no identified list, when its initial reads share one request line; held in `documents` with that `path`, whether or not the capture writes it, unless its value changed across any recorded write between those reads (`fields-differ`) |
| `recorded` | A `reason` the tool replays its recordings instead |
An item or list plan is kept only when it reproduces the latest initial
@@ -594,11 +608,13 @@ regard to member order; a volatile field compares its recorded value. Each
plan lists the `checked_record_ordinals`. Otherwise the tool is a recorded view
with reason `envelope-varies`, `fields-differ` or `list-differs`; the other
reasons are `query-body` (a query `POST`), `no-init-recording`,
`body-unusable`, `not-entity`, `no-identity` and `query-varies` (a document
read under several queries). Documents are held only for list-shaped paths
that a `POST`, `PUT` or `PATCH` tool writes: a non-entity read of an item path
such as `/api/files/{id}`, or of a path that is only deleted, stays recorded
with `no-identity` or `not-entity`. A body is usable when its content type is
`body-unusable`, `no-identity` (an item read without a proved identity),
`path-varies` (another read under more than one path, as a template with a
parameter before its end), `query-varies` (another read of one path under
more than one query) and `rendered-oversize` (a document value whose JSON
renders to more than 256 KiB). A document holds the value of its one `path`, so for such
a template it says nothing about the template's other paths. A body is usable
when its content type is
JSON-like, it is UTF-8 text of at most 256 KiB, every number survives native
conversion exactly, and no member is named `__proto__`.
@@ -610,7 +626,9 @@ that the state establishes no authorization model, no way the application
computes responses or derived values, and no effects of writes. The state
accumulator charges the manifest's admission and grouping, every record line,
each body as temporary bytes before it is decoded and parsed, and every value
it keeps, within the same 96 MiB budget. The view performs no network activity
it keeps, within the same 96 MiB budget. Documents hold whole recorded response
values whose recorded text and rendered JSON are each at most 256 KiB, so a
capture with many read-only paths yields a correspondingly larger state result. The view performs no network activity
and serves nothing; `rea environment-serve` derives it from the capture at
startup, serves read tools from it and loads replacement states through the
AgentEnv data plane, as
@@ -619,13 +637,14 @@ AgentEnv data plane, as
## Resource and lifecycle boundaries
Complete evidence has a 32 MiB input, 96 MiB decoder reply and 64-level nesting
budget. Exceeding a budget returns no partial success.
budget. Those budgets have no runtime override. `capture_timeout_ms` does not
raise them. Exceeding a budget returns no partial success.
Oversized selected captures return an input error identifying `capture_path`
and the byte budget; owned reply/output failures remain output or provider errors.
HAR, SAZ and qlog run in an owned
192 MiB Node old-generation heap with one V8 worker. Mitmproxy decoding applies
Linux limits of 768 MiB address space, 30 seconds CPU and 96 MiB file output;
both commands have a 30-second wall deadline and independently supervised
both commands use `capture_timeout_ms` (default 30000) and independently supervised
process cleanup. Owned decoder diagnostics retain at most 1 MiB across stdout
and stderr; the collector drops overflow as it arrives, reports the limit
failure and verifies cleanup. Heap limits alone do not establish aggregate RSS limits.
+1 -1
View File
@@ -52,7 +52,7 @@ The result contains directly registered listener types/flags, callback coordinat
## Bounds and verification
Each protocol message has a 64 MiB budget; script/context metadata and request metadata each have an 8 MiB retained budget. Complete retained source text has a 32 MiB budget. Oversized or malformed evidence returns no partial successful result. Each command and the complete inspection/source-join phase have a 20-second deadline; cleanup has a separate 5-second command deadline plus transport closure. The selected observation duration must fit the host timer's positive 32-bit millisecond range. REA does not control the externally owned browser's CPU or memory.
Each protocol message has a 64 MiB budget; script/context metadata and request metadata each have an 8 MiB retained budget. Complete retained source text has a 32 MiB budget. Oversized or malformed evidence returns no partial successful result. Each command and the complete inspection/source-join phase use `command_timeout_ms` (default 20000). Cleanup uses a separate 5000 ms grace; `command_timeout_ms` does not raise or shorten it, and this release has no runtime override for that grace. Transport closure follows the grace. The selected observation duration must fit the host timer's positive 32-bit millisecond range. REA does not control the externally owned browser's CPU or memory.
`npm run verify:browser:runtime` uses caller-supplied `REA_BROWSER_EXECUTABLE` and the native `ps` command on Linux/macOS against an owned synthetic fixture, through real public CLI and stdio MCP. It checks exact Unicode/CRLF text/digests, same-URL identities, same-document navigation, inert declared names, executed and zero-count branches, function-only unknowns on repeated observation, listener selection, initiator association, actual arming and preservation of the external page. A separate fixture page closes during an armed MCP window to verify prompt termination and inline cleanup diagnostics. The optional positional entrypoint to `scripts/verify-browser-runtime.mjs` permits the same checks against an isolated installed tarball.
+3 -2
View File
@@ -71,7 +71,7 @@ implement every current source-map extension.
One owned Node subprocess decodes the map. Its private request directory is
removed before success is returned. Only this child's inherited `NODE_OPTIONS`
is removed; explicit V8 flags set a 192 MiB old-generation and 8 MiB semi-space
budget, one V8 worker and a 20-second deadline. Reported actual V8 heap limits
budget, one V8 worker and `decode_timeout_ms` (default 20000). The codec heap, map budget, and reply budget have no runtime override. Reported actual V8 heap limits
are checked. These are heap limits, not an aggregate process RSS limit. When
REA runs on Bun, the subprocess is Bun, which ignores these V8 flags: `runtime`
records `id: "bun"`, the Bun version, its Node.js compatibility version and a
@@ -86,7 +86,8 @@ cleanup overrides success with `cleanup_incomplete` and the owning
operation/resources.
Complete evidence is bounded by a 4 MiB map, 32 MiB codec reply, 262144 decoded
rows/segments and 64 nested section levels. Exceeding a budget returns an explicit
rows/segments and 64 nested section levels. Those structural budgets and the
192 MiB codec heap have no runtime override. Exceeding a budget returns an explicit
failure, without partial results. The existing 32 MiB manifest and 16 MiB selected
script boundaries also apply. No global toolchain or browser configuration changes
are made.
@@ -0,0 +1,84 @@
"""Summarize completed CLI trials; token counts are estimates, not model usage."""
import argparse
import hashlib
import json
import statistics
from pathlib import Path
import tiktoken
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
root = args.root
inputs = json.loads((root / 'measurement-inputs.json').read_text())
expected = inputs['expected_runtime_bindings']
assert expected['complete']['git_revision'] == inputs['baseline_revision']
assert expected['selected']['git_revision'] == inputs['feature_parent_revision']
assert expected['complete']['root'] != expected['selected']['root']
assert expected['complete']['compiled_cli_sha256'] != expected['selected']['compiled_cli_sha256']
assert expected['complete']['source_diff_sha256'] == hashlib.sha256(b'').hexdigest()
assert expected['complete']['untracked_source_sha256'] == hashlib.sha256(b'[]').hexdigest()
enc = tiktoken.get_encoding('cl100k_base')
count = lambda text: len(enc.encode(text, disallowed_special=()))
compact = lambda obj: json.dumps(obj, ensure_ascii=False, separators=(',', ':'))
runs = []
for case in sorted(root.glob('pair-*')):
if not case.is_dir():
continue
report = json.loads((case / 'report.json').read_text())
assert report['status'] == 'passed' and report['cleanup_verified']
assert report['runtime_binding_verified']
assert report['runtime_binding'] == expected[report['mode']], case.name
assert str(Path(report['runtime']).resolve()) == report['runtime_binding']['root']
receipt = next(json.loads(line) for line in reversed(case.with_name(case.name + '.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
obs = receipt['payload_resource_observation']
assert receipt['abort'] is None and receipt['payload_resource_observation_verified']
assert obs['payload_exit_code'] == 0 and obs['payload_cleanup_verified']
assert obs['resource_events']['memory.events']['oom_kill'] == 0
tokens = {'question_and_recipe': count(report['question'] + '\n' + report['recipe']),
'arguments': sum(count(compact(call['arguments'])) for call in report['calls']),
'help_and_replies': sum(count((case / call['stdout_file']).read_text()) for call in report['calls'])}
tokens['whole_task'] = sum(tokens.values())
runs.append({'case': case.name, 'mode': report['mode'], 'cache': report['cache'], 'runtime': report['runtime'], 'runtime_binding': report['runtime_binding'], 'runtime_binding_audit_ms': report['runtime_binding_audit_ms'],
'wall_seconds': receipt['elapsed_seconds'], 'cpu_seconds': obs['cpu_stat']['usage_usec'] / 1e6,
'peak_process_family_bytes_including_page_cache': obs['memory_peak_bytes'],
'tokens_estimated': tokens, 'calls': report['calls'],
'parent_evidence_id': report['parent_evidence_id'], 'parent_semantic_sha256': report['parent_semantic_sha256'],
'answer_sha256': report['answer_sha256'], 'oracle_verified': report['oracle_verified'],
'cleanup_verified': True, 'resource_events': obs['resource_events']})
assert len(runs) == 12
for key in ['answer_sha256', 'parent_semantic_sha256', 'parent_evidence_id']:
assert len({row[key] for row in runs}) == 1, key
summary = {}
for cache in ['cold', 'warm']:
summary[cache] = {}
for mode in ['complete', 'selected']:
rows = [r for r in runs if r['mode'] == mode and r['cache'] == cache]
assert len(rows) == 3
summary[cache][mode] = {key: {'median': statistics.median(r[key] for r in rows), 'min': min(r[key] for r in rows), 'max': max(r[key] for r in rows)} for key in ['wall_seconds', 'cpu_seconds', 'peak_process_family_bytes_including_page_cache']}
summary[cache][mode]['whole_task_tokens_estimated'] = [r['tokens_estimated']['whole_task'] for r in rows]
result = {'status': 'passed', 'tokenizer': 'tiktoken==0.12.0/cl100k_base; estimates, not model usage',
'context_scope': 'Question, supplied command recipe, actual selected-command help, all logical CLI argv and all stdout. No model invoked; reasoning, framing, irrelevant tool discovery and actual native-agent usage unmeasured.',
'cache_scope': 'cold disables Node compile cache; warm primes the identical workflow once per runtime/mode before timing. Fresh processes and isolated HOME each trial; OS page cache uncontrolled.',
'resource_scope': 'Whole task driver plus all CLI children; includes pre/post source-and-compiled-runtime binding verification, bundle publication/authentication, help, extra calls and local answer/oracle checking. Kernel cgroup peak includes file cache; not sampled RSS. Download/build/oracle preparation and priming excluded.',
'acquisition': json.loads((root / 'acquisition.json').read_text()), 'measurement_inputs': inputs,
'summary': summary, 'runs': runs}
native = root / 'native-control'
if native.exists():
binding = json.loads((native / 'runtime-binding.json').read_text())
assert binding['runtime_binding_verified'] and binding['runtime_binding'] == expected['complete']
assert str(Path(binding['runtime']).resolve()) == binding['runtime_binding']['root']
receipt = next(json.loads(line) for line in reversed((root / 'native-control.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
obs = receipt['payload_resource_observation']
assert receipt['abort'] is None and receipt['payload_resource_observation_verified'] and obs['payload_exit_code'] == 0 and obs['payload_cleanup_verified']
baseline_case = root / next(run['case'] for run in runs if run['mode'] == 'complete')
assert json.loads((native / 'answer.json').read_text()) == json.loads((baseline_case / 'answer.json').read_text())
control_source = Path(__file__).with_name('managed-cli-local-control.py').read_text()
result['existing_native_control'] = {'runtime': binding['runtime'], 'runtime_binding': binding['runtime_binding'], 'runtime_binding_audit_ms': binding['runtime_binding_audit_ms'], 'repetitions': 1,
'tokens_estimated_script_plus_help_plus_answer': count(control_source) + count((native / 'help.txt').read_text()) + count((native / 'answer.json').read_text()),
'token_scope': 'Projection source, original CLI help and returned answer; excludes shared question and launcher argv. No actual model usage measured.',
'wall_seconds': receipt['elapsed_seconds'], 'cpu_seconds': obs['cpu_stat']['usage_usec'] / 1e6,
'peak_process_family_bytes_including_page_cache': obs['memory_peak_bytes'], 'answer_matches': True, 'cleanup_verified': True}
args.output.write_text(json.dumps(result, indent=2) + '\n')
print(json.dumps(summary, indent=2))
@@ -0,0 +1,44 @@
"""Existing native code-composition control: save complete stdout, return selected facts."""
import argparse
import json
import subprocess
import time
from pathlib import Path
parser = argparse.ArgumentParser()
parser.add_argument('runtime', type=Path)
parser.add_argument('target', type=Path)
parser.add_argument('output', type=Path)
parser.add_argument('--measurement-inputs', type=Path, required=True)
args = parser.parse_args()
args.output.mkdir()
node = 'node'
inputs = json.loads(args.measurement_inputs.read_text())
expected = inputs['expected_runtime_bindings']['complete']
assert expected['git_revision'] == inputs['baseline_revision']
helper = Path(__file__).with_name('managed-cli-runtime.mjs')
def capture_binding():
return json.loads(subprocess.check_output([node, str(helper), str(args.runtime)], text=True))
binding_started = time.monotonic()
binding = capture_binding()
binding_audit_ms = {'before': (time.monotonic() - binding_started) * 1000}
assert binding == expected
entry = str(args.runtime / 'scripts/rea.mjs')
help_result = subprocess.run([node, entry, 'inspect-managed-members', '--help'], check=True, capture_output=True, text=True)
(args.output / 'help.txt').write_text(help_result.stdout)
with (args.output / 'parent.json').open('w') as file:
subprocess.run([node, entry, 'inspect-managed-members', str(args.target), '--type', 'Newtonsoft.Json.JsonConvert', '--json'], check=True, stdout=file)
parent = json.loads((args.output / 'parent.json').read_text())
data = parent['normalized_result']
answer = {'parent_evidence_id': parent['evidence_id'], 'artifact_sha256': data['artifact']['sha256'],
'selection': data['selection'], 'source_coverage': data['coverage'],
'total_methods': len(data['methods']), 'total_member_refs': len(data['member_refs']),
'methods': data['methods'][:8], 'member_refs': data['member_refs'][:16],
'limitation': 'MemberRef rows are assembly-wide; methods are selected-type rows. Page exhaustion is not an absence claim.'}
(args.output / 'answer.json').write_text(json.dumps(answer))
binding_finished = time.monotonic()
assert capture_binding() == binding
binding_audit_ms['after'] = (time.monotonic() - binding_finished) * 1000
(args.output / 'runtime-binding.json').write_text(json.dumps({'runtime': str(args.runtime), 'runtime_binding': binding, 'runtime_binding_verified': True, 'runtime_binding_audit_ms': binding_audit_ms}))
print(json.dumps(answer))
@@ -0,0 +1,18 @@
import json,hashlib,argparse
from pathlib import Path
import dnfile
parser=argparse.ArgumentParser();parser.add_argument('root',type=Path);root=parser.parse_args().root;path=root/'Newtonsoft.Json.dll'
pe=dnfile.dnPE(str(path));assert pe.net is not None
counts={name:len(getattr(pe.net.mdtables,name).rows) if getattr(pe.net.mdtables,name,None) else 0 for name in ['TypeDef','MethodDef','MemberRef','Field','Property','Event','Param','StandAloneSig','TypeSpec','ExportedType']}
selected=[]
for index,row in enumerate(pe.net.mdtables.TypeDef.rows,1):
name=(str(row.TypeNamespace)+'.' if str(row.TypeNamespace) else '')+str(row.TypeName)
if name!='Newtonsoft.Json.JsonConvert':continue
selected.append({'token':f'0x{0x02000000|index:08x}','name':name,'methods':[{'token':f'0x{0x06000000|m.row_index:08x}','name':str(m.row.Name),'rva':m.row.Rva,'signature_hex':m.row.Signature.value.hex()} for m in row.MethodList]})
assert len(selected)==1
refs=[{'token':f'0x{0x0a000000|i:08x}','name':str(row.Name),'signature_hex':row.Signature.value.hex()} for i,row in enumerate(pe.net.mdtables.MemberRef.rows[:16],1)]
assembly=pe.net.mdtables.Assembly.rows[0]
oracle={'decoder':'dnfile==0.18.0 (independent Python PE/CLI parser)','assembly_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'assembly_name':str(assembly.Name),'assembly_version':'.'.join(str(getattr(assembly,k)) for k in ['MajorVersion','MinorVersion','BuildNumber','RevisionNumber']),'row_counts':counts,'selected_types':selected,'member_refs_first_16':refs,'target_execution':False}
(root/'oracle.json').write_text(json.dumps(oracle,indent=2)+'\n')
print(json.dumps({'assembly':oracle['assembly_name'],'version':oracle['assembly_version'],'row_counts':counts,'selected_type':selected[0]['name'],'selected_methods':len(selected[0]['methods']),'target_execution':False}))
pe.close()
@@ -0,0 +1,87 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile, readdir, realpath } from "node:fs/promises";
import { join, relative } from "node:path";
import { fileURLToPath } from "node:url";
const sha = (value) => createHash("sha256").update(value).digest("hex");
/** Bind a trial to its source checkout and the compiled CLI actually launched. */
export const captureRuntimeBinding = async (runtime) => {
const root = await realpath(runtime);
const git = (args) =>
execFileSync("git", args, { cwd: root, maxBuffer: 16 * 1024 * 1024 });
const gitRevision = git(["rev-parse", "HEAD"]).toString().trim();
const sourcePaths = ["src", "scripts", "package.json", "package-lock.json"];
const sourceDiff = git([
"diff",
"--no-ext-diff",
"--no-textconv",
"--binary",
"HEAD",
"--",
...sourcePaths,
]);
const untracked = git([
"ls-files",
"--others",
"--exclude-standard",
"-z",
"--",
...sourcePaths,
])
.toString()
.split("\0")
.filter(Boolean)
.sort();
const untrackedEntries = [];
for (const name of untracked)
untrackedEntries.push([name, sha(await readFile(join(root, name)))]);
const files = ["scripts/rea.mjs", "package.json", "package-lock.json"];
const visit = async (directory) => {
const entries = await readdir(directory, { withFileTypes: true });
for (const entry of entries.sort((a, b) =>
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
)) {
const path = join(directory, entry.name);
assert.ok(!entry.isSymbolicLink(), `Unbound compiled CLI link: ${path}`);
if (entry.isDirectory()) await visit(path);
else if (entry.isFile() && /\.(?:[cm]?js|json)$/.test(entry.name))
files.push(relative(root, path));
}
};
await visit(join(root, "dist/cli-runtime"));
assert.ok(
files.includes(join("dist", "cli-runtime", "cli.js")),
"Missing compiled CLI entry",
);
const hash = createHash("sha256");
let bytes = 0;
for (const name of files.sort()) {
const value = await readFile(join(root, name));
bytes += value.length;
hash.update(JSON.stringify([name, value.length, sha(value)]) + "\n");
}
return {
binding_version: 1,
root,
node: process.version,
git_revision: gitRevision,
source_diff_sha256: sha(sourceDiff),
untracked_source_sha256: sha(JSON.stringify(untrackedEntries)),
compiled_cli_sha256: hash.digest("hex"),
compiled_cli_files: files.length,
compiled_cli_bytes: bytes,
};
};
export const assertRuntimeBinding = (actual, expected, mode) => {
assert.ok(expected, `Missing expected runtime binding for ${mode}`);
assert.deepEqual(actual, expected, `Runtime binding differs for ${mode}`);
};
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
console.log(
JSON.stringify(await captureRuntimeBinding(process.argv[2]), null, 2),
);
}
@@ -0,0 +1,72 @@
{
"status": "passed",
"scope": "Single completed SDK regression after shared JSON reader/admission changes; not a new paired source-to-bundle performance comparison. Existing semantic-node cap one answers static correspondence only; no equal semantic-graph coverage claim. Corpus/reference preparation performed earlier and excluded.",
"artifact": {
"name": "Redux Toolkit 2.2.7",
"tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9"
},
"runtime_parent_revision": "4fab2dfda16236dc407aa179920780d6a970dda3",
"production_manifest": "managed-cli.json#measurement_inputs.production_sources_sha256",
"input_schema_profile": "default compact",
"output_schema_profile": "full",
"tokens_estimated": {
"input_catalog_once": 49054,
"arguments": 307,
"text_replies": 7017,
"whole_task": 56378
},
"token_scope": "tiktoken==0.12.0/cl100k_base; one name/description/input-schema catalog plus actual arguments and MCP text replies; excludes output schemas, framing, model prompts/reasoning and actual model usage. Not actual native-agent token usage.",
"actual_model_usage": null,
"wall_seconds": 21.38,
"cpu_seconds": 21.022683,
"peak_process_family_bytes_including_page_cache": 974041088,
"calls": [
{
"name": "analyze_javascript_application",
"elapsed_ms": 9837.265845000002,
"is_error": false,
"reply_bytes": 25941
},
{
"name": "compare_source_to_bundle",
"elapsed_ms": 256.3669919999993,
"is_error": false,
"reply_bytes": 13199
},
{
"name": "inspect_analysis_view",
"elapsed_ms": 3012.057904000001,
"is_error": false,
"reply_bytes": 19393
},
{
"name": "export_evidence_bundle",
"elapsed_ms": 148.9011160000009,
"is_error": false,
"reply_bytes": 364
}
],
"independent_proof": {
"status": "passed",
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
"source_files": 180,
"embedded_nodes": 99,
"candidate_relations": 210,
"strong_location_rows": 33,
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
},
"resource_events": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"cleanup_verified": true
}
@@ -0,0 +1,266 @@
import {
captureRuntimeBinding,
assertRuntimeBinding,
} from "./managed-cli-runtime.mjs";
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { createHash } from "node:crypto";
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { join, resolve } from "node:path";
const execute = promisify(execFile);
const [runtime, root, output, mode, cache = "cold"] = process.argv.slice(2);
for (const value of [runtime, root, output])
assert.equal(resolve(value), value);
assert.ok(["complete", "selected"].includes(mode));
assert.ok(["cold", "warm"].includes(cache));
await mkdir(output);
await mkdir(join(output, "tmp"));
const oracle = JSON.parse(await readFile(join(root, "oracle.json"), "utf8"));
const target = join(root, "Newtonsoft.Json.dll");
const hash = (value) => createHash("sha256").update(value).digest("hex");
assert.equal(hash(await readFile(target)), oracle.assembly_sha256);
const stable = (value) =>
Array.isArray(value)
? value.map(stable)
: value !== null && typeof value === "object"
? Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, stable(value[key])]),
)
: value;
const report = {
status: "incomplete",
runtime,
mode,
cache,
node: process.version,
question:
"Report identity/selection/source coverage for JsonConvert, its total method count and the first eight complete method records, plus the first sixteen assembly-wide MemberRef records and continuation coverage.",
recipe:
mode === "selected"
? "Use --detail summary --evidence-bundle FILE; pass the exact parent_evidence_id to inspect-analysis-view --evidence-bundle FILE with source kind retained-evidence and view {kind:managed-members,collection:methods|member_refs,offset:0,limit:8|16}."
: "Use inspect-managed-members --type Newtonsoft.Json.JsonConvert; the completed full Evidence supplies the requested rows and remains accessible.",
calls: [],
cleanup_verified: false,
};
const began = performance.now();
const environment = {
PATH: process.env.PATH,
HOME: output,
TMPDIR: join(output, "tmp"),
LANG: "C.UTF-8",
REA_LOG_LEVEL: "silent",
UV_THREADPOOL_SIZE: "1",
NODE_OPTIONS: "--max-old-space-size=2048",
...(cache === "cold"
? { NODE_DISABLE_COMPILE_CACHE: "1" }
: { NODE_COMPILE_CACHE: join(root, `compile-cache-${mode}`) }),
};
const call = async (label, args, json = true) => {
const started = performance.now();
const result = await execute(
process.execPath,
[join(runtime, "scripts/rea.mjs"), ...args],
{
cwd: output,
env: environment,
maxBuffer: 16 * 1024 * 1024,
timeout: 120000,
},
);
await writeFile(join(output, label + ".stdout"), result.stdout);
await writeFile(join(output, label + ".stderr"), result.stderr);
report.calls.push({
label,
arguments: ["rea", ...args],
elapsed_ms: performance.now() - started,
stdout_bytes: Buffer.byteLength(result.stdout),
stdout_file: label + ".stdout",
});
return json ? JSON.parse(result.stdout) : result.stdout;
};
let parent, summary, methods, refs;
try {
const inputs = JSON.parse(
await readFile(join(root, "measurement-inputs.json"), "utf8"),
);
const expected = inputs.expected_runtime_bindings?.[mode];
assert.ok(expected, `Missing expected runtime for ${mode}`);
assert.equal(
expected.git_revision,
mode === "complete"
? inputs.baseline_revision
: inputs.feature_parent_revision,
);
assert.notEqual(
inputs.expected_runtime_bindings.complete.root,
inputs.expected_runtime_bindings.selected.root,
);
assert.notEqual(
inputs.expected_runtime_bindings.complete.compiled_cli_sha256,
inputs.expected_runtime_bindings.selected.compiled_cli_sha256,
);
const bindingStarted = performance.now();
report.runtime_binding = await captureRuntimeBinding(runtime);
report.runtime_binding_audit_ms = {
before: performance.now() - bindingStarted,
};
assertRuntimeBinding(report.runtime_binding, expected, mode);
await call("help-members", ["inspect-managed-members", "--help"], false);
if (mode === "complete") {
parent = await call("complete", [
"inspect-managed-members",
target,
"--type",
"Newtonsoft.Json.JsonConvert",
"--json",
]);
methods = parent.normalized_result.methods.slice(0, 8);
refs = parent.normalized_result.member_refs.slice(0, 16);
} else {
await call("help-view", ["inspect-analysis-view", "--help"], false);
const bundlePath = join(output, "parent.json");
summary = await call("summary", [
"inspect-managed-members",
target,
"--type",
"Newtonsoft.Json.JsonConvert",
"--detail",
"summary",
"--evidence-bundle",
bundlePath,
"--json",
]);
const id = summary.normalized_result.parent_evidence_id;
assert.ok(id);
const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
assert.equal(bundle.records.length, 1);
parent = bundle.records[0];
assert.equal(parent.evidence_id, id);
for (const [collection, limit] of [
["methods", 8],
["member_refs", 16],
]) {
const value = await call(collection, [
"inspect-analysis-view",
JSON.stringify({
source: { kind: "retained-evidence", evidence_id: id },
view: { kind: "managed-members", collection, offset: 0, limit },
}),
"--evidence-bundle",
bundlePath,
"--json",
]);
const page = value.normalized_result;
assert.equal(page.parent_evidence_id, id);
assert.deepEqual(page.source_coverage, parent.normalized_result.coverage);
assert.deepEqual(
page.page.rows,
parent.normalized_result[collection].slice(0, limit),
);
assert.equal(
page.coverage.total,
parent.normalized_result[collection].length,
);
assert.equal(page.coverage.next_offset, limit);
assert.equal(page.coverage.exhausted, false);
if (collection === "methods") methods = page.page.rows;
else refs = page.page.rows;
}
}
const data = parent.normalized_result;
assert.equal(data.artifact.sha256, oracle.assembly_sha256);
assert.equal(data.methods.length, 72);
assert.equal(data.member_refs.length, 2149);
assert.equal(data.types.length, 1);
assert.equal(data.types[0].token, oracle.selected_types[0].token);
for (let i = 0; i < data.methods.length; i++) {
const row = data.methods[i],
wanted = oracle.selected_types[0].methods[i];
assert.equal(row.token, wanted.token);
assert.equal(row.name, wanted.name);
assert.equal(row.rva, wanted.rva);
assert.equal(
row.signature.raw_sha256,
hash(Buffer.from(wanted.signature_hex, "hex")),
);
}
for (let i = 0; i < refs.length; i++) {
const row = refs[i],
wanted = oracle.member_refs_first_16[i];
assert.equal(row.token, wanted.token);
assert.equal(row.name, wanted.name);
assert.equal(
row.signature.raw_sha256,
hash(Buffer.from(wanted.signature_hex, "hex")),
);
}
assert.equal(methods.length, 8);
assert.equal(refs.length, 16);
if (summary) {
assert.equal(summary.normalized_result.summary.reported_counts.methods, 72);
assert.equal(
summary.normalized_result.summary.reported_counts.member_refs,
2149,
);
assert.deepEqual(summary.normalized_result.summary.coverage, data.coverage);
assert.deepEqual(
summary.normalized_result.summary.selection,
data.selection,
);
}
const answer = {
parent_evidence_id: parent.evidence_id,
artifact_sha256: oracle.assembly_sha256,
selection: data.selection,
source_coverage: data.coverage,
total_methods: data.methods.length,
total_member_refs: data.member_refs.length,
methods,
member_refs: refs,
limitation:
"MemberRef rows are assembly-wide; methods are selected-type rows. Page exhaustion is not an absence claim.",
};
report.parent_evidence_id = parent.evidence_id;
report.parent_semantic_sha256 = hash(JSON.stringify(stable(parent)));
report.answer_sha256 = hash(JSON.stringify(stable(answer)));
report.oracle_verified = {
all_selected_method_metadata: 72,
returned_methods: 8,
returned_member_refs: 16,
};
const bindingFinished = performance.now();
assertRuntimeBinding(
await captureRuntimeBinding(runtime),
report.runtime_binding,
mode,
);
report.runtime_binding_audit_ms.after = performance.now() - bindingFinished;
report.runtime_binding_verified = true;
report.status = "passed";
await writeFile(join(output, "parent-evidence.json"), JSON.stringify(parent));
await writeFile(join(output, "answer.json"), JSON.stringify(answer));
} finally {
report.elapsed_ms = performance.now() - began;
report.cleanup_verified = true;
await writeFile(
join(output, "report.json"),
JSON.stringify(report, null, 2) + "\n",
);
}
console.log(
JSON.stringify({
status: report.status,
mode,
cache,
parent: report.parent_evidence_id,
answer: report.answer_sha256,
calls: report.calls.map((c) => ({
label: c.label,
elapsed_ms: c.elapsed_ms,
bytes: c.stdout_bytes,
})),
}),
);
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,219 @@
# Portable managed CLI member views
Issue #5148 / PR #5167. Baseline `main@3dbe314ada9f6dcc994ca4655dc0faf082503847`;
feature integrates the original `7b3ed4d025edd24d062cde0327a53bd311e174fc`.
The measured production-file SHA-256 manifest is in [managed-cli.json](managed-cli.json).
These are working-tree measurements. Each trial verifies both source and the
complete compiled CLI graph before and after execution against its explicitly
selected build. The two roles have distinct checkout roots and compiled digests.
Identity-audit time is included in whole-task costs and recorded separately.
The audit reads compiled files before timing individual CLI calls, warming those
filesystem pages; cold means disabled Node compile cache, not disk-cold startup.
## Question and independent checks
Report identity, selected type, source coverage, the total method count, the first
eight complete method records, and the first sixteen assembly-wide MemberRef
records, including continuation coverage. Keep complete original Evidence available.
The question deliberately does not ask for every method or reference.
Public artifact: [Newtonsoft.Json 13.0.3](https://www.nuget.org/packages/Newtonsoft.Json/13.0.3),
from the NuGet flat-container archive. The 2,441,966-byte archive SHA-256 is
`872fc189e638ab1056555b03aaa38f68bcb54286e221aa646eb1129babf63c77`.
Selected member `lib/net6.0/Newtonsoft.Json.dll`, 712,464 bytes, SHA-256
`22c649f75fce5be7c7ccda8880473b634ef69ecf33f5d1ab8ad892caf47d5a07`.
Package version 13.0.3 is distinct from metadata assembly version 13.0.0.0.
No library code is executed; REA parses PE/CLI statically.
An independent Python oracle uses `dnfile==0.18.0` and `pefile==2024.8.26`.
It finds 494 TypeDefs, 4,208 MethodDefs and 2,149 MemberRefs. The selected
`Newtonsoft.Json.JsonConvert` has 72 methods. Every trial checks all 72 method
tokens, names, RVAs and original signature hashes, plus the sixteen returned
MemberRef tokens, names and signature hashes, against this oracle. This does
not independently establish complete CIL semantics or opcode correctness.
The full parent Evidence and requested answers are identical across all twelve
trials; returned rows equal the exact parent slices. Selection and source
coverage remain distinct from page exhaustion. MemberRefs remain assembly-wide.
## Completed-task cost
Linux x64, Node 24.18.0/npm 11.16.0. Three alternating pairs per cache condition,
serial, one CPU; 4 GiB hard / 3 GiB early-stop family guard, no swap, 192-task
ceiling. Every task includes actual selected-command help, all CLI calls,
provider parsing, bundle publication/authentication, local answer extraction
and oracle comparisons, plus both runtime-identity audits. Build, download, oracle preparation and warm priming
are excluded. OS page cache is uncontrolled. No build/test overlapped timing.
Complete mode starts two CLI processes (help plus analysis). Selected mode starts
five (two help commands, summary/publication, methods page, MemberRef page).
Cold disables the Node compile cache; warm primes the exact workflow once per
runtime/mode, then still starts fresh processes with isolated HOME each trial.
All trials complete with verified process cleanup and no OOM/resource-stop event.
| Whole task, median | Complete | Summary + two pages |
| ----------------------------------------------- | -------: | ------------------: |
| Estimated context tokens, both cache conditions | 451,373 | 13,078 |
| Cold wall seconds | 9.969 | 14.801 |
| Cold family CPU seconds | 9.743 | 14.555 |
| Cold kernel family peak MiB | 393.281 | 400.098 |
| Warm wall seconds | 10.339 | 24.141 |
| Warm family CPU seconds | 10.097 | 23.842 |
| Warm kernel family peak MiB | 401.809 | 407.164 |
Context estimates include the question, supplied command recipe, all logical
argv, actual help and stdout. Tokenizer `tiktoken==0.12.0`, `cl100k_base`.
The reduction is 97.1% for this fixed task. There is no model invocation;
actual billed/native-agent usage, reasoning and transport framing are unmeasured.
The memory counter is the kernel cgroup process-family peak including file
cache, not sampled RSS. Ranges and individual receipts are in the JSON.
This is an opt-in context/portability improvement with extra latency and CPU,
not a speed or memory optimization. Complete provider parsing and full authentication of file-supplied bundles
remain. Fresh managed Evidence is sealed at its producing factory and reused
without another payload clone, including low-headroom error projection. Complete output remains the default and is preferable
for small outputs or questions requiring every row. No cache or parser cap
was introduced.
## Existing native code composition
The unchanged baseline can already redirect full stdout to a file and select
rows in code. [The measured control](managed-cli-local-control.py) returns the
identical answer and retains the complete original Evidence. One cold run:
11.249 s wall, 11.000 s family CPU,
375.3 MiB kernel peak. Projection source,
original help and returned answer together estimate 10,555 tokens; the shared
question and launcher argv are excluded. This single-run control does not
establish stable performance or actual native-agent usage. It does demonstrate
that raw CLI reply savings are not universal savings over code composition.
The additive interface supplies authenticated reusable projections through the
same application/domain owner as MCP, without caller-maintained JSON slicing.
## Correctness and resource boundaries
The public CLI regression pages all eight managed collections after process
exit, target deletion and a working-directory change. Complete parent identity,
original row order, signatures, source/page coverage and unknowns survive.
Existing outputs are not replaced; failed publication retains completed facts.
The Evidence factory parses mutable caller data into independent trees, computes
identity, then seals its fresh owned snapshot with active ancestry/iterators.
It does not retain a set/array of every completed object. Existing sealed roots
can be shared; mutable input aliases are cloned independently. General graph
sealing keeps its separate cycle, sharing, hidden-field and atomic-failure rules.
Managed and derived-view factories seal their own completed Evidence, so CLI
error projection can borrow it without reauthentication or another payload copy.
The view owner retains strict JSON normalization for optional-field contracts;
the CLI no longer copies its already validated completed projection again.
Shared heap admission traverses by depth rather than queueing every report row
or allocating an entries array. It reserves parsed working copies and actual
escaping expansion before cloning/hashing, reporting UTF-16 storage explicitly.
Canonical bundle equality streams existing ordinary JSON encoder parts, preserving
native key order, omission, negative zero and Unicode semantics without two
whole-document strings. All production bundle readers explicitly pass their
operation: CLI import/export/compare/inspection, retained views, MCP import/
comparison and both native-matching read passes. Errors retain the original
cause, cleanup primary and lease through retry.
One process primitive dispatches queued callbacks and then their OS signal
notifications. The managed producer checks it before accepting complete Evidence.
Publication/read stages use it before their final checks. Inline, file and bundle
CLI views share the existing command finalizer and actual resource scope; the
finalizer checks interruption after close while preserving any primary/cleanup
failure. Existing inline error labels remain intact. Synchronous analysis is
cooperatively cancellable between stages, not preemptible.
Four compiled publication cases (SIGINT/SIGTERM × event/actual POSIX signal)
fail without their checkpoint and pass restored. Eight additional compiled
completion cases cover default complete delivery and inline projection; all fail
with only their new completion checkpoints removed and pass restored. Fixture
hooks match the selected target path or exact parent ID. Two real Newtonsoft
publication runs and four real default/inline completion runs retain the identical
parent or independently checked selected summary, exit 130/143 and preserve
committed receipts where applicable. No target library code is executed.
All 556 focused tests across 36 files pass, including direct
finalizer/cleanup ownership, WASM/Android cancellation and native matching.
All six `npm run check` tasks plus architecture guards, and all six
`npm run docs:check` tasks plus five README tests pass. The expanded domain
project passes 5,617 tests in 287 files with
3 skipped cases; this overlaps the focused gate. Its first
batched shard hit two existing 30-second timeouts in unchanged large synthetic
tests. Both passed isolated and in the fresh shard rerun under the same timeout;
the cause of the batch variation is unverified. No production/timeout change
was made for those cases. An earlier unsplit expansion was manually stopped
before its heap ceiling; that incomplete attempt is not counted as a pass.
Eight producing/operation negative controls also catch graph-sized sealing/
admission scratch and wrong operation attribution, including native reload.
Native built-in instrumentation is confined to the isolated process-global lane.
Initial test-only spy/probe/schema-import issues were corrected before these
passing gates. No actual heap exhaustion is induced. Simulated providers do not
qualify foreign engines; Windows OS signals remain unverified locally.
## Shared-reader real-artifact regression
The existing Redux Toolkit 2.2.7 source/bundle task also completes through the
updated reader with default full output schemas and compact input schemas.
An independent Python SHA-256/path oracle verifies 180 historical files,
99 embedded source nodes and 210 candidate relations (33 strong-location rows,
147 unknown). Original signals and partial source coverage remain visible.
This uses the existing explicit semantic-node cap of one for static correspondence;
it does not establish equivalent semantic-graph coverage or behavioral equivalence.
One completed task: 21.380 s wall,
21.023 s family CPU,
928.9 MiB kernel family peak,
56,378 estimated context tokens (one input catalog,
all arguments and MCP text replies; excludes output schemas, framing and model
reasoning). Corpus/reference preparation was performed earlier and excluded.
This is a regression verification with resource observation, not a new paired
performance comparison. Actual model usage remains unmeasured. Details:
[managed-cli-shared-read.json](managed-cli-shared-read.json).
## Reproduction
Extract the selected DLL to a private lab as `Newtonsoft.Json.dll`. Create an
`acquisition.json` containing the URL, hashes, sizes and selected ZIP member
shown above; never commit the DLL. Prepare the independent oracle:
```bash
uv run --no-project --with dnfile==0.18.0 --with pefile==2024.8.26 python \
experiments/analysis-efficiency/managed-cli-oracle.py /absolute/lab
```
Build baseline and feature separately with the pinned toolchain. Run each under
the shared resource guard, alternating complete/selected, selected/complete,
complete/selected for cold and warm. Use a new output directory for every run:
```bash
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/analysis-efficiency/managed-cli-task.mjs \
/absolute/runtime /absolute/lab /absolute/lab/pair-cold-1-selected selected cold
```
Save each guard output to the adjacent `pair-*.log`. Warm mode uses the lab's
mode-specific compile-cache directory; prime each identical workflow once first.
Supply a `measurement-inputs.json` with exact revisions and production hashes,
plus `expected_runtime_bindings.complete` and `.selected`. Capture each with
`node experiments/analysis-efficiency/managed-cli-runtime.mjs /absolute/runtime`
after building. Bind complete to the clean baseline revision, selected to the
feature parent revision plus tracked/untracked source digests, with different
roots and compiled-graph digests. The collector rejects a wrong build before any
CLI call, rechecks it afterward, and aggregation independently enforces identity.
A real wrong-runtime attempt is rejected before analysis. A copied trial with a
tampered revision fails aggregation; removing only that consumer guard accepts it.
The twelve-trial aggregation reproduces without private pilots. The native
control also requires `--measurement-inputs /absolute/lab/measurement-inputs.json`
and performs the same before/after identity checks.
Then aggregate with:
```bash
uv run --no-project --with tiktoken==0.12.0 python \
experiments/analysis-efficiency/managed-cli-costs.py /absolute/lab \
--output /absolute/results.json
```
The original issue reporter's executable, foreign operating systems and actual
model sessions remain unverified. Setup pilots and earlier unbound cohorts, including one failed setup run
while compiled artifacts were absent after a typecheck failure, are excluded.
+40 -14
View File
@@ -1,6 +1,31 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface CiRun {
readonly id?: string | number;
readonly run_attempt?: number;
readonly status?: string;
readonly conclusion?: string | null;
readonly event?: string;
readonly path?: string;
readonly head_sha?: string;
readonly head_branch?: string;
readonly repository?: { readonly full_name?: string };
readonly pull_requests?: readonly {
readonly base?: {
readonly sha?: string;
readonly repo?: { readonly full_name?: string };
};
}[];
}
export interface CiJob {
readonly name?: string;
readonly status?: string;
readonly conclusion?: string | null;
readonly run_id?: number;
readonly run_attempt?: number;
}
export interface StaticDiagnostic {
readonly kind: "lint" | "jscpd";
@@ -11,15 +36,20 @@ export interface StaticDiagnostic {
readonly severity?: string;
}
export interface StaticEvidence {
readonly context: StaticTaskContext;
readonly diagnostics: readonly StaticDiagnostic[];
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: Record<string, unknown> & NormalizedWorkflowRun;
job: Record<string, unknown>;
run: CiRun & NormalizedWorkflowRun;
job: CiJob;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: Record<string, unknown> & NormalizedWorkflowRun;
mainJob: Record<string, unknown>;
mainRun: CiRun & NormalizedWorkflowRun;
mainJob: CiJob;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
@@ -47,7 +77,7 @@ export function safeSha(value: unknown): string | null;
export function validRepoPath(value: unknown): value is string;
export function selectStaticCheckArtifact(
page: unknown,
run: { id?: string | number; head_sha?: string },
run: Pick<CiRun, "id" | "head_sha">,
runAttempt: number,
name?: string,
): unknown | null;
@@ -57,12 +87,8 @@ export function readStaticCheckArtifact(options: {
artifactId: number | string;
token: string;
fetchImpl?: typeof fetch;
}): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
}): Promise<StaticEvidence>;
export function parseStaticArchive(archivePath: string): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
export function parseStaticArchive(
archivePath: string,
): Promise<StaticEvidence>;
+1 -53
View File
@@ -1,38 +1,5 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticDiagnostic } from "./artifacts.mjs";
export type { StaticDiagnostic } from "./artifacts.mjs";
import type { CiRun, CiJob, StaticEvidence } from "./artifacts.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface CiRun {
readonly id?: string | number;
readonly run_attempt?: number;
readonly status?: string;
readonly conclusion?: string | null;
readonly event?: string;
readonly path?: string;
readonly head_sha?: string;
readonly head_branch?: string;
readonly repository?: { readonly full_name?: string };
readonly pull_requests?: readonly {
readonly base?: {
readonly sha?: string;
readonly repo?: { readonly full_name?: string };
};
}[];
}
export interface CiJob {
readonly name?: string;
readonly status?: string;
readonly conclusion?: string | null;
readonly run_id?: number;
readonly run_attempt?: number;
}
export interface StaticEvidence {
readonly context: StaticTaskContext;
readonly diagnostics: readonly StaticDiagnostic[];
}
export interface CiFailureReport {
readonly version: 1;
@@ -99,25 +66,6 @@ export interface FailureReportApi {
): Promise<Readonly<Record<string, boolean>>>;
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: CiRun & NormalizedWorkflowRun;
job: CiJob;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: CiRun & NormalizedWorkflowRun;
mainJob: CiJob;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
ancestry: { status: string; mergeBaseSha: string | null };
blobs: Readonly<Record<string, boolean>>;
}): {
classification: "matching-main-diagnostics" | "unknown";
reason: string;
};
export function reportCiFailures(options: {
repository: string;
defaultBranch: string;
-2
View File
@@ -26,8 +26,6 @@ const MAX_MAIN_RUN_LOOKUPS = 100;
const MAX_PATHS_PER_COMPARISON = 50;
const MAX_API_REQUESTS = 250;
export { classifyFailure } from "./artifacts.mjs";
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
export async function reportCiFailures({
repository,
@@ -142,6 +142,9 @@ export const verifyScenarioPolicySetup = async (
page,
) => {
const result = {};
// Exercise a caller-selected handshake deadline within the existing teardown
// guard; the default command deadline may legitimately outlast that guard.
const setupCommandTimeoutMs = 2_000;
for (const mode of ["http-websocket", "http-discovery", "deadline"]) {
const proxy = await startProxy(endpoint, mode);
const provider = createBrowserScenarioProvider(process.env);
@@ -152,6 +155,9 @@ export const verifyScenarioPolicySetup = async (
target_id: targetId,
},
start_url: { url: origin },
...(mode === "deadline"
? { command_timeout_ms: setupCommandTimeoutMs }
: {}),
environment: { service_workers: "block" },
actions: [
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
@@ -189,7 +195,9 @@ export const verifyScenarioPolicySetup = async (
if (!next.ok) throw next.error;
assert.equal(next.value.browser.cleanup, "disconnected-external");
result[mode] = {
...(mode === "deadline" ? { timed_out: true } : { cancelled: true }),
...(mode === "deadline"
? { timed_out: true, command_timeout_ms: setupCommandTimeoutMs }
: { cancelled: true }),
socket_closed: true,
target_reused: true,
};
+7
View File
@@ -47,6 +47,13 @@ export function profileMcpModuleLoading(
/** Parse Linux procfs VmRSS text into bytes. */
export function parseLinuxVmRssBytes(status: string): number | null;
/** Name a fixed MCP startup phase budget. Mirrors `fixedMcpPhaseBudgetRecovery`. */
export function fixedMcpStartupPhaseBudgetRecovery(
phase: string,
value: number,
unit: "ms" | "bytes",
): string;
/** Collect unique file-backed ESM module URLs from Node diagnostics. */
export function createEsmModuleProfileCollector(packageName?: string): {
append(chunk: string): void;
+71 -14
View File
@@ -35,11 +35,22 @@ export async function measureMcpStartup({
const startedAt = performance.now();
rss.start();
try {
await client.connect(transport, { timeout: policy.initializeBudgetMs });
await withFixedPhaseBudget(
"MCP initialize phase budget",
policy.initializeBudgetMs,
"ms",
() => client.connect(transport, { timeout: policy.initializeBudgetMs }),
);
const initializedAt = performance.now();
const tools = await client.listTools(undefined, {
timeout: policy.toolsListBudgetMs,
});
const tools = await withFixedPhaseBudget(
"MCP tools/list phase budget",
policy.toolsListBudgetMs,
"ms",
() =>
client.listTools(undefined, {
timeout: policy.toolsListBudgetMs,
}),
);
const listedAt = performance.now();
rss.stop();
const measurement = {
@@ -52,7 +63,11 @@ export async function measureMcpStartup({
assertStartupBudgets(measurement, policy);
return measurement;
} catch (cause) {
throw new Error(`packaged MCP startup probe failed: ${stderr}`, { cause });
const detail = cause instanceof Error ? cause.message : String(cause);
throw new Error(
`packaged MCP startup probe failed: ${detail}${stderr.length === 0 ? "" : `\n${stderr}`}`,
{ cause },
);
} finally {
rss.stop();
await Promise.allSettled([client.close(), transport.close()]);
@@ -82,12 +97,24 @@ export async function profileMcpModuleLoading({
version: "1.0.0",
});
try {
await client.connect(transport, {
timeout: policy.doctorDeadlineMs,
});
await client.listTools(undefined, {
timeout: policy.toolsListBudgetMs,
});
await withFixedPhaseBudget(
"MCP doctor startup deadline",
policy.doctorDeadlineMs,
"ms",
() =>
client.connect(transport, {
timeout: policy.doctorDeadlineMs,
}),
);
await withFixedPhaseBudget(
"MCP tools/list phase budget",
policy.toolsListBudgetMs,
"ms",
() =>
client.listTools(undefined, {
timeout: policy.toolsListBudgetMs,
}),
);
return profile.result();
} finally {
await Promise.allSettled([client.close(), transport.close()]);
@@ -197,16 +224,46 @@ const createLinuxRssSampler = (pid) => {
};
};
/** Same sentence as `fixedMcpPhaseBudgetRecovery`; this script cannot import that TypeScript module. */
export const fixedMcpStartupPhaseBudgetRecovery = (phase, value, unit) =>
`REA's fixed ${phase} is ${String(value)} ${unit} and this release has no runtime override.`;
const phaseTimeout = (cause) => {
const message = cause instanceof Error ? cause.message : String(cause);
return (
message.includes("Request timed out") ||
message.includes("Maximum total timeout exceeded") ||
message.includes("timed out")
);
};
const withFixedPhaseBudget = async (phase, value, unit, operation) => {
try {
return await operation();
} catch (cause) {
if (!phaseTimeout(cause)) throw cause;
const detail = cause instanceof Error ? cause.message : String(cause);
throw new Error(
`${detail} ${fixedMcpStartupPhaseBudgetRecovery(phase, value, unit)}`,
{ cause },
);
}
};
const assertStartupBudgets = (measurement, policy) => {
if (measurement.process_started_to_initialized_ms > policy.initializeBudgetMs)
throw new Error("packaged MCP exceeded the initialize regression budget");
throw new Error(
`packaged MCP exceeded the initialize phase budget. ${fixedMcpStartupPhaseBudgetRecovery("MCP initialize phase budget", policy.initializeBudgetMs, "ms")}`,
);
if (measurement.initialized_to_tools_list_ms > policy.toolsListBudgetMs)
throw new Error("packaged MCP exceeded the tools/list regression budget");
throw new Error(
`packaged MCP exceeded the tools/list phase budget. ${fixedMcpStartupPhaseBudgetRecovery("MCP tools/list phase budget", policy.toolsListBudgetMs, "ms")}`,
);
if (
measurement.process_started_to_tools_list_ms > policy.firstCatalogBudgetMs
)
throw new Error(
"packaged MCP exceeded the first-catalog regression budget",
`packaged MCP exceeded the first-catalog phase budget. ${fixedMcpStartupPhaseBudgetRecovery("MCP first-catalog phase budget", policy.firstCatalogBudgetMs, "ms")}`,
);
};
+27
View File
@@ -784,6 +784,33 @@ it("projects package details from dumpsys package output", async () => {
});
});
// Package Manager prints an absent installer as the text "null"; the
// normalized result must report the absence, not a package named "null"
// (morluto/rea#3943).
it("reports a null installer declaration as an absent installer", async () => {
const { binary } = await writeAdbStub(
shellStub({
"dumpsys:package": `Packages:
Package [com.example] (abcdef):
versionName=1.2.3
installerPackageName=null
requested permissions:
android.permission.INTERNET
`,
}),
);
const result = await evidenceResult(providerOver(binary), {
operation: "inspect_adb_package",
input: { serial: "emulator-5554", package: "com.example" },
});
expect(result).toMatchObject({
version_name: "1.2.3",
installer_package_name: null,
requested_permissions_count: 1,
coverage: "complete",
});
});
// PackageInfo.getLongVersionCode() combines versionCodeMajor and versionCode
// into a signed 64-bit long, which JSON numbers cannot hold exactly.
it.each(["9007199254740993", "-4294967296"])(
+46
View File
@@ -7,6 +7,52 @@ import {
parseWindowFocusOutput,
} from "./AdbShellOutput.js";
const packageDump = (installerLine: string | undefined): string =>
[
"Packages:",
" Package [com.example] (abcdef):",
" versionName=1.2.3",
...(installerLine === undefined ? [] : [` ${installerLine}`]),
" pkgFlags=[ HAS_CODE ]",
"",
].join("\n");
describe("parseDumpsysPackageOutput", () => {
it("keeps a reported installer package name", () => {
expect(
parseDumpsysPackageOutput(
packageDump("installerPackageName=com.android.vending"),
).installerPackageName,
).toBe("com.android.vending");
});
// Package Manager prints an absent String field as the text "null"; a
// missing or empty declaration means the same absence (morluto/rea#3943).
it.each([
["the literal null sentinel", "installerPackageName=null"],
["an empty value", "installerPackageName="],
["a missing line", undefined],
])("reports an absent installer as null for %s", (_case, line) => {
expect(parseDumpsysPackageOutput(packageDump(line))).toMatchObject({
versionName: "1.2.3",
installerPackageName: null,
});
});
// A manifest may declare the literal string "null" as its version name and
// dumpsys prints it the same way as an absent one, so the text is kept.
it("keeps a version name printed as the text null", () => {
expect(
parseDumpsysPackageOutput(
packageDump("installerPackageName=null").replace(
"versionName=1.2.3",
"versionName=null",
),
).versionName,
).toBe("null");
});
});
describe("parseWindowFocusOutput display contexts", () => {
it.each([false, true])(
"retains both display owners independent of block order: %s",
+12 -1
View File
@@ -275,6 +275,17 @@ export interface AdbPackageDetails {
const DETAIL_LINE = /^\s+([A-Za-z][A-Za-z0-9_]*)=(.*)$/u;
/**
* Package Manager prints its `String` fields by concatenation, so an absent
* `installerPackageName` appears as the literal text `null`
* (morluto/rea#3943). No installer can be named "null", so an empty value or
* that sentinel is the absence of the fact and becomes the nullable result's
* null. `versionName` keeps the printed text: a manifest may declare the
* string "null" on purpose, and the parser cannot tell the two apart.
*/
const absentInstallerOrName = (value: string): string | null =>
value === "" || value === "null" ? null : value;
/**
* Project `dumpsys package PACKAGE` output. Only the listed fields are
* extracted; the input is version-dependent and everything else is ignored
@@ -340,7 +351,7 @@ export const parseDumpsysPackageOutput = (
key === "installerPackageName" &&
details.installerPackageName === null
)
details.installerPackageName = value === "" ? null : value;
details.installerPackageName = absentInstallerOrName(value);
else if (key === "userId" && details.userId === null) {
const parsed = /^(\d+)/u.exec(value);
if (parsed !== null) details.userId = Number(parsed[1]!);
+89
View File
@@ -0,0 +1,89 @@
import { EventEmitter } from "node:events";
import { win32 } from "node:path";
import { PassThrough } from "node:stream";
import { afterEach, expect, it, vi } from "vitest";
const { spawnMock, tmpdirMock, lstatMock } = vi.hoisted(() => ({
spawnMock: vi.fn(),
tmpdirMock: vi.fn(),
lstatMock: vi.fn(),
}));
vi.mock("node:child_process", async (importOriginal) => ({
...(await importOriginal<typeof import("node:child_process")>()),
spawn: spawnMock,
}));
vi.mock("node:os", async (importOriginal) => ({
...(await importOriginal<typeof import("node:os")>()),
tmpdir: tmpdirMock,
}));
vi.mock("node:fs/promises", async (importOriginal) => ({
...(await importOriginal<typeof import("node:fs/promises")>()),
lstat: lstatMock,
}));
import { createJadxWorkspaceRoot } from "./JadxWorkspaceRoot.js";
afterEach(() => {
vi.restoreAllMocks();
vi.clearAllMocks();
});
it("decodes a Windows guardian path across split UTF-8 stdout writes", async () => {
const previousPlatform = process.platform;
Object.defineProperty(process, "platform", {
configurable: true,
value: "win32",
});
const parent = "C:\\Users\\研究者\\AppData\\Local\\Temp";
const path = win32.join(parent, `rea-android-${"a".repeat(32)}`);
const child = Object.assign(new EventEmitter(), {
stdin: new PassThrough(),
stdout: new PassThrough(),
stderr: new PassThrough(),
unref: vi.fn(),
});
child.stdin.once("finish", () => {
child.stdout.end();
child.stderr.end();
child.emit("close", 0, null);
});
tmpdirMock.mockReturnValue(parent);
const missingPath = Object.assign(new Error("missing after cleanup"), {
code: "ENOENT",
});
lstatMock.mockRejectedValue(missingPath);
spawnMock.mockReturnValue(child);
const ready = Buffer.from(
`${JSON.stringify({
path,
volumeSerial: "0123456789abcdef",
fileId: "0123456789abcdef0123456789abcdef",
privateDacl: true,
})}\n`,
);
const split = ready.indexOf(Buffer.from("研")) + 1;
try {
const start = createJadxWorkspaceRoot();
setImmediate(() => {
child.stdout.write(ready.subarray(0, split), () => {
setImmediate(() => child.stdout.write(ready.subarray(split)));
});
});
const root = await start;
try {
expect(root.path).toBe(path);
} finally {
await expect(root.close()).resolves.toBeUndefined();
}
expect(lstatMock).toHaveBeenCalledWith(path);
} finally {
Object.defineProperty(process, "platform", {
configurable: true,
value: previousPlatform,
});
}
});
+6 -4
View File
@@ -105,8 +105,9 @@ class WindowsJadxWorkspaceRoot implements JadxWorkspaceRoot {
},
});
let diagnostics = "";
child.stderr.on("data", (chunk: Buffer) => {
diagnostics = (diagnostics + chunk.toString()).slice(-8192);
child.stderr.setEncoding("utf8");
child.stderr.on("data", (chunk: string) => {
diagnostics = (diagnostics + chunk).slice(-8192);
});
const exit = new Promise<GuardianExit>((resolve) => {
child.once("error", (error: Error) =>
@@ -116,8 +117,9 @@ class WindowsJadxWorkspaceRoot implements JadxWorkspaceRoot {
});
const ready = new Promise<string>((resolve, reject) => {
let frame = "";
child.stdout.on("data", (chunk: Buffer) => {
frame += chunk.toString();
child.stdout.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => {
frame += chunk;
if (frame.length > 4096) {
reject(
new Error("Android workspace guardian readiness exceeded 4 KiB"),
+3
View File
@@ -1,3 +1,4 @@
import type { EvidenceCreationOptions } from "../domain/evidence.js";
import type { ProviderInspectionScope } from "../providers/ProviderInspectionScope.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import type { OfficialToolName } from "../contracts/officialToolContracts.js";
@@ -146,6 +147,7 @@ export const createAnalysisExecutionEvidence = (
input: unknown,
execution: AnalysisExecution,
confidence: "observed" | "derived",
options?: EvidenceCreationOptions,
): Evidence =>
createEvidence(
execution.subject === undefined
@@ -167,6 +169,7 @@ export const createAnalysisExecutionEvidence = (
locations: execution.locations,
confidence,
},
options,
);
/** Provider-neutral application capability for one analysis operation. */
+8
View File
@@ -138,6 +138,9 @@ const browserParameters = (
: "target_id" in input && observedOrigin !== undefined
? [observedOrigin]
: [],
...("command_timeout_ms" in input && input.command_timeout_ms !== undefined
? { command_timeout_ms: input.command_timeout_ms }
: {}),
};
if (!("target_id" in input)) return scope;
return {
@@ -171,6 +174,11 @@ const browserParameters = (
...(input.total_analysis_timeout_ms === undefined
? {}
: { total_analysis_timeout_ms: input.total_analysis_timeout_ms }),
...(input.source_map_fetch_timeout_ms === undefined
? {}
: {
source_map_fetch_timeout_ms: input.source_map_fetch_timeout_ms,
}),
}
: {}),
};
+90 -23
View File
@@ -1,4 +1,5 @@
import type { ExecutableFormatHint } from "../domain/dosCom.js";
import { dispatchQueuedInterrupts } from "../process/queuedInterrupts.js";
import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js";
@@ -59,7 +60,11 @@ import {
import type { AnalysisProviderSelector } from "../contracts/providerSelection.js";
import { artifactInspectionResultSchema } from "../domain/artifactInspection.js";
import { resolveXrefsAddress } from "./XrefsAddressResolution.js";
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
import {
AnalysisCancelledError,
AnalysisInputError,
} from "../domain/analysisErrorCore.js";
import { deliverAnalysisEvidence } from "./EvidenceBundleCommands.js";
import type { BinaryTargetError } from "../domain/configurationErrors.js";
import {
directAnalysisCleanupFailure,
@@ -146,24 +151,81 @@ export const runDirectAnalysis = async (
/** Execute one provider-native semantic operation with atomic provenance. */
export const runProviderAnalysis = async (
dependencies: DirectAnalysisDependencies,
...[path, tool, arguments_, logger = silentLogger, signal]: readonly [
path: string,
tool: NativeToolName | ArtifactAnalysisOperation | ManagedToolName,
arguments_: Readonly<Record<string, JsonValue>>,
logger?: Logger,
signal?: AbortSignal,
]
...[path, tool, arguments_, logger = silentLogger, signal, delivery]:
| readonly [
path: string,
tool: NativeToolName | ArtifactAnalysisOperation | ManagedToolName,
arguments_: Readonly<Record<string, JsonValue>>,
logger?: Logger | undefined,
signal?: AbortSignal | undefined,
]
| readonly [
path: string,
tool: "inspect_managed_members",
arguments_: Readonly<Record<string, JsonValue>>,
logger: Logger | undefined,
signal: AbortSignal | undefined,
delivery: {
readonly detail?: "complete" | "summary";
readonly evidenceBundlePath?: string;
},
]
): Promise<JsonValue> => {
if (isManagedToolName(tool))
return runManagedProviderAnalysis(
dependencies,
path,
tool,
arguments_,
signal,
if (delivery !== undefined && tool !== "inspect_managed_members")
return cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["delivery"],
reason: "invalid_value",
message:
"Evidence delivery options are only supported by inspect_managed_members",
},
]),
);
return withProcessCancellation(signal, (operationSignal) =>
runAnalysis(dependencies, path, tool, arguments_, {
if (
delivery?.detail === "summary" &&
delivery.evidenceBundlePath === undefined
)
return cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["evidence_bundle"],
reason: "missing_argument",
message:
"summary detail requires --evidence-bundle so complete Evidence remains available to later CLI commands",
},
]),
);
return withProcessCancellation(signal, async (operationSignal) => {
if (isManagedToolName(tool)) {
const original = await runManagedProviderAnalysis(
dependencies,
path,
tool,
arguments_,
operationSignal,
);
await dispatchQueuedInterrupts();
if (!original.ok) return cliError(original.error);
if (operationSignal.aborted)
return cliError(
new AnalysisCancelledError(tool, {
cause: operationSignal.reason,
partialObservation: original.value,
}),
);
const result =
delivery?.evidenceBundlePath === undefined
? original
: await deliverAnalysisEvidence(
original.value,
delivery.evidenceBundlePath,
delivery.detail,
operationSignal,
);
return result.ok ? result.value : cliError(result.error);
}
return runAnalysis(dependencies, path, tool, arguments_, {
logger,
snapshotPath: undefined,
signal: operationSignal,
@@ -172,8 +234,8 @@ export const runProviderAnalysis = async (
: NATIVE_INSPECTION_OPERATIONS.has(tool)
? { createSession: dependencies.createNativeBinarySession }
: {}),
}),
);
});
});
};
/** Execute managed metadata inspection in an isolated managed-only session. */
@@ -199,13 +261,14 @@ export const runManagedProviderExecution = async (
);
});
/** Preserve typed managed Evidence before selecting its CLI delivery. */
const runManagedProviderAnalysis = async (
dependencies: DirectAnalysisDependencies,
dependencies: Pick<DirectAnalysisDependencies, "createManagedBinarySession">,
path: string,
tool: ManagedToolName,
arguments_: Readonly<Record<string, JsonValue>>,
signal?: AbortSignal,
): Promise<JsonValue> => {
): Promise<Result<Evidence, AnalysisError>> => {
const execution = await runManagedProviderExecution(
dependencies,
path,
@@ -213,9 +276,13 @@ const runManagedProviderAnalysis = async (
signal,
arguments_,
);
if (!execution.ok) return cliError(execution.error);
if (!execution.ok) return execution;
const value = execution.value;
return createAnalysisExecutionEvidence(tool, arguments_, value, "observed");
return ok(
createAnalysisExecutionEvidence(tool, arguments_, value, "observed", {
immutable: true,
}),
);
};
const runAnalysis = async (
+159 -12
View File
@@ -1,9 +1,23 @@
import { dispatchQueuedInterrupts } from "../process/queuedInterrupts.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import { requireAuthenticationHeadroom } from "./JsonAuthenticationHeadroom.js";
import {
EvidenceFileError,
EvidenceIntegrityError,
type PublishedEvidenceFile,
} from "../domain/evidenceErrors.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import type { EvidenceBundle } from "../domain/evidenceBundle.js";
import {
AnalysisCancelledError,
AnalysisInputError,
} from "../domain/analysisErrorCore.js";
import type { Evidence } from "../domain/evidence.js";
import {
createImmutableEvidenceBundle,
type EvidenceBundle,
} from "../domain/evidenceBundle.js";
import type { InspectAnalysisViewInput } from "../domain/analysisView/analysisView.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { jsonValueSchema } from "../domain/jsonValue.js";
import { err, ok, type Result } from "../domain/result.js";
@@ -14,12 +28,18 @@ import {
readImmutableEvidenceBundle,
writeEvidenceBundle,
} from "./EvidenceBundleFiles.js";
import { inspectAnalysisViewValidated } from "./analysisView/AnalysisViewService.js";
/** Validate a standalone bundle and report its complete record/revision counts. */
export const importEvidenceBundleCommand = async (
path: string,
): Promise<Result<JsonValue, AnalysisError>> => {
const loaded = await readEvidenceBundle(path);
const loaded = await readEvidenceBundle(
path,
undefined,
undefined,
"import_evidence_bundle",
);
if (!loaded.ok) return loaded;
return ok({
imported: loaded.value.records.length,
@@ -35,16 +55,136 @@ export const exportEvidenceBundleCommand = async (
overwrite: boolean,
): Promise<Result<JsonValue, AnalysisError>> => {
// The sealed snapshot lets the writer reuse this authentication.
const loaded = await readImmutableEvidenceBundle(sourcePath);
const loaded = await readImmutableEvidenceBundle(
sourcePath,
undefined,
undefined,
"export_evidence_bundle",
);
if (!loaded.ok) return loaded;
return projectWrite(
loaded.value,
await persistEvidenceBundleCommand(loaded.value, outputPath, overwrite),
);
};
/** Publish complete parent Evidence before delivering it or its linked summary. */
export const deliverAnalysisEvidence = async (
evidence: Evidence,
outputPath: string,
detail: "complete" | "summary" = "complete",
signal?: AbortSignal,
): Promise<Result<Evidence, AnalysisError>> => {
const headroom = requireAuthenticationHeadroom(evidence, {
operation: evidence.operation,
path: absoluteCallerPath(outputPath),
description:
"Insufficient heap headroom to authenticate the complete parent Evidence",
});
if (!headroom.ok)
return err(headroom.error.retainPartialObservation(evidence));
const bundle = createImmutableEvidenceBundle([evidence]);
const delivered =
detail === "summary"
? inspectAnalysisViewValidated(
{
source: {
kind: "retained-evidence",
evidence_id: evidence.evidence_id,
},
view: { kind: "summary" },
},
(id) => bundle.records.find((record) => record.evidence_id === id),
)
: ok(evidence);
if (!delivered.ok) return delivered;
const written = await persistEvidenceBundleCommand(
bundle,
outputPath,
false,
signal,
evidence.operation,
);
// Deliver interrupts queued during final publication cleanup before the
// command's signal owner accepts completion and removes its listeners.
await dispatchQueuedInterrupts();
if (!written.ok) return err(written.error.retainPartialObservation(evidence));
if (signal?.aborted)
return err(
new AnalysisCancelledError(evidence.operation, {
cause: signal.reason,
partialObservation: delivered.value,
diagnostics: { evidence_bundle: { ...written.value } },
}),
);
return delivered;
};
/** Resolve the caller's exact retained parent from an authenticated portable bundle. */
export const inspectAnalysisViewFromBundle = async (
input: InspectAnalysisViewInput,
bundlePath: string,
resources: ArtifactResourceScope,
signal?: AbortSignal,
): Promise<Result<Evidence, AnalysisError>> => {
if (input.source.kind !== "retained-evidence")
return err(
new AnalysisInputError("inspect_analysis_view", undefined, [
{
path: ["source", "kind"],
reason: "invalid_value",
message:
"--evidence-bundle requires a retained-evidence source with the exact parent evidence_id; inline Evidence already supplies its source",
},
]),
);
const loaded = await readImmutableEvidenceBundle(
bundlePath,
signal,
resources,
"inspect_analysis_view",
);
// Authentication and projection are synchronous; dispatch queued interrupts before accepting either result.
await dispatchQueuedInterrupts();
if (!loaded.ok) return loaded;
if (signal?.aborted)
return err(
new AnalysisCancelledError("inspect_analysis_view", {
cause: signal.reason,
}),
);
const projected = inspectAnalysisViewValidated(input, (id) =>
loaded.value.records.find((record) => record.evidence_id === id),
);
await dispatchQueuedInterrupts();
if (projected.ok && signal?.aborted)
return err(
new AnalysisCancelledError("inspect_analysis_view", {
cause: signal.reason,
partialObservation: projected.value,
}),
);
return projected;
};
/** Publish a completed bundle and retry its exact staged-output cleanup owner. */
const persistEvidenceBundleCommand = async (
bundle: EvidenceBundle,
outputPath: string,
overwrite: boolean,
signal?: AbortSignal,
operation?: string,
): Promise<Result<PublishedEvidenceFile, AnalysisError>> => {
const resources = new ArtifactResourceScope();
const written = await writeEvidenceBundle(
loaded.value,
bundle,
outputPath,
overwrite,
resources,
signal,
operation,
);
if (written.ok) return projectWrite(loaded.value, written);
if (written.ok) return written;
// Confirmed release resolves the bound error to its primary outcome; a
// failed retry leaves that error incomplete with its exact retained owner.
@@ -54,10 +194,7 @@ export const exportEvidenceBundleCommand = async (
written.error instanceof EvidenceFileError &&
written.error.published
) {
return projectWrite(loaded.value, {
ok: true,
value: written.error.published,
});
return ok(written.error.published);
}
return written;
};
@@ -70,8 +207,18 @@ export const compareEvidenceBundlesCommand = async (input: {
// Sealed snapshots let comparison and its bundle digests reuse this
// authentication instead of re-validating every record twice more.
const [left, right] = await Promise.all([
readImmutableEvidenceBundle(input.leftPath),
readImmutableEvidenceBundle(input.rightPath),
readImmutableEvidenceBundle(
input.leftPath,
undefined,
undefined,
"compare_bundles",
),
readImmutableEvidenceBundle(
input.rightPath,
undefined,
undefined,
"compare_bundles",
),
]);
if (!left.ok) return left;
if (!right.ok) return right;
@@ -88,7 +235,7 @@ export const compareEvidenceBundlesCommand = async (input: {
const projectWrite = (
bundle: EvidenceBundle,
written: Awaited<ReturnType<typeof writeEvidenceBundle>>,
written: Result<PublishedEvidenceFile, AnalysisError>,
): Result<JsonValue, AnalysisError> =>
written.ok
? ok({
+24 -18
View File
@@ -1,3 +1,4 @@
import { requireAuthenticationHeadroom } from "./JsonAuthenticationHeadroom.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import {
@@ -39,45 +40,47 @@ export const readEvidenceBundle = (
path: string,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "import_evidence_bundle",
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
readBundle(path, parseEvidenceBundle, signal, resources);
readBundle(path, parseEvidenceBundle, signal, resources, operation);
/** Read an authenticated immutable snapshot for a subsequent session import. */
export const readImmutableEvidenceBundle = (
path: string,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "import_evidence_bundle",
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
readBundle(path, parseImmutableEvidenceBundle, signal, resources);
readBundle(path, parseImmutableEvidenceBundle, signal, resources, operation);
const readBundle = async (
path: string,
parseBundle: (input: unknown) => EvidenceBundle,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "import_evidence_bundle",
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> => {
if (signal?.aborted)
return err(
new AnalysisCancelledError("import_evidence_bundle", {
new AnalysisCancelledError(operation, {
cause: signal.reason,
}),
);
const loaded = await readJsonFile(path, signal, resources);
if (!loaded.ok) {
if (loaded.error instanceof AnalysisCancelledError)
return err(
new AnalysisCancelledError("import_evidence_bundle", {
cause: loaded.error.cause,
}),
);
return loaded;
}
const loaded = await readJsonFile(path, signal, resources, operation);
if (!loaded.ok) return loaded;
if (signal?.aborted)
return err(
new AnalysisCancelledError("import_evidence_bundle", {
new AnalysisCancelledError(operation, {
cause: signal.reason,
}),
);
const headroom = requireAuthenticationHeadroom(loaded.value, {
operation,
path: absoluteCallerPath(path),
description:
"Insufficient heap headroom to authenticate the Evidence bundle",
});
if (!headroom.ok) return headroom;
let bundle: EvidenceBundle;
try {
bundle = parseBundle(loaded.value);
@@ -121,6 +124,7 @@ export const writeEvidenceBundle = async (
overwrite: boolean,
resources: ArtifactResourceScope,
signal?: AbortSignal,
operation = "export_evidence_bundle",
): Promise<
Result<
{ readonly path: string; readonly bytes: number },
@@ -128,7 +132,11 @@ export const writeEvidenceBundle = async (
>
> => {
if (signal?.aborted === true)
return err(new AnalysisCancelledError("export_evidence_bundle"));
return err(
new AnalysisCancelledError(operation, {
cause: signal.reason,
}),
);
let checked: EvidenceBundle;
try {
checked = parseEvidenceBundle(bundle);
@@ -144,8 +152,6 @@ export const writeEvidenceBundle = async (
path,
overwrite,
resources,
signal === undefined
? undefined
: { signal, operation: "export_evidence_bundle" },
signal === undefined ? undefined : { signal, operation },
);
};
@@ -22,7 +22,9 @@ describe("exact Evidence input resolution", () => {
actual: null,
},
});
expect(projected.message).toContain("not retained in this session");
expect(projected.message).toContain(
"not available from the selected source",
);
expect(projected.remediation.action).toContain(
"complete inline Evidence",
);
+2 -2
View File
@@ -1,4 +1,4 @@
import type { EvidenceInput } from "../contracts/evidenceInputContracts.js";
import type { EvidenceInput } from "../domain/evidenceReference.js";
import type { Evidence } from "../domain/evidence.js";
import { EvidenceReferenceError } from "../domain/evidenceErrors.js";
import { err, ok, type Result } from "../domain/result.js";
@@ -18,7 +18,7 @@ export const resolveEvidenceInput = (
new EvidenceReferenceError(
input.evidence_id,
"missing",
"Evidence retained in this session",
"Evidence available to the selected source",
null,
),
);
+17 -7
View File
@@ -9,6 +9,7 @@ import {
type EvidenceSubjectTarget,
} from "../domain/evidence.js";
import { jsonValueSchema } from "../domain/jsonValue.js";
import type { MobileApplicationProjectionWithSources } from "../domain/mobileApplicationGraphProjection.js";
import { err, ok, type Result } from "../domain/result.js";
import { requestInputError, workflowInputError } from "./workflowInputError.js";
@@ -23,13 +24,20 @@ interface InventoryProjectionResult {
readonly limitations: readonly string[];
}
export interface InventoryProjectionEvidenceResult {
readonly evidence: Evidence;
readonly sources: readonly Evidence[];
}
interface InventoryProjectionOptions<
Input extends InventoryProjectionInput,
Output extends InventoryProjectionResult,
> {
readonly rawInput: unknown;
readonly schema: z.ZodType<Input>;
readonly project: (input: Input) => Output;
readonly project: (
input: Input,
) => MobileApplicationProjectionWithSources<Output>;
readonly operation: string;
readonly predicateType: string;
readonly provider: EvidenceProvider;
@@ -43,17 +51,18 @@ export const projectInventoryEvidence = <
Output extends InventoryProjectionResult,
>(
options: InventoryProjectionOptions<Input, Output>,
): Result<Evidence, AnalysisError> => {
): Result<InventoryProjectionEvidenceResult, AnalysisError> => {
const parsed = options.schema.safeParse(options.rawInput);
if (!parsed.success)
return err(
requestInputError(options.operation, parsed.error, options.rawInput),
);
try {
const result = options.project(parsed.data);
const first = parsed.data.inventory_evidence[0];
return ok(
createEvidence(
const projected = options.project(parsed.data);
const result = projected.projection;
const first = projected.sources[0];
return ok({
evidence: createEvidence(
first?.subject === null || first?.subject === undefined
? undefined
: {
@@ -80,7 +89,8 @@ export const projectInventoryEvidence = <
evidenceLinks: result.source_evidence_ids,
},
),
);
sources: projected.sources,
});
} catch (cause: unknown) {
return err(
cause instanceof TypeError || cause instanceof z.ZodError
-15
View File
@@ -3,7 +3,6 @@ import {
createAnalysisProfile,
type AnalysisProfileCommitment,
} from "../domain/analysisProfile.js";
/** Provider identity for deterministic composed analysis workflows. */
export const REA_WORKFLOW_PROVIDER = {
id: "rea-workflow",
@@ -112,20 +111,6 @@ export const JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER = {
version: "1",
} as const;
/** Provider identity for deterministic cross-artifact comparisons. */
export const ARTIFACT_COMPARISON_PROVIDER = {
id: "rea-artifact-comparison",
name: "REA artifact comparison",
version: "1",
} as const;
/** Provider identity for deterministic cross-function comparisons. */
export const FUNCTION_COMPARISON_PROVIDER = {
id: "rea-function-comparison",
name: "REA function comparison",
version: "1",
} as const;
/** Provider identity for deterministic Evidence bundle comparisons. */
export const BUNDLE_COMPARISON_PROVIDER = {
id: "rea-bundle-comparison",
@@ -4,6 +4,50 @@ import {
requireAuthenticationHeadroom,
} from "./JsonAuthenticationHeadroom.js";
describe("JSON authentication headroom", () => {
it("does not queue every row while deciding whether a wide report can fit", () => {
let pending = 0;
let peak = 0;
const rows: { admission_probe: boolean }[] = [];
for (let index = 0; index < 512; index++) {
const row = {
get admission_probe() {
pending--;
return true;
},
};
// Observe reading a row separately from examining its scalar. A width
// queue reads every row first; an iterator finishes each row before advancing.
Object.defineProperty(rows, index, {
enumerable: true,
get: () => {
peak = Math.max(peak, ++pending);
if (pending > 32)
throw new Error(
"Admission queued report rows before refusing their allocation",
);
return row;
},
});
}
const value = { rows };
const result = requireAuthenticationHeadroom(value, {
operation: "inspect_managed_members",
path: "/tmp/parent.json",
description: "Cannot authenticate",
availableBytes: 0,
});
expect(pending).toBe(0);
expect(peak).toBe(1);
expect(result).toMatchObject({
ok: false,
error: {
_tag: "AnalysisResourceConstraintError",
reportedLimits: {
serialized_json_utf16_bytes: JSON.stringify(value).length * 2,
},
},
});
});
it("estimates parsed JSON at no less than its UTF-16 text", () => {
const text = "x".repeat(1000);
expect(estimateJsonValueBytes(text)).toBeGreaterThanOrEqual(2000);
@@ -42,3 +86,28 @@ describe("JSON authentication headroom", () => {
).toEqual({ ok: true, value: undefined });
});
});
it.each([
["control characters", { text: "\u0000\u0001\b\t\n\f\r".repeat(2000) }],
[
"escaped keys",
{ ['\u0000\\"'.repeat(2000)]: [true, false, null, -0, 1e30] },
],
["surrogate boundaries", { text: "😀\ud800x\udfff".repeat(2000) }],
])("reserves actual escaped JSON storage for %s", (_label, value) => {
const result = requireAuthenticationHeadroom(value, {
operation: "inspect_analysis_view",
path: "/tmp/bundle.json",
description: "Cannot authenticate bundle",
availableBytes: estimateJsonValueBytes(value) * 2 + 1,
});
expect(result).toMatchObject({
ok: false,
error: {
_tag: "AnalysisResourceConstraintError",
reportedLimits: {
serialized_json_utf16_bytes: JSON.stringify(value).length * 2,
},
},
});
});
+58 -20
View File
@@ -1,32 +1,66 @@
import { jsonStringCodeUnits } from "../domain/jsonSerialization.js";
import { getHeapStatistics } from "node:v8";
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { err, ok, type Result } from "../domain/result.js";
/** Authentication clones parsed JSON and hashes its canonical commitment. */
const AUTHENTICATION_COPIES = 2;
/** Conservative bytes a parsed JSON value occupies: UTF-16 text plus container and slot overhead. */
export const estimateJsonValueBytes = (value: unknown): number => {
let bytes = 0;
const stack: unknown[] = [value];
/** Parsed working copies and conservative UTF-16 storage for escaped JSON. */
const estimateJsonStorage = (
value: unknown,
): { valueBytes: number; serializedUtf16Bytes: number } => {
let valueBytes = 0;
let serializedCharacters = 0;
const stack: Iterator<unknown>[] = [[value][Symbol.iterator]()];
while (stack.length > 0) {
const item = stack.pop();
if (typeof item === "string") bytes += 64 + item.length * 2;
else if (Array.isArray(item)) {
bytes += 128 + item.length * 16;
for (const entry of item) stack.push(entry);
const next = stack[stack.length - 1]!.next();
if (next.done) {
stack.pop();
continue;
}
const item = next.value;
if (typeof item === "string") {
valueBytes += 64 + item.length * 2;
serializedCharacters += jsonStringCodeUnits(item);
} else if (Array.isArray(item)) {
valueBytes += 128 + item.length * 16;
serializedCharacters += 2 + Math.max(0, item.length - 1);
stack.push(item.values());
} else if (typeof item === "object" && item !== null) {
bytes += 128;
for (const [key, entry] of Object.entries(item)) {
bytes += 64 + key.length * 2;
stack.push(entry);
}
} else bytes += 16;
valueBytes += 128;
serializedCharacters += 2;
stack.push(
(function* () {
let count = 0;
for (const key in item) {
if (!Object.hasOwn(item, key)) continue;
valueBytes += 64 + key.length * 2;
serializedCharacters +=
jsonStringCodeUnits(key) + 1 + (count++ === 0 ? 0 : 1);
yield (item as Record<string, unknown>)[key];
}
})(),
);
} else {
valueBytes += 16;
serializedCharacters +=
typeof item === "number"
? String(item).length
: item === true
? 4
: item === false
? 5
: 4;
}
}
return bytes;
return { valueBytes, serializedUtf16Bytes: serializedCharacters * 2 };
};
/** Conservative parsed JSON bytes, independent of its escaped representation. */
export const estimateJsonValueBytes = (value: unknown): number =>
estimateJsonStorage(value).valueBytes;
/** Refuse before authentication when live heap headroom cannot hold its validation and commitment working copies. */
export const requireAuthenticationHeadroom = (
value: unknown,
@@ -36,11 +70,14 @@ export const requireAuthenticationHeadroom = (
readonly description: string;
readonly availableBytes?: number;
},
): Result<void, AnalysisError> => {
): Result<void, AnalysisResourceConstraintError> => {
const availableBytes =
options.availableBytes ?? getHeapStatistics().total_available_size;
const valueBytes = estimateJsonValueBytes(value);
const requiredBytes = valueBytes * AUTHENTICATION_COPIES;
const { valueBytes, serializedUtf16Bytes } = estimateJsonStorage(value);
// Canonical hashing and manifest normalization can still encode scalars or
// metadata; account for their escaped representations before allocating them.
const requiredBytes =
(valueBytes + serializedUtf16Bytes) * AUTHENTICATION_COPIES;
if (requiredBytes <= availableBytes) return ok(undefined);
return err(
new AnalysisResourceConstraintError(
@@ -50,6 +87,7 @@ export const requireAuthenticationHeadroom = (
{
input_path: options.path,
projected_value_bytes: valueBytes,
serialized_json_utf16_bytes: serializedUtf16Bytes,
authentication_bytes: requiredBytes,
available_heap_bytes: availableBytes,
},
+8 -4
View File
@@ -52,13 +52,14 @@ export const readJsonFile = async (
path: string,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "read_evidence_file",
): Promise<
Result<
unknown,
EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError
>
> => {
const read = await readJsonFileWithSource(path, signal, resources);
const read = await readJsonFileWithSource(path, signal, resources, operation);
return read.ok ? ok(read.value.data) : read;
};
@@ -67,6 +68,7 @@ export const readJsonFileWithSource = (
path: string,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "read_evidence_file",
): Promise<
Result<
{ readonly data: unknown; readonly path: string },
@@ -78,13 +80,14 @@ export const readJsonFileWithSource = (
async (selected) => {
const read = await readJsonInputFile(
selected,
"read_evidence_file",
operation,
signal,
resources,
);
return read.ok ? ok({ data: read.value, path: selected }) : read;
},
signal,
operation,
);
/** Read selected JSON members through the same regular-file error boundary. */
@@ -124,6 +127,7 @@ const readJsonFileInput = async <Value>(
>
>,
signal?: AbortSignal,
operation = "read_evidence_file",
): Promise<
Result<
Value,
@@ -153,7 +157,7 @@ const readJsonFileInput = async <Value>(
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
if (cause instanceof JsonInputCancellationFailure)
return err(
new AnalysisCancelledError("read_evidence_file", {
new AnalysisCancelledError(operation, {
cause: cause.reason,
}),
);
@@ -166,7 +170,7 @@ const readJsonFileInput = async <Value>(
);
if (isJsonReadCancellation(cause, signal))
return err(
new AnalysisCancelledError("read_evidence_file", {
new AnalysisCancelledError(operation, {
cause: signal?.reason,
}),
);
@@ -35,7 +35,13 @@ import { MANAGED_WORKFLOW_PROVIDER } from "./InvestigationProviders.js";
import { createProcessCaptureEvidence } from "./process/ProcessEvidence.js";
import { deriveReconstructionObligationCandidates } from "./ReconstructionObligationCandidates.js";
const applicationWithUnknowns = (keys: readonly string[]) => {
const applicationWithUnknowns = (
keys: readonly string[],
{
kind = "unknown",
label = "Unclassified application boundary",
}: { kind?: "unknown" | "package"; label?: string } = {},
) => {
const source = createEvidence(
undefined,
{ id: "fixture-source", name: "Fixture source", version: "1" },
@@ -50,7 +56,7 @@ const applicationWithUnknowns = (keys: readonly string[]) => {
const base = buildSyntheticJavaScriptApplicationGraph();
const unknownNodes = keys.map((key) =>
createJavaScriptApplicationNode({
kind: "unknown",
kind,
identity: {
strategy: "artifact-local-key",
stability: "artifact-version",
@@ -60,7 +66,7 @@ const applicationWithUnknowns = (keys: readonly string[]) => {
},
observations: [
{
label: "Unclassified application boundary",
label,
properties: {},
evidence: artifactEvidence(
APPLICATION_GRAPH_DIGESTS.asar,
@@ -144,6 +150,22 @@ describe("reconstruction obligation candidates", () => {
).toBe(false);
});
it("falls back to a kind-based title when an application node label is blank", () => {
const {
unknownNodes: [node],
generated,
} = applicationWithUnknowns(["blank-label"], {
kind: "package",
label: " ",
});
expect(
generated.candidates.find(
({ target }) => target.application_node_id === node?.node_id,
)?.title,
).toBe("package boundary");
});
it("does not bind an unknown process image to a packaged artifact obligation", () => {
const scenario = parseProcessScenario({
executable: "/path/whose-launch-identity-is-unknown",
@@ -327,3 +349,87 @@ it("keeps each semantic candidate bound to its own file and authority context",
});
}
});
it("falls back to a kind-based title when a semantic node label is blank", () => {
const contexts = new JavaScriptSemanticEvidenceContextRegistry();
const base = javascriptApplicationAnalysisResultSchema.parse(
JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result,
);
const source = proofEvidence("blank-label-file");
const module = createJavaScriptSemanticGraphNode(
{
kind: "module",
identity: {
artifact_sha256: base.root_artifact_sha256,
module_path: "app.js",
source_range: null,
role_key: "module",
},
function_node_id: null,
application_node_ids: [],
label: "app.js",
properties: {},
evidence: artifactEvidence(base.root_artifact_sha256, "app.js"),
},
contexts,
);
const sha256 = "4".repeat(64);
const request = createJavaScriptSemanticGraphNode(
{
kind: "request",
identity: {
artifact_sha256: sha256,
module_path: "file.js",
source_range: {
start: { line: 1, column: 0 },
end: { line: 1, column: 8 },
},
role_key: "request-1",
},
function_node_id: null,
application_node_ids: [],
label: " ",
properties: {},
evidence: {
...artifactEvidence(sha256, "file.js", "ast-static-analysis"),
evidence_ids: [source.evidence_id],
},
},
contexts,
);
const { graph_id: _graphId, ...semantic } = base.semantic_graph;
const semanticGraph = createJavaScriptSemanticGraph({
...semantic,
root_node_ids: [module.node_id],
nodes: [module, request],
evidence_contexts: contexts.contexts,
});
const example = JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE;
const application = createEvidence(
{
path: base.input_path,
sha256: base.root_artifact_sha256,
format: base.format,
},
example.provider,
{
predicateType: example.predicate_type,
operation: example.operation,
parameters: example.parameters,
result: jsonValueSchema.parse({ ...base, semantic_graph: semanticGraph }),
confidence: "derived",
authority: "shipped-artifact",
evidenceLinks: [source.evidence_id],
},
);
const generated = deriveReconstructionObligationCandidates(
createEvidenceBundle([source, application]),
[],
);
expect(
generated.candidates.find(
({ target }) => target.semantic_node_id === request.node_id,
)?.title,
).toBe("request semantic boundary");
});
@@ -114,7 +114,7 @@ const deriveApplicationCandidates = (
candidates,
generatedCandidate({
key: `application:${node.node_id}:${candidatePolicy.family}`,
title: node.observations[0]?.label ?? `${node.kind} boundary`,
title: node.observations[0]?.label?.trim() || `${node.kind} boundary`,
policy: candidatePolicy,
artifactSha256: source.rootArtifactSha256,
applicationNodeId: node.node_id,
@@ -143,7 +143,7 @@ const deriveApplicationCandidates = (
candidates,
generatedCandidate({
key: `semantic:${node.node_id}:${candidatePolicy.family}`,
title: node.label ?? `${node.kind} semantic boundary`,
title: node.label?.trim() || `${node.kind} semantic boundary`,
policy: candidatePolicy,
artifactSha256: node.identity.artifact_sha256,
applicationNodeId: node.application_node_ids[0] ?? null,
+11 -40
View File
@@ -1,9 +1,11 @@
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import {
createHistoricalSourceGraph,
historicalSourceInventoryHasPartialFacts,
historicalSourceLanguages,
historicalSourceManifests,
historicalSourceParseFailureKey,
historicalSourceRelationshipKey,
type HistoricalSourceGraph,
type HistoricalSourceGraphInput,
} from "../domain/referenceSourceGraph.js";
@@ -45,18 +47,13 @@ const cancelled = (
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
const relationshipKey = (
relationship: HistoricalSourceGraphInput["relationships"][number],
): string =>
`${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`;
const deduplicateRelationships = (
relationships: HistoricalSourceGraphInput["relationships"],
): HistoricalSourceGraphInput["relationships"] => {
const seen = new Set<string>();
const result: HistoricalSourceGraphInput["relationships"] = [];
for (const relationship of relationships) {
const key = relationshipKey(relationship);
const key = historicalSourceRelationshipKey(relationship);
if (seen.has(key)) continue;
seen.add(key);
result.push(relationship);
@@ -93,37 +90,6 @@ const buildProvenance = (
caller: options.caller,
});
const deriveInventoryState = (
input: Pick<
HistoricalSourceGraphInput,
| "entries"
| "relationships"
| "parse_failures"
| "exclusions"
| "limitations"
>,
): "complete" | "partial" | "unknown" => {
if (input.limitations.length > 0) return "partial";
if (input.exclusions.length > 0) return "partial";
if (input.parse_failures.length > 0) return "partial";
const partialEntry = input.entries.some(
(entry) =>
entry.limitations.length > 0 ||
(entry.kind === "file" && entry.content_state !== "hashed") ||
(entry.kind === "directory" && entry.tree_state !== "enumerated") ||
(entry.kind === "symlink" && entry.target_state !== "internal") ||
entry.kind === "unknown",
);
if (partialEntry) return "partial";
const partialRelationship = input.relationships.some(
({ parse_state, resolution }) =>
parse_state !== "parsed" ||
["unresolved", "unknown"].includes(resolution),
);
if (partialRelationship) return "partial";
return "complete";
};
const sortExclusions = (
exclusions: HistoricalSourceGraphInput["exclusions"],
): HistoricalSourceGraphInput["exclusions"] =>
@@ -261,16 +227,21 @@ export const importReferenceSource = async (
schema: "HistoricalSourceGraph",
authority: "historical-reference",
root_alias: "$REFERENCE_ROOT",
inventory_state: deriveInventoryState({
inventory_state: historicalSourceInventoryHasPartialFacts({
entries: sortedEntries,
relationships: uniqueRelationships,
parse_failures: uniqueFailures,
exclusions: sortedExclusions,
limitations: sortedLimitations,
}),
})
? "partial"
: "complete",
entries: sortedEntries,
relationships: uniqueRelationships.sort((left, right) =>
compareUnicodeCodePoints(relationshipKey(left), relationshipKey(right)),
compareUnicodeCodePoints(
historicalSourceRelationshipKey(left),
historicalSourceRelationshipKey(right),
),
),
parse_failures: uniqueFailures,
exclusions: sortedExclusions,
+13 -40
View File
@@ -405,36 +405,20 @@ export const clientConfigurationAligned = async (
}
const original = await readRegularFileText(client.configPath);
const parsed = parseClientConfiguration(original, client.format);
const policy = await readClientPolicyBlock(
const desired = await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
files.value,
resolveProviderRequestBudget,
),
files.value,
resolveProviderRequestBudget,
);
const policy = await readClientPolicyBlock(client, parsed, desired);
if (!policy.ok) {
throwCleanupFailure(policy.error);
return false;
}
return (
policy.value === undefined &&
registrationCurrent(
parsed,
await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
files.value,
resolveProviderRequestBudget,
),
)
);
return policy.value === undefined && registrationCurrent(parsed, desired);
} catch (cause: unknown) {
if (cause instanceof RegularFileCleanupFailure) throw cause;
// Unreadable configuration is treated as not aligned so setup repairs it.
@@ -544,18 +528,15 @@ const inspectClientDocument = async (
}
try {
const parsed = parseClientConfiguration(original, client.format);
const policy = await readClientPolicyBlock(
const desired = await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
files.value,
resolveProviderRequestBudget,
),
files.value,
resolveProviderRequestBudget,
);
const policy = await readClientPolicyBlock(client, parsed, desired);
if (!policy.ok) {
throwCleanupFailure(policy.error);
return {
@@ -565,14 +546,6 @@ const inspectClientDocument = async (
}
if (policy.value !== undefined)
return { status: "invalid", remediation: policy.value };
const desired = await clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed,
files.value,
resolveProviderRequestBudget,
);
if (registrationCurrent(parsed, desired))
return { status: "already_current" };
const fallback = await readZcodeFallbackServers(client, parsed);
-1
View File
@@ -53,7 +53,6 @@ import {
redirectedWriteBlocker,
type MsixPackageView,
} from "./MsixPackageStorage.js";
export { isUninstallFailure } from "./UninstallFailure.js";
interface ManagedPathStats {
readonly uid?: number;
-1
View File
@@ -3,7 +3,6 @@ import { gt, valid } from "semver";
import { PRODUCT_IDENTITY } from "../identity.js";
import type { Result } from "../domain/result.js";
import type { IntegrationMaintenance } from "./UpdateMaintenance.js";
export { isUpdateFailure } from "./UpdateFailure.js";
/** Global npm installation that owns the running package and its entry point. */
export interface NpmInstallation {
-10
View File
@@ -1,6 +1,5 @@
import { closeOwnedWebResources } from "./closeOwnedWebResources.js";
import { isDeepStrictEqual } from "node:util";
import { isAbsolute } from "node:path";
import { z } from "zod";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import {
@@ -94,15 +93,6 @@ export class WebModuleTraceService {
}),
);
const input = parsed.data;
for (const [path, value] of [
["manifest_path", input.manifest_path],
["import_map.path", input.import_map?.path],
] as const)
if (value !== undefined && !isAbsolute(value))
return invalid(
path.split("."),
"Expected an absolute filesystem path on this host.",
);
for (const [path, value] of [
["importer_url", input.importer_url],
["import_map.base_url", input.import_map?.base_url],
@@ -43,6 +43,30 @@ it.each(["REDACTED", "[", "]", "/capture.har"])(
},
);
it("records an explicit capture deadline and omits the default", async () => {
const seen: Array<number | undefined> = [];
const service = new WebNetworkCaptureService({
inspect: (input) => {
seen.push(input.capture_timeout_ms);
return Promise.resolve(ok(fixture()));
},
});
const explicit = await service.inspect({
capture_path: "/capture.har",
format: "har",
capture_timeout_ms: 250,
});
if (!explicit.ok) throw explicit.error;
expect(explicit.value.parameters.capture_timeout_ms).toBe(250);
const omitted = await service.inspect({
capture_path: "/capture.har",
format: "har",
});
if (!omitted.ok) throw omitted.error;
expect(omitted.value.parameters).not.toHaveProperty("capture_timeout_ms");
expect(seen).toEqual([250, undefined]);
});
it("keeps observed artifact identity when the entire sensitive path is excluded", async () => {
const result = await new WebNetworkCaptureService({
inspect: () => Promise.resolve(ok(fixture())),
+3 -24
View File
@@ -1,7 +1,6 @@
import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js";
import { excludeCaptureCoordinates } from "../domain/webNetworkCaptureCoordinates.js";
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "./AnalysisProvider.js";
import type { WebNetworkCapturePort } from "./WebNetworkCapturePort.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
@@ -57,16 +56,6 @@ export class WebNetworkCaptureService {
}),
);
const input = parsed.data;
if (!isAbsolute(input.capture_path))
return failure(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["capture_path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
if (
input.record_ordinals !== undefined &&
new Set(input.record_ordinals).size !== input.record_ordinals.length
@@ -91,19 +80,6 @@ export class WebNetworkCaptureService {
},
]),
);
if (
input.protobuf?.schema !== undefined &&
!isAbsolute(input.protobuf.schema.descriptor_set_path)
)
return failure(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["protobuf", "schema", "descriptor_set_path"],
reason: "invalid_format",
message: "Expected an absolute descriptor-set path on this host.",
},
]),
);
const loaded = await this.port.inspect(input, options);
if (!loaded.ok) {
const partial = webNetworkCaptureSchema.safeParse(
@@ -250,6 +226,9 @@ export class WebNetworkCaptureService {
},
}),
sensitive_value_count: input.sensitive_values.length,
...(input.capture_timeout_ms === undefined
? {}
: { capture_timeout_ms: input.capture_timeout_ms }),
});
return ok(
createEvidence(
@@ -36,6 +36,7 @@ export interface WebSourceMapPort {
readonly path: string;
readonly url: string;
readonly position: WebSourceLocationInput["generated_position"];
readonly decodeTimeoutMs?: number;
},
options?: ExecutionOptions,
): Promise<Result<WebSourceMapReport, AnalysisError>>;
@@ -11,12 +11,55 @@ import {
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import { ok } from "../domain/result.js";
import { webSourceLocationResultSchema } from "../domain/webSourceLocation.js";
import {
DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS,
webSourceLocationResultSchema,
} from "../domain/webSourceLocation.js";
import { WebSourceLocationService } from "./WebSourceLocationService.js";
import type { WebSourceMapPort } from "./WebSourceLocationPorts.js";
const service = (data = webSourceLocationFixture(), port = decoder) =>
new WebSourceLocationService({ load: () => Promise.resolve(ok(data)) }, port);
const expectCodecDeadline = (
evidence: {
readonly limitations: readonly string[];
readonly normalized_result: unknown;
},
timeoutMs: number,
): void => {
const phrase = `${String(timeoutMs)} ms decode_timeout_ms deadline`;
const result = webSourceLocationResultSchema.parse(
evidence.normalized_result,
);
expect(
evidence.limitations.filter((text) => text.includes(phrase)),
).toHaveLength(1);
expect(
result.limitations.filter((text) => text.includes(phrase)),
).toHaveLength(1);
};
it("passes an explicit decode deadline and omits the default from Evidence", async () => {
const seen: Array<number | undefined> = [];
const port: WebSourceMapPort = {
trace: (input) => {
seen.push(input.decodeTimeoutMs);
return decoder.trace(input);
},
};
const explicit = await service(undefined, port).trace({
...args,
decode_timeout_ms: 250,
});
if (!explicit.ok) throw explicit.error;
expect(explicit.value.parameters.decode_timeout_ms).toBe(250);
expectCodecDeadline(explicit.value, 250);
const omitted = await service(undefined, port).trace(args);
if (!omitted.ok) throw omitted.error;
expect(omitted.value.parameters).not.toHaveProperty("decode_timeout_ms");
expectCodecDeadline(omitted.value, DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS);
expect(seen).toEqual([250, undefined]);
});
it.each([
{ ...args, manifest_path: "relative.json" },
{ ...args, script_index: -1 },
+14 -16
View File
@@ -1,6 +1,5 @@
import { closeOwnedWebResources } from "./closeOwnedWebResources.js";
import { createHash } from "node:crypto";
import { isAbsolute } from "node:path";
import { z } from "zod";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import {
@@ -17,6 +16,7 @@ import {
webArtifactFileSchema,
} from "../domain/webScriptArtifacts.js";
import {
DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS,
WEB_SOURCE_MAP_LIMITS,
webSourceLocationInputSchema,
webSourceLocationResultSchema,
@@ -52,17 +52,19 @@ const LIMITATIONS = [
"Lookup returns all segments at the greatest generated position not greater than the selected point, including earlier lines and generated-only segments. A mapped point does not establish an executed range or UI causality.",
"Regular maps and inline indexed sections are supported. External section URLs are unsupported; unknown extensions are retained in raw map text and are not interpreted.",
];
const CODEC_BUDGET =
"Complete evidence has a 4 MiB map, 32 MiB reply, 262144 decoded rows/segments and 64-level section budget. The owned codec has a 20-second deadline";
const codecBudget = (timeoutMs: number): string =>
`Complete evidence has a 4 MiB map, 32 MiB reply, 262144 decoded rows/segments and 64-level section budget. The owned codec has a ${String(timeoutMs)} ms decode_timeout_ms deadline`;
/** Bun accepts but ignores the codec's V8 heap flags; Windows enforces a commit limit instead. */
const codecBudgetLimitation = (
runtime: WebSourceMapReport["runtime"],
timeoutMs: number,
): string => {
const budget = codecBudget(timeoutMs);
if (runtime.id === "node-v8")
return `${CODEC_BUDGET}, ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB old-generation heap and one V8 worker; cleanup may extend the deadline. V8 heap limits do not establish aggregate RSS limits.`;
return `${budget}, ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB old-generation heap and one V8 worker; cleanup may extend the deadline. V8 heap limits do not establish aggregate RSS limits.`;
return runtime.process_memory_limit_bytes === null
? `${CODEC_BUDGET}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags, so no codec heap limit is established.`
: `${CODEC_BUDGET}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags; a Windows Job Object limited its committed memory to ${String(runtime.process_memory_limit_bytes)} bytes, which is not a heap or RSS limit.`;
? `${budget}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags, so no codec heap limit is established.`
: `${budget}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags; a Windows Job Object limited its committed memory to ${String(runtime.process_memory_limit_bytes)} bytes, which is not a heap or RSS limit.`;
};
/** Join verified captured bytes with an explicitly selected source-map point. */
@@ -125,15 +127,20 @@ export class WebSourceLocationService {
path: map.file.path,
url: map.url,
position: input.generated_position,
...(input.decode_timeout_ms === undefined
? {}
: { decodeTimeoutMs: input.decode_timeout_ms }),
},
options,
);
if (!traced.ok) return traced;
const report = verifyReport(input, map.file.sha256, traced.value);
if (!report.ok) return report;
const decodeTimeoutMs =
input.decode_timeout_ms ?? DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS;
const limitations = [
...LIMITATIONS,
codecBudgetLimitation(report.value.runtime),
codecBudgetLimitation(report.value.runtime, decodeTimeoutMs),
];
const normalized = webSourceLocationResultSchema.safeParse({
...report.value,
@@ -201,15 +208,6 @@ const invalid = (
const admitContext = (
input: WebSourceLocationInput,
): Result<null, AnalysisError> => {
for (const [path, value] of [
["manifest_path", input.manifest_path],
["source_map.path", input.source_map.path],
] as const)
if (!isAbsolute(value))
return invalid(
path.split("."),
"Expected an absolute filesystem path on this host.",
);
if (!URL.canParse(input.source_map.url))
return invalid(
["source_map", "url"],
@@ -1,5 +1,6 @@
import { setImmediate } from "node:timers/promises";
import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import { z } from "zod";
import { ANALYSIS_VIEW_PROVIDER } from "../InvestigationProviders.js";
@@ -25,7 +26,6 @@ import {
type AnalysisViewRequest,
type InspectAnalysisViewInput,
} from "../../domain/analysisView/analysisView.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import {
immutableJsonSnapshotMember,
isImmutableJsonSnapshot,
@@ -332,39 +332,44 @@ const createAnalysisViewEvidence = (
? {}
: { architecture: parent.subject.architecture }),
};
return createEvidence(target, ANALYSIS_VIEW_PROVIDER, {
predicateType: "rea.analysis-view",
operation: INSPECT_ANALYSIS_VIEW_OPERATION,
parameters: {
parent_evidence_id: parent.evidence_id,
view: jsonValueSchema.parse(input.view),
return createEvidence(
target,
ANALYSIS_VIEW_PROVIDER,
{
predicateType: "rea.analysis-view",
operation: INSPECT_ANALYSIS_VIEW_OPERATION,
parameters: {
parent_evidence_id: parent.evidence_id,
view: jsonValueSchema.parse(input.view),
},
result: jsonValueSchema.parse(result),
rawResult: null,
confidence:
// Grouping is whole-capture, so any templated group or parameter makes the result inferred;
// entity types and identity in an extracted state always are.
result.kind === "environment-state" ||
((result.kind === "api-surface" ||
result.kind === "environment-manifest") &&
result.templated_groups > 0)
? "inferred"
: "derived",
authority: parent.authority,
limitations: result.limitations,
...(result.kind === "native" && parent.relationship_coverage !== null
? { relationshipCoverage: parent.relationship_coverage }
: {}),
locations: [
...(result.artifact != null && result.artifact.path.length > 0
? [{ kind: "artifact-path" as const, path: result.artifact.path }]
: []),
...(result.kind === "native" &&
result.procedure_address !== null &&
result.procedure_address.length > 0
? [{ kind: "address" as const, address: result.procedure_address }]
: []),
],
evidenceLinks: [parent.evidence_id],
},
result: jsonValueSchema.parse(result),
rawResult: null,
confidence:
// Grouping is whole-capture, so any templated group or parameter makes the result inferred;
// entity types and identity in an extracted state always are.
result.kind === "environment-state" ||
((result.kind === "api-surface" ||
result.kind === "environment-manifest") &&
result.templated_groups > 0)
? "inferred"
: "derived",
authority: parent.authority,
limitations: result.limitations,
...(result.kind === "native" && parent.relationship_coverage !== null
? { relationshipCoverage: parent.relationship_coverage }
: {}),
locations: [
...(result.artifact != null && result.artifact.path.length > 0
? [{ kind: "artifact-path" as const, path: result.artifact.path }]
: []),
...(result.kind === "native" &&
result.procedure_address !== null &&
result.procedure_address.length > 0
? [{ kind: "address" as const, address: result.procedure_address }]
: []),
],
evidenceLinks: [parent.evidence_id],
});
{ immutable: true },
);
};
@@ -32,16 +32,13 @@ export const readEvidenceFileSource = async (
cause: signal.reason,
}),
);
const loaded = await readJsonFile(path, signal, resources);
if (!loaded.ok) {
if (loaded.error instanceof AnalysisCancelledError)
return err(
new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, {
cause: loaded.error.cause,
}),
);
return loaded;
}
const loaded = await readJsonFile(
path,
signal,
resources,
INSPECT_ANALYSIS_VIEW_OPERATION,
);
if (!loaded.ok) return loaded;
if (signal?.aborted)
return err(
new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, {
@@ -1,19 +1,21 @@
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { Evidence } from "../../domain/evidence.js";
import {
androidApplicationProjectionInputSchema,
projectAndroidApplication,
} from "../../domain/android/androidApplication.js";
import type { Result } from "../../domain/result.js";
import { ANDROID_APPLICATION_PROVIDER } from "../InvestigationProviders.js";
import { projectInventoryEvidence } from "../InventoryProjectionEvidence.js";
import {
projectInventoryEvidence,
type InventoryProjectionEvidenceResult,
} from "../InventoryProjectionEvidence.js";
const OPERATION = "project_android_application_graph" as const;
/** Project authenticated APK inventory Evidence into Android application Evidence. */
export const projectAndroidApplicationEvidence = (
rawInput: unknown,
): Result<Evidence, AnalysisError> => {
): Result<InventoryProjectionEvidenceResult, AnalysisError> => {
return projectInventoryEvidence({
rawInput,
schema: androidApplicationProjectionInputSchema,
@@ -1,19 +1,21 @@
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { Evidence } from "../../domain/evidence.js";
import {
appleApplicationProjectionInputSchema,
projectAppleApplication,
} from "../../domain/apple/appleApplication.js";
import type { Result } from "../../domain/result.js";
import { APPLE_APPLICATION_PROVIDER } from "../InvestigationProviders.js";
import { projectInventoryEvidence } from "../InventoryProjectionEvidence.js";
import {
projectInventoryEvidence,
type InventoryProjectionEvidenceResult,
} from "../InventoryProjectionEvidence.js";
const OPERATION = "project_apple_application_graph" as const;
/** Project authenticated IPA or macOS app inventory Evidence into Apple application evidence. */
export const projectAppleApplicationEvidence = (
rawInput: unknown,
): Result<Evidence, AnalysisError> => {
): Result<InventoryProjectionEvidenceResult, AnalysisError> => {
return projectInventoryEvidence({
rawInput,
schema: appleApplicationProjectionInputSchema,
@@ -1,4 +1,3 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { DesktopApplicationPort } from "./DesktopApplicationPort.js";
import { ARTIFACT_GRAPH_PROVIDER } from "../InvestigationProviders.js";
@@ -41,16 +40,6 @@ export class DesktopApplicationService {
cause: parsed.error,
}),
);
if (!isAbsolute(parsed.data.path))
return err(
new AnalysisInputError(operation, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute directory path on this host.",
},
]),
);
const result = await this.reader.inspect(parsed.data, options);
if (!result.ok) return result;
const report = desktopApplicationSchema.safeParse(result.value);
@@ -25,10 +25,18 @@ it.each([
throw new Error("must not run");
},
});
expect(await service.inspect(input)).toMatchObject({
const result = await service.inspect(input);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError" },
});
if (input.path === "relative.elf")
expect(result).toMatchObject({
ok: false,
error: {
issues: [{ path: ["path"], reason: "invalid_format" }],
},
});
},
);
@@ -1,10 +1,8 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { BinaryLayoutPort } from "./BinaryLayoutPort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -66,16 +64,6 @@ export class BinaryLayoutService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const inspected = await inspect(input.data);
if (!inspected.ok) return inspected;
const diagnostics = binaryLayoutSchema.shape.diagnostics.safeParse(
@@ -1,10 +1,8 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { PeResourcesPort } from "./PeResourcesPort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -47,16 +45,6 @@ export class PeResourcesService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(operation, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const result = await this.provider.inspect(input.data, options);
if (!result.ok) return result;
const report = peResourcesSchema.safeParse(result.value);
@@ -70,10 +70,18 @@ it.each([
throw new Error("must not run");
},
});
expect(await service.inspect(input)).toMatchObject({
const result = await service.inspect(input);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError" },
});
if (input.path === "relative.elf")
expect(result).toMatchObject({
ok: false,
error: {
issues: [{ path: ["path"], reason: "invalid_format" }],
},
});
},
);
@@ -1,10 +1,8 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { RecordedCrashPort } from "./RecordedCrashPort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -67,16 +65,6 @@ export class RecordedCrashService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const inspected = await inspect(input.data);
if (!inspected.ok) return inspected;
const diagnostics = recordedCrashSchema.shape.diagnostics.safeParse(
@@ -0,0 +1,102 @@
import { expect, it } from "vitest";
import {
createAnalysisExecution,
type ExecutionOptions,
} from "../AnalysisProvider.js";
import { PacketCaptureService } from "./PacketCaptureService.js";
import { createAnalysisProfile } from "../../domain/analysisProfile.js";
import { inspectPacketCaptureInputSchema } from "../../domain/captures/packetCapture.js";
import { ok } from "../../domain/result.js";
it("retains provider metadata in a late-cancellation observation", async () => {
const input = inspectPacketCaptureInputSchema.parse({
path: process.platform === "win32" ? "C:\\capture.pcap" : "/capture.pcap",
fields: ["ip.addr"],
});
const provider = {
id: "fixture-capture",
name: "Fixture capture provider",
version: "1",
} as const;
const subject = {
path: input.path,
sha256: "a".repeat(64),
format: "file",
} as const;
const report = {
artifact: { path: input.path, sha256: subject.sha256, bytes: 12 },
selection: input,
packets: [],
output_window: {
matching_packets: 0,
returned_packets: 0,
has_more: false,
},
read_scope: "to-eof",
diagnostics: "",
tool: {
configured_path: "/fixture/tshark",
path: "/fixture/tshark",
sha256: "b".repeat(64),
version_banner: "TShark fixture",
version_diagnostics: "",
arguments: [],
},
configuration: "private-empty-personal-profile",
resource_limiter: {
configured_path: "/fixture/prlimit",
path: "/fixture/prlimit",
sha256: "c".repeat(64),
arguments: [],
},
limits: {
input_bytes: 1,
output_bytes: 1,
command_timeout_ms: 1,
address_space_bytes: 0,
file_bytes: 0,
cpu_seconds: 0,
},
hardware_acquisition: "not-performed",
} as const;
const rawResult = { producer: "fixture", bytes: 12 };
const profile = createAnalysisProfile(provider, { filter: null });
const relationshipCoverage = {
status: "unknown",
reason: "Packet relationships are not reported.",
} as const;
const execution = createAnalysisExecution(report, provider, {
subject,
rawResult,
analysisProfile: profile,
relationshipCoverage,
locations: [{ kind: "artifact-path", path: input.path }],
});
const controller = new AbortController();
const reason = Object.freeze({ requested: "cancel after capture result" });
const service = new PacketCaptureService({
inspect: async (_input, options?: ExecutionOptions) => {
controller.abort(reason);
expect(options?.signal).toBe(controller.signal);
return ok(execution);
},
inspectFields: async () => {
throw new Error("Unexpected field discovery call");
},
close: async () => {},
});
const result = await service.inspect(input, { signal: controller.signal });
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected late cancellation");
expect(result.error).toMatchObject({ _tag: "AnalysisCancelledError" });
expect(result.error.cause).toBe(reason);
expect(result.error.partialObservation).toMatchObject({
operation: "inspect_packet_capture",
raw_result: rawResult,
analysis_profile: profile,
relationship_coverage: relationshipCoverage,
});
});
@@ -3,26 +3,25 @@ import {
packetFieldsCatalogSchema,
type InspectPacketFieldsInput,
} from "../../domain/captures/packetFields.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { isAbsolute } from "node:path";
import type {
AnalysisExecution,
ExecutionOptions,
import {
createAnalysisExecutionEvidence,
type AnalysisExecution,
type ExecutionOptions,
} from "../AnalysisProvider.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
import type { Evidence } from "../../domain/evidence.js";
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
import {
inspectPacketCaptureInputSchema,
packetCaptureSchema,
type InspectPacketCaptureInput,
} from "../../domain/captures/packetCapture.js";
import { err, ok, type Result } from "../../domain/result.js";
import { err, type Result } from "../../domain/result.js";
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
export interface PacketCapturePort {
/** Release per-call resources before returning, or retain failed cleanup for close/retry. */
@@ -59,16 +58,6 @@ export class PacketCaptureService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(operation, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute local path.",
},
]),
);
const decoded = await this.provider.inspect(input.data, options);
if (!decoded.ok) return decoded;
const execution = decoded.value;
@@ -86,7 +75,15 @@ export class PacketCaptureService {
"Capture analyzer returned malformed data or changed the selected artifact identity/options.",
),
);
return this.#evidence(operation, input.data, execution, options);
return completeProviderEvidence(
createAnalysisExecutionEvidence(
operation,
input.data,
execution,
"observed",
),
options?.signal,
);
}
async inspectFields(
raw: unknown,
@@ -120,33 +117,14 @@ export class PacketCaptureService {
"Field discovery returned malformed data or changed the selected installation/options.",
),
);
return this.#evidence(operation, input.data, execution, options);
}
#evidence(
operation: string,
parameters: Readonly<Record<string, JsonValue>>,
execution: AnalysisExecution,
options?: ExecutionOptions,
): Result<Evidence, AnalysisError> {
const evidence = createEvidence(
execution.subject ?? undefined,
execution.provider,
{
return completeProviderEvidence(
createAnalysisExecutionEvidence(
operation,
parameters,
result: execution.result,
confidence: "observed",
locations: execution.locations,
limitations: execution.limitations,
},
input.data,
execution,
"observed",
),
options?.signal,
);
return options?.signal?.aborted
? err(
new AnalysisCancelledError(operation, {
cause: options.signal.reason,
partialObservation: evidence,
}),
)
: ok(evidence);
}
}
@@ -1,11 +1,8 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { ElectronFusePort } from "./ElectronFusePort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import {
@@ -45,16 +42,6 @@ export class ElectronFuseService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const inspected = await this.provider.inspect(input.data, options);
if (!inspected.ok) return inspected;
const report = electronFuseReportSchema.safeParse(inspected.value);
@@ -387,10 +387,11 @@ describe("planEnvironmentBundle without a state", () => {
it("serves and predicts replay when the state's catalog exceeds the discovery budget that replay's fits", () => {
// Read tools only, so leaving out write tools saves nothing while each state description is longer.
// 300 sits between the two catalogs on this fixture: replay still fits and the state catalog does not.
const reads = servedSource.manifest.tools.filter(
({ method }) => method === "GET",
);
const tools = Array.from({ length: 350 }, (_, index) => {
const tools = Array.from({ length: 300 }, (_, index) => {
const tool = reads[index % reads.length]!;
return index < reads.length
? tool
@@ -416,7 +417,7 @@ describe("planEnvironmentBundle without a state", () => {
served: false,
code: "resource_constraint",
});
expect(planned.value.toolCount).toBe(350);
expect(planned.value.toolCount).toBe(300);
expect(planned.value.check).toEqual({
tool: "get_annotations",
recordOrdinal: 25,
@@ -13,13 +13,13 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js";
import {
REPLAY_BODY_LIMIT_BYTES,
type EnvironmentSource,
type RecordedExchange,
} from "./environmentSource.js";
import {
replayEnvironmentTool,
selectReplayExchange,
} from "./environmentReplay.js";
import type {
EnvironmentSource,
RecordedExchange,
} from "./environmentSource.js";
const source = environmentServeSource();
@@ -1,8 +1,5 @@
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
import { mediaTypeEssence } from "../../domain/analysisView/webNetworkRouteTemplates.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { parseExactJson } from "../../domain/exactJsonNumber.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import {
iterateUrlQueryParameters,
@@ -13,13 +10,12 @@ import { environmentToolArguments } from "./environmentTools.js";
import {
REPLAY_BODY_LIMIT_BYTES,
ENVIRONMENT_SERVE_OPERATION,
replayedBody,
type EnvironmentManifestTool,
type EnvironmentSource,
type RecordedExchange,
} from "./environmentSource.js";
export { REPLAY_BODY_LIMIT_BYTES } from "./environmentSource.js";
/** Caller arguments as the MCP tool receives them. */
export type ReplayArguments = Readonly<Record<string, unknown>>;
@@ -183,11 +179,6 @@ const utf8Prefix = (text: string, maxBytes: number): string => {
return text.slice(0, read);
};
/** Bytes a padded base64 string decodes to. */
const decodedBase64Bytes = (encoded: string): number =>
Math.floor((encoded.length * 3) / 4) -
(encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0);
/** Base64 characters, in whole groups, whose bytes fit the replay limit. */
const BASE64_LIMIT_CHARS = Math.floor(REPLAY_BODY_LIMIT_BYTES / 3) * 4;
@@ -196,29 +187,26 @@ interface ReplayBody {
readonly truncated: boolean;
}
/** The limit applies to the recorded bytes, not to their base64 spelling. */
const replayBase64Body = (encoded: string, cut: boolean): ReplayBody =>
!cut && decodedBase64Bytes(encoded) <= REPLAY_BODY_LIMIT_BYTES
? { body: encoded, truncated: false }
: { body: encoded.slice(0, BASE64_LIMIT_CHARS), truncated: true };
/** Parse a retained body as JSON when its content type is JSON-like; otherwise return the text. */
/** The reply body and whether it was cut, as the source classifies the retained body. */
const replayBody = (exchange: RecordedExchange): ReplayBody => {
const text = exchange.bodyText;
if (text === null) return { body: null, truncated: false };
if (exchange.bodyEncoding === "base64")
return replayBase64Body(text, exchange.bodyCut === true);
const prefix = utf8Prefix(text, REPLAY_BODY_LIMIT_BYTES);
if (prefix.length < text.length || exchange.bodyCut === true)
return { body: prefix, truncated: true };
if (
exchange.contentType !== null &&
isJsonLike(mediaTypeEssence(exchange.contentType))
) {
const value = parseExactJson(text);
if (value !== undefined) return { body: value, truncated: false };
const body = replayedBody(exchange);
switch (body.kind) {
case "none":
return { body: null, truncated: false };
case "base64":
return body.whole
? { body: body.encoded, truncated: false }
: { body: body.encoded.slice(0, BASE64_LIMIT_CHARS), truncated: true };
case "cut":
return {
body: utf8Prefix(body.text, REPLAY_BODY_LIMIT_BYTES),
truncated: true,
};
case "json":
return { body: body.value, truncated: false };
case "text":
return { body: body.text, truncated: false };
}
return { body: text, truncated: false };
};
/** Recorded paths and queries are cut to this many UTF-8 bytes in replies. */
@@ -9,7 +9,11 @@ import {
readRecordedResponseBody,
type RecordedResponseBody,
} from "../../domain/analysisView/recordedResponseBody.js";
import { readRecordedRequestLine } from "../../domain/analysisView/webNetworkRouteTemplates.js";
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
import {
mediaTypeEssence,
readRecordedRequestLine,
} from "../../domain/analysisView/webNetworkRouteTemplates.js";
import { browserScenarioCaptureSchema } from "../../domain/browserScenarioCapture.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
@@ -17,8 +21,10 @@ import {
type AnalysisInputIssue,
} from "../../domain/analysisErrorCore.js";
import { immutableEvidence, type Evidence } from "../../domain/evidence.js";
import { parseExactJson } from "../../domain/exactJsonNumber.js";
import { projectInputIssues } from "../../domain/inputIssueProjection.js";
import { immutableJsonSnapshotMember } from "../../domain/immutableJson.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import {
safeParseOwnedWebNetworkCapture,
@@ -47,6 +53,52 @@ export interface RecordedExchange extends RecordedResponseBody {
/** Replayed bodies larger than this are cut at a UTF-8 boundary and flagged. */
export const REPLAY_BODY_LIMIT_BYTES = 256 * 1024;
/** How replay answers with an exchange's body; replay and tool descriptions both read it from here. */
export type ReplayedBody =
| { readonly kind: "none" }
/** Non-UTF-8 bytes as base64; `whole` when every recorded byte fits the replay limit. */
| {
readonly kind: "base64";
readonly encoded: string;
readonly whole: boolean;
}
/** UTF-8 text over the replay limit or cut when retained; replay answers with a prefix. */
| { readonly kind: "cut"; readonly text: string }
| { readonly kind: "json"; readonly value: JsonValue }
| { readonly kind: "text"; readonly text: string };
/** Bytes a padded base64 string decodes to. */
const decodedBase64Bytes = (encoded: string): number =>
Math.floor((encoded.length * 3) / 4) -
(encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0);
/** Classify a retained body once: whole JSON-like UTF-8 text within the limit that parses exactly is JSON. */
export const replayedBody = (exchange: RecordedExchange): ReplayedBody => {
const text = exchange.bodyText;
if (text === null) return { kind: "none" };
if (exchange.bodyEncoding === "base64")
return {
kind: "base64",
encoded: text,
whole:
exchange.bodyCut !== true &&
decodedBase64Bytes(text) <= REPLAY_BODY_LIMIT_BYTES,
};
if (
exchange.bodyCut === true ||
// UTF-8 never takes fewer bytes than UTF-16 units, so the length check avoids encoding huge text.
text.length > REPLAY_BODY_LIMIT_BYTES ||
Buffer.byteLength(text, "utf8") > REPLAY_BODY_LIMIT_BYTES
)
return { kind: "cut", text };
const value =
exchange.contentType !== null &&
isJsonLike(mediaTypeEssence(exchange.contentType))
? parseExactJson(text)
: undefined;
return value === undefined ? { kind: "text", text } : { kind: "json", value };
};
/** A HAR inspected by inspect_web_network_capture, or a session recorded by capture_browser_scenario. */
export type EnvironmentCaptureKind = "web-network-capture" | "browser-scenario";
@@ -418,6 +418,30 @@ describe("EnvironmentStateStore", () => {
});
});
describe("caller-owned state replacement", () => {
it("does not retain mutable data from a caller-owned replacement", () => {
const store = new EnvironmentStateStore(state);
const document = documentWith((value) => {
const entity = value.rea_state.entities.find((item) => item.id === "2");
if (!entity) throw new Error("Expected the user entity");
entity.fields = { ...(entity.fields as object), login: "renamed" };
});
expect(store.replace(document).ok).toBe(true);
const callerEntity = document.rea_state.entities.find(
(entity) => entity.id === "2",
);
if (!callerEntity) throw new Error("Expected the caller's user entity");
(callerEntity.fields as Record<string, unknown>).login = "later edit";
expect(store.snapshot().rea_state.entities).toContainEqual(
expect.objectContaining({
id: "2",
fields: expect.objectContaining({ login: "renamed" }),
}),
);
});
});
describe("rea-state/v1 validation", () => {
it.each([
[
@@ -461,12 +485,12 @@ describe("rea-state/v1 validation", () => {
"a document for a tool that is not a document view",
(document: Editable) => {
document.rea_state.documents.push({
tool: "get_org",
path: "/api/org",
tool: "get_folders",
path: "/api/folders",
value: {},
});
},
"rea_state/documents/0/tool",
`rea_state/documents/${String(state.documents.length)}/tool`,
],
[
"a member the format does not define",
@@ -570,3 +594,36 @@ describe("rea-state/v1 validation", () => {
).toMatchObject({ path: "rea_state/version" });
});
});
describe("wide state value validation", () => {
it("preserves a wide caller value through the state replacement boundary", () => {
const wide = Array.from({ length: 200_000 }, (_, index) => index % 10);
const wideObject = Object.fromEntries(
Array.from({ length: 20_000 }, (_, index) => [
`field_${String(index)}`,
index,
]),
);
const document = documentWith((value) => {
const fields = value.rea_state.entities[0]!.fields as Record<
string,
unknown
>;
value.rea_state.entities[0]!.fields = {
...fields,
caller_wide: wide,
caller_wide_object: wideObject,
};
});
const store = new EnvironmentStateStore(state);
expect(store.replace(document).ok).toBe(true);
const retained = store.snapshot().rea_state.entities[0]?.fields.caller_wide;
if (!Array.isArray(retained)) throw new Error("Expected the wide value");
expect(retained).toHaveLength(wide.length);
expect(retained.at(-1)).toBe(wide.at(-1));
const retainedObject =
store.snapshot().rea_state.entities[0]?.fields.caller_wide_object;
expect(retainedObject).toEqual(wideObject);
});
});
@@ -61,17 +61,19 @@ export const servedCatalog = (
operation: string = ENVIRONMENT_SERVE_OPERATION,
): Result<ServedCatalog, AnalysisResourceConstraintError> => {
const dataPlane: EnvironmentDataPlane = fromState ? "state" : "none";
const tools = environmentToolDefinitions(
fromState
? {
const catalogSource: EnvironmentSource = fromState
? {
...source,
manifest: {
...source.manifest,
tools: source.manifest.tools.filter(servedFromState),
}
: source.manifest,
source.evidenceId,
name,
{ operation, dataPlane },
);
},
}
: source;
const tools = environmentToolDefinitions(catalogSource, name, {
operation,
dataPlane,
});
return tools.ok ? ok({ tools: tools.value, dataPlane }) : tools;
};
@@ -9,8 +9,10 @@ import {
environmentStateServeInputs,
} from "../../../tests/fixtures/environmentServe.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { REPLAY_BODY_LIMIT_BYTES } from "./environmentReplay.js";
import type { EnvironmentSource } from "./environmentSource.js";
import {
REPLAY_BODY_LIMIT_BYTES,
type EnvironmentSource,
} from "./environmentSource.js";
import {
answerFromState,
type StateServing,
@@ -311,6 +313,17 @@ describe("recorded and document answers", () => {
});
});
it("replays a read whose rendered document exceeds the body limit", () => {
const values = Array.from({ length: 60_000 }, () => "1e3").join(",");
const body = `{"values":[${values}]}`;
const serving = servingFrom([["GET", "/api/config", body]]);
expect(serving.store.state.documents).toEqual([]);
expect(answer(serving, "get_config", {})).toMatchObject({
replay: { record_ordinal: 0 },
state: { source: "recording", reason: "rendered-oversize" },
});
});
it("answers a document's recorded line with the state's value for it", () => {
const capture = timedCaptureEvidence([
["GET", "/api/settings", { theme: "dark", density: "compact" }],
@@ -24,7 +24,6 @@ import { err, ok, type Result } from "../../domain/result.js";
import {
decodedPathSegment,
pathValueMatches,
REPLAY_BODY_LIMIT_BYTES,
replayAnswer,
replayEnvironmentTool,
selectReplayExchange,
@@ -34,6 +33,7 @@ import {
} from "./environmentReplay.js";
import {
ENVIRONMENT_SERVE_OPERATION,
REPLAY_BODY_LIMIT_BYTES,
type EnvironmentManifestTool,
type EnvironmentSource,
type RecordedExchange,
@@ -1,3 +1,4 @@
import { types } from "node:util";
import { z } from "zod";
import {
@@ -16,7 +17,11 @@ import {
} from "../../domain/analysisErrorCore.js";
import { canonicalJson } from "../../domain/comparisonSemantics.js";
import { projectInputIssues } from "../../domain/inputIssueProjection.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import {
jsonValueSchema,
ownedJsonValueSchema,
type JsonValue,
} from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { compositeKey } from "../../domain/unicodeCodePointOrder.js";
import type { EnvironmentState } from "./environmentState.js";
@@ -37,37 +42,42 @@ export interface StateContent {
readonly documents: readonly HeldDocument[];
}
const fieldsSchema = z.record(z.string(), jsonValueSchema);
const ordinalSchema = z.number().int().nonnegative();
const stateDocumentSchemaFor = (jsonValue: z.ZodType<JsonValue, unknown>) => {
const fieldsSchema = z.record(z.string(), jsonValue);
const ordinalSchema = z.number().int().nonnegative();
const stateDocumentSchema = z.strictObject({
rea_state: z.strictObject({
version: z.literal(1),
// `data/get` reports these; accepting them back lets its output be loaded unchanged.
state_evidence_id: z.string().optional(),
types: z.array(jsonValueSchema).optional(),
entities: z.array(
z.strictObject({
type: z.string(),
collection: z.string(),
id: z.string(),
fields: fieldsSchema,
view_fields: z
.array(z.strictObject({ tool: z.string(), fields: fieldsSchema }))
.optional(),
record_ordinals: z.array(ordinalSchema).optional(),
}),
),
documents: z.array(
z.strictObject({
tool: z.string(),
path: z.string(),
value: jsonValueSchema,
record_ordinal: ordinalSchema.optional(),
}),
),
}),
});
return z.strictObject({
rea_state: z.strictObject({
version: z.literal(1),
// `data/get` reports these; accepting them back lets its output be loaded unchanged.
state_evidence_id: z.string().optional(),
types: z.array(jsonValue).optional(),
entities: z.array(
z.strictObject({
type: z.string(),
collection: z.string(),
id: z.string(),
fields: fieldsSchema,
view_fields: z
.array(z.strictObject({ tool: z.string(), fields: fieldsSchema }))
.optional(),
record_ordinals: z.array(ordinalSchema).optional(),
}),
),
documents: z.array(
z.strictObject({
tool: z.string(),
path: z.string(),
value: jsonValue,
record_ordinal: ordinalSchema.optional(),
}),
),
}),
});
};
const stateDocumentSchema = stateDocumentSchemaFor(jsonValueSchema);
const ownedStateDocumentSchema = stateDocumentSchemaFor(ownedJsonValueSchema);
type ParsedDocument = z.output<typeof stateDocumentSchema>["rea_state"];
type ParsedEntity = ParsedDocument["entities"][number];
@@ -105,31 +115,82 @@ const memberProblem = (path: Path): AnalysisInputIssue => {
/** The path of the first `__proto__` member in document order; schema parsing would drop such a member rather than refuse it. */
const prototypeMemberPath = (root: unknown): Path | undefined => {
interface Node {
interface Cursor {
readonly value: unknown;
readonly parent: Node | undefined;
readonly parent: Cursor | undefined;
readonly key: string | number;
readonly entries?: Generator<string>;
index: number;
}
const pathOf = (node: Node): (string | number)[] => {
const pathOf = (node: Cursor): (string | number)[] => {
const path: (string | number)[] = [];
for (let at: Node = node; at.parent !== undefined; at = at.parent)
for (let at: Cursor = node; at.parent !== undefined; at = at.parent)
path.push(at.key);
return path.reverse();
};
const pending: Node[] = [{ value: root, parent: undefined, key: "" }];
for (let node = pending.pop(); node !== undefined; node = pending.pop()) {
function* ownKeys(value: Record<string, unknown>): Generator<string> {
// `for...in` is lazy. The ownership check keeps inherited enumerable
// properties out, matching Object.entries without allocating all entries.
for (const key in value) if (Object.hasOwn(value, key)) yield key;
}
const cursor = (
value: unknown,
parent: Cursor | undefined,
key: string | number,
): Cursor => ({
value,
parent,
key,
index: 0,
...(!Array.isArray(value) && typeof value === "object" && value !== null
? { entries: ownKeys(value as Record<string, unknown>) }
: {}),
});
const pending: Cursor[] = [cursor(root, undefined, "")];
while (pending.length > 0) {
const node = pending[pending.length - 1]!;
const { value } = node;
if (typeof value !== "object" || value === null) continue;
if (typeof value !== "object" || value === null) {
pending.pop();
continue;
}
// The canonical JSON validator reports proxies without invoking their traps.
if (types.isProxy(value)) {
pending.pop();
continue;
}
const array = Array.isArray(value);
if (!array && Object.hasOwn(value, "__proto__"))
return [...pathOf(node), "__proto__"];
const entries = Object.entries(value);
for (let index = entries.length - 1; index >= 0; index -= 1)
pending.push({
value: entries[index]![1],
parent: node,
key: array ? index : entries[index]![0],
});
if (array) {
if (node.index >= value.length) {
pending.pop();
continue;
}
const index = node.index++;
const descriptor = Object.getOwnPropertyDescriptor(value, index);
if (
descriptor?.enumerable !== true ||
!Object.hasOwn(descriptor, "value")
)
continue;
const child: unknown = descriptor.value;
if (typeof child === "object" && child !== null)
pending.push(cursor(child, node, index));
continue;
}
const next = node.entries?.next();
if (next === undefined || next.done) {
pending.pop();
continue;
}
const key = next.value;
const descriptor = Object.getOwnPropertyDescriptor(value, key);
if (descriptor?.enumerable !== true || !Object.hasOwn(descriptor, "value"))
continue;
const child: unknown = descriptor.value;
if (typeof child === "object" && child !== null)
pending.push(cursor(child, node, key));
}
return undefined;
};
@@ -365,9 +426,12 @@ const heldContent = (
export const parseStateDocument = (
state: EnvironmentState,
value: unknown,
owned = false,
): Result<StateContent, AnalysisInputError> => {
const prototype = prototypeMemberPath(value);
const parsed = stateDocumentSchema.safeParse(value);
const parsed = (
owned ? ownedStateDocumentSchema : stateDocumentSchema
).safeParse(value);
const initial = new Map(
state.entities.map((entity) => [entityKey(entity), entity]),
);
@@ -58,9 +58,21 @@ export class EnvironmentStateStore {
this.#install(this.state);
}
/** Validate a `rea-state/v1` document and, only when it is valid, make it the whole state. */
/** Validate a caller-owned document and copy its JSON values before retaining them. */
replace(document: unknown): Result<void, AnalysisInputError> {
const content = parseStateDocument(this.state, document);
return this.#replace(document, false);
}
/** Validate and retain a JSON document created exclusively by the serving boundary. */
replaceOwnedDocument(document: unknown): Result<void, AnalysisInputError> {
return this.#replace(document, true);
}
#replace(
document: unknown,
owned: boolean,
): Result<void, AnalysisInputError> {
const content = parseStateDocument(this.state, document, owned);
if (!content.ok) return content;
// data/get writes members a document may omit, so the state it reports must still be one data/add loads.
if (jsonBytes(this.#snapshotOf(content.value)) > STATE_INTAKE_LIMIT_BYTES)
@@ -1,8 +1,12 @@
import { describe, expect, it } from "vitest";
import { environmentServeSource } from "../../../tests/fixtures/environmentServe.js";
import type { EnvironmentManifestTool } from "./environmentSource.js";
import type {
EnvironmentManifestTool,
RecordedExchange,
} from "./environmentSource.js";
import {
ENVIRONMENT_CATALOG_LIMIT_BYTES,
environmentCard,
environmentToolArguments,
environmentToolDefinitions as prepareDefinitions,
@@ -17,18 +21,16 @@ const environmentToolDefinitions = (
if (!result.ok) throw result.error;
return result.value;
};
const withTools = (tools: readonly EnvironmentManifestTool[]) => ({
...source,
manifest: { ...source.manifest, tools: [...tools] },
});
const environmentToolInputSchema = (tool: EnvironmentManifestTool) => {
const [definition] = environmentToolDefinitions(
{ ...source.manifest, tools: [tool] },
source.evidenceId,
);
const [definition] = environmentToolDefinitions(withTools([tool]));
if (definition === undefined) throw new Error("Missing tool definition");
return definition.inputSchema;
};
const definitions = environmentToolDefinitions(
source.manifest,
source.evidenceId,
);
const definitions = environmentToolDefinitions(source);
const tool = (name: string): EnvironmentManifestTool => {
const found = source.manifest.tools.find((entry) => entry.name === name);
@@ -44,7 +46,7 @@ describe("environmentToolDefinitions", () => {
const users = tool("get_users_by_id");
const definition = definitions.find(({ name }) => name === users.name);
expect(definition?.description).toBe(
`${REPLAY_CATALOG_INSTRUCTIONS} GET ${users.path_template}; replays retained records [${users.record_ordinals.join(", ")}]; evidence ${source.evidenceId}`,
`${REPLAY_CATALOG_INSTRUCTIONS} GET ${users.path_template}; returns a JSON object with fields "id", "uid", "login", "email"; recorded statuses 200; replays retained records [${users.record_ordinals.join(", ")}]; evidence ${source.evidenceId}`,
);
expect(definition?.tool).toBe(users);
});
@@ -108,6 +110,243 @@ describe("environmentToolDefinitions", () => {
});
});
const described = (name: string): string | undefined =>
definitions.find((definition) => definition.name === name)?.description;
/** The users tool citing exactly the given exchanges, with the statuses its manifest recorded. */
const usersAnswering = (
answers: readonly Partial<RecordedExchange>[],
statuses: readonly number[] = [200],
): string | undefined => {
const exchanges = new Map(
answers.map((answer, index): [number, RecordedExchange] => [
index,
{
ordinal: index,
path: "/api/users/2",
query: null,
status: 200,
contentType: "application/json",
bodyText: null,
...answer,
},
]),
);
const users = {
...tool("get_users_by_id"),
record_ordinals: [...exchanges.keys()],
statuses: [...statuses],
};
const [definition] = environmentToolDefinitions({
...withTools([users]),
exchanges,
});
return definition?.description;
};
const returnsOf = (bodyText: string): string | undefined =>
/\{numeric\}; (?<returns>[^;]*); recorded statuses/u.exec(
usersAnswering([{ bodyText }]) ?? "",
)?.groups?.returns;
describe("environment tool response shapes", () => {
it("describes array, enveloped list and object responses one level deep with their statuses", () => {
expect(described("get_folders")).toContain(
'GET /api/folders; returns a JSON array of objects with fields "id", "uid", "title"; recorded statuses 200; replays',
);
expect(described("get_dashboards_versions_by_uid")).toContain(
'; returns a JSON object whose "versions" lists objects with fields "id", "version"; recorded statuses 200; replays',
);
expect(described("get_dashboards_by_uid")).toContain(
'; returns a JSON object with fields "meta", "dashboard"; recorded statuses 200; replays',
);
});
it("describes empty, scalar and fieldless bodies and bounds a long member name", () => {
expect(
[
"[]",
"[1,2]",
'"text"',
"null",
"true",
"3",
"{}",
"[{},{}]",
'{"items":[]}',
].map(returnsOf),
).toEqual([
"returns a JSON array",
"returns a JSON array",
"returns a JSON string",
"returns a JSON null",
"returns a JSON boolean",
"returns a JSON number",
"returns a JSON object with no fields",
"returns a JSON array of objects with no fields",
'returns a JSON object with fields "items"',
]);
const member = "m".repeat(2048);
expect(
usersAnswering([
{ bodyText: JSON.stringify({ [member]: [{ id: 1 }, { id: 2 }] }) },
]),
).toContain(
`{numeric}; returns a JSON object whose "${"m".repeat(1022)}"… (member display truncated; full value in capture Evidence) lists objects with fields "id"; recorded statuses 200;`,
);
});
it("quotes captured names, so a separator in one cannot read as more of the description", () => {
const description = usersAnswering([
{
bodyText: JSON.stringify({
"id, name": 1,
"x; recorded statuses 500": 2,
}),
},
]);
expect(description).toContain(
'returns a JSON object with fields "id, name", "x; recorded statuses 500"; recorded statuses 200;',
);
expect(
usersAnswering([
{ bodyText: JSON.stringify({ "a\nb": [{ id: 1 }, { id: 2 }] }) },
]),
).toContain('whose "a\\nb" lists objects with fields "id";');
// A name too long to show whole is shown as a quoted prefix, so its newline cannot start a line of description.
const long = `items; recorded statuses 500\n${"m".repeat(2048)}`;
const shown = usersAnswering([
{ bodyText: JSON.stringify({ [long]: [{ id: 1 }, { id: 2 }] }) },
]);
expect(shown).toContain('whose "items; recorded statuses 500\\nmmm');
expect(shown).not.toContain("items; recorded statuses 500\n");
});
it("applies the sample budget after JSON escaping", () => {
const member = "\u0000".repeat(1024);
const description = usersAnswering([
{ bodyText: JSON.stringify({ [member]: [{ id: 1 }] }) },
]);
const shown = /whose (?<name>"(?:\\u0000)*")/u.exec(description ?? "")
?.groups?.name;
expect(shown).toBeDefined();
expect(Buffer.byteLength(shown ?? "")).toBeLessThanOrEqual(1024);
expect(description).toContain("member display truncated");
expect(description).not.toContain("\u0000");
});
});
describe("environment tool response summaries", () => {
it("describes the first cited 2xx record replay answers as JSON, then the manifest's statuses", () => {
const description = usersAnswering(
[
{
contentType: "text/html",
bodyText: '{"looks":"like JSON"}',
},
{ status: 401, bodyText: '{"message":"Unauthorized","url":"/login"}' },
{ bodyText: '{"id":2,"login":"probe"}' },
{ bodyText: '[{"id":3,"email":"viewer@example.test"}]' },
],
[200, 401],
);
expect(description).toContain(
'{numeric}; returns a JSON object with fields "id", "login"; recorded statuses 200, 401; replays retained records [0, 1, 2, 3];',
);
});
it("labels a JSON record of another status only when no 2xx record replays as JSON", () => {
expect(
usersAnswering(
[
{ contentType: "text/html", bodyText: "<p>Signed out</p>" },
{ status: 401, bodyText: '{"message":"Unauthorized"}' },
{ status: 403, bodyText: '{"errors":[{"message":"Forbidden"}]}' },
],
[200, 401, 403],
),
).toContain(
'{numeric}; status 401 returns a JSON object with fields "message"; recorded statuses 200, 401, 403; replays',
);
expect(
usersAnswering([{ status: null, bodyText: '{"id":2}' }], []),
).toContain(
'{numeric}; a record without a status returns a JSON object with fields "id"; no recorded status; replays',
);
});
it("states only the statuses when no cited exchange replays as JSON", () => {
for (const answer of [
{ contentType: "text/html", bodyText: "<p>Granite Ledger</p>" },
{ bodyText: '{"id":9007199254740993}' },
{ bodyText: '{"id":2}', bodyEncoding: "base64" as const },
{ bodyText: '{"id":2}', bodyCut: true as const },
{ bodyText: null },
])
expect(usersAnswering([answer])).toMatch(
/\{numeric\}; recorded statuses 200; replays retained records/u,
);
});
it("charges response summaries to the catalog reservation before retaining them", () => {
const fields = Object.fromEntries(
Array.from({ length: 300 }, (_, index) => [
`field_${String(index).padStart(3, "0")}`,
index,
]),
);
const users = tool("get_users_by_id");
const ordinal = users.record_ordinals[0] ?? 0;
const catalog = (contentType: string) =>
prepareDefinitions({
...withTools(
Array.from({ length: 200 }, (_, index) => ({
...users,
name: `get_users_${String(index)}`,
record_ordinals: [ordinal],
})),
),
exchanges: new Map([
[
ordinal,
{
ordinal,
path: "/api/users/2",
query: null,
status: 200,
contentType,
bodyText: JSON.stringify(fields),
},
],
]),
});
expect(catalog("text/html").ok).toBe(true);
const bounded = catalog("application/json");
expect(bounded).toMatchObject({
ok: false,
error: {
_tag: "AnalysisResourceConstraintError",
resource: "capacity",
reportedLimits: { max_catalog_bytes: ENVIRONMENT_CATALOG_LIMIT_BYTES },
},
});
const [single] = environmentToolDefinitions({
...withTools([users]),
exchanges: new Map([
[
ordinal,
{
...source.exchanges.get(ordinal)!,
bodyText: JSON.stringify(fields),
},
],
]),
});
expect(Buffer.byteLength(single?.description ?? "")).toBeLessThan(1700);
expect(single?.description).toContain(
"/300 shown; full list in capture Evidence)",
);
});
});
describe("environment catalog description budgets", () => {
it("bounds captured scalar prose while preserving complete lookup identities", () => {
const query = "query-".repeat(1000);
@@ -117,10 +356,7 @@ describe("environment catalog description budgets", () => {
path_template: `/${"😀".repeat(1000)}`,
query_parameter_names: [query],
};
const [definition] = environmentToolDefinitions(
{ ...source.manifest, tools: [expanded] },
source.evidenceId,
);
const [definition] = environmentToolDefinitions(withTools([expanded]));
expect(definition?.tool).toBe(expanded);
expect(Buffer.byteLength(definition?.description ?? "")).toBeLessThan(3500);
expect(definition?.description).toContain(
@@ -152,10 +388,7 @@ describe("environment catalog description budgets", () => {
})),
record_ordinals: ordinals,
};
const [definition] = environmentToolDefinitions(
{ ...source.manifest, tools: [expanded] },
source.evidenceId,
);
const [definition] = environmentToolDefinitions(withTools([expanded]));
expect(definition?.tool).toBe(expanded);
expect(expanded.params[0]?.observed_values).toBe(observations);
expect(expanded.record_ordinals).toBe(ordinals);
@@ -180,10 +413,7 @@ describe("environment catalog description budgets", () => {
...users,
path_template: `/api/${"segment/".repeat(200_000)}{id}`,
};
const [definition] = environmentToolDefinitions(
{ ...source.manifest, tools: [long] },
source.evidenceId,
);
const [definition] = environmentToolDefinitions(withTools([long]));
expect(Buffer.byteLength(definition?.description ?? "")).toBeLessThan(2048);
expect(definition?.description).toContain(
"path display truncated; full value in manifest Evidence",
@@ -330,3 +560,22 @@ describe("environmentCard", () => {
expect(environmentCard("renamed", definitions).name).toBe("renamed");
});
});
describe("environment tool annotations", () => {
it("marks OPTIONS and TRACE safe and idempotent, as GET is", () => {
const get = tool("get_users_by_id");
const hints = environmentToolDefinitions(
withTools(
(["OPTIONS", "TRACE"] as const).map((method) => ({
...get,
name: `${method.toLowerCase()}_users_by_id`,
method,
})),
),
).map((definition) => definition.annotations);
expect(hints).toEqual([
{ readOnlyHint: true, idempotentHint: true, openWorldHint: false },
{ readOnlyHint: true, idempotentHint: true, openWorldHint: false },
]);
});
});
+175 -22
View File
@@ -1,9 +1,20 @@
import {
isObject,
isQueryPost,
listItems,
locateList,
pointerTokens,
type JsonObject,
} from "../../domain/analysisView/environmentStateRules.js";
import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { ENVIRONMENT_SERVE_OPERATION } from "./environmentSource.js";
import type {
EnvironmentManifest,
EnvironmentManifestTool,
import {
ENVIRONMENT_SERVE_OPERATION,
replayedBody,
type EnvironmentManifestTool,
type EnvironmentSource,
type RecordedExchange,
} from "./environmentSource.js";
/** Methods whose tools accept an optional request body object. */
@@ -27,36 +38,98 @@ export interface EnvironmentToolInputSchema {
readonly additionalProperties: false;
}
/** One manifest tool as an MCP tool: name, description and input schema. */
/** MCP hints for the captured method; answers come from the capture, never the open world. */
export interface EnvironmentToolAnnotations {
readonly readOnlyHint: boolean;
/** Set for `DELETE` only; other writes leave the MCP default (destructive). */
readonly destructiveHint?: true;
readonly idempotentHint: boolean;
readonly openWorldHint: false;
}
/** One manifest tool as an MCP tool: name, description, input schema and the hints MCP advertises; the card omits the hints. */
export interface EnvironmentToolDefinition {
readonly name: string;
readonly description: string;
readonly inputSchema: EnvironmentToolInputSchema;
readonly annotations: EnvironmentToolAnnotations;
readonly tool: EnvironmentManifestTool;
}
/** RFC 9110 safe methods: a preflight `OPTIONS` reads, as `GET` does. */
const SAFE_METHODS: ReadonlySet<string> = new Set([
"GET",
"HEAD",
"OPTIONS",
"TRACE",
]);
const IDEMPOTENT_METHODS: ReadonlySet<string> = new Set([
...SAFE_METHODS,
"PUT",
"DELETE",
]);
const environmentToolAnnotations = ({
method,
path_template: path,
}: EnvironmentManifestTool): EnvironmentToolAnnotations => ({
readOnlyHint: SAFE_METHODS.has(method) || isQueryPost(method, path),
...(method === "DELETE" ? { destructiveHint: true as const } : {}),
idempotentHint: IDEMPOTENT_METHODS.has(method),
openWorldHint: false,
});
/** Agent-context budget for each captured display sample, excluding its provenance notice. */
const DESCRIPTION_SAMPLE_BYTES = 1024;
/** A display prefix is explicitly separate from the complete identity held by the manifest. */
const describeText = (text: string, label: string): string => {
/** Whether a rendered sample fits the display budget. Callers' `show` never shortens its input. */
const sampleFits = (shown: string): boolean =>
shown.length <= DESCRIPTION_SAMPLE_BYTES &&
Buffer.byteLength(shown, "utf8") <= DESCRIPTION_SAMPLE_BYTES;
/**
* A display prefix is explicitly separate from the complete identity held by the Evidence named.
* `show` renders the text or its prefix. The budget applies to that rendering, so JSON quoting of a
* name full of NULs or newlines cannot outgrow the sample.
*/
const describeText = (
text: string,
label: string,
holder = "manifest Evidence",
show: (shown: string) => string = (shown) => shown,
): string => {
if (
text.length <= DESCRIPTION_SAMPLE_BYTES &&
Buffer.byteLength(text, "utf8") <= DESCRIPTION_SAMPLE_BYTES
)
return text;
const { read } = new TextEncoder().encodeInto(
text,
new Uint8Array(DESCRIPTION_SAMPLE_BYTES),
);
return `${text.slice(0, read)}… (${label} display truncated; full value in manifest Evidence)`;
) {
const shown = show(text);
if (sampleFits(shown)) return shown;
}
let low = 0;
let high = Math.min(text.length, DESCRIPTION_SAMPLE_BYTES);
let best = 0;
while (low <= high) {
const mid = (low + high) >> 1;
if (sampleFits(show(text.slice(0, mid)))) {
best = mid;
low = mid + 1;
} else high = mid - 1;
}
// A lone surrogate is not a character. Step off a high surrogate so the prefix stays whole.
if (best > 0 && best < text.length) {
const unit = text.charCodeAt(best - 1);
if (unit >= 0xd8_00 && unit <= 0xdb_ff) best -= 1;
}
return `${show(text.slice(0, best))}… (${label} display truncated; full value in ${holder})`;
};
/** Sample whole values before joining; keep full lists in the manifest that owns them. */
/** Sample whole values before joining; keep full lists in the Evidence that owns them. */
const describeList = <Value>(
values: Iterable<Value>,
count: number,
display: (value: Value) => string = String,
holder = "manifest Evidence",
): string => {
if (count === 0) return "none";
const sample: string[] = [];
@@ -76,7 +149,80 @@ const describeList = <Value>(
const listed = sample.join(", ");
return sample.length === count
? listed
: `${listed}${sample.length === 0 ? "" : ", "}… (${String(sample.length)}/${String(count)} shown; full list in manifest Evidence)`;
: `${listed}${sample.length === 0 ? "" : ", "}… (${String(sample.length)}/${String(count)} shown; full list in ${holder})`;
};
/**
* A captured name as a JSON string, so a separator, quote or newline in it cannot read as more of the
* description. A name too long for a sample is left as it is: the sample refuses it either way.
*/
const quotedName = (name: string): string =>
name.length > DESCRIPTION_SAMPLE_BYTES ? name : JSON.stringify(name);
/** Field names of the objects in first-seen order; the recorded bodies stay in the capture Evidence. */
const withFields = (objects: readonly JsonObject[]): string => {
const names = new Set<string>();
for (const object of objects)
for (const name of Object.keys(object)) names.add(name);
return names.size === 0
? "with no fields"
: `with fields ${describeList(names, names.size, quotedName, "capture Evidence")}`;
};
/** One level of a JSON body; a list is an array of objects found by the state's list rule. */
const describeJson = (value: JsonValue): string => {
const pointer = locateList(value, undefined);
const items = pointer === undefined ? [] : (listItems(value, pointer) ?? []);
if (pointer !== undefined && items.length > 0) {
const [member] = pointerTokens(pointer);
return member === undefined
? `a JSON array of objects ${withFields(items)}`
: `a JSON object whose ${describeText(member, "member", "capture Evidence", JSON.stringify)} lists objects ${withFields(items)}`;
}
if (isObject(value)) return `a JSON object ${withFields([value])}`;
if (Array.isArray(value)) return "a JSON array";
return `a JSON ${value === null ? "null" : typeof value}`;
};
const isSuccess = (status: number | null): boolean =>
status !== null && status >= 200 && status < 300;
/** The first cited record replay answers as JSON, preferring a 2xx one; each body is parsed at most once. */
const firstJson = (
tool: EnvironmentManifestTool,
exchanges: ReadonlyMap<number, RecordedExchange>,
):
| { readonly status: number | null; readonly value: JsonValue }
| undefined => {
for (const success of [true, false])
for (const ordinal of tool.record_ordinals) {
const exchange = exchanges.get(ordinal);
if (exchange === undefined || isSuccess(exchange.status) !== success)
continue;
const body = replayedBody(exchange);
if (body.kind === "json")
return { status: exchange.status, value: body.value };
}
return undefined;
};
/** What the tool returns as JSON, labelled with the status unless it succeeded, then the manifest's recorded statuses. */
const describeResponses = (
tool: EnvironmentManifestTool,
exchanges: ReadonlyMap<number, RecordedExchange>,
): string => {
const recorded =
tool.statuses.length === 0
? "no recorded status"
: `recorded statuses ${describeList(tool.statuses, tool.statuses.length)}`;
const found = firstJson(tool, exchanges);
if (found === undefined) return recorded;
const returns = isSuccess(found.status)
? "returns"
: found.status === null
? "a record without a status returns"
: `status ${String(found.status)} returns`;
return `${returns} ${describeJson(found.value)}; ${recorded}`;
};
/** Captured argument names longer than this are advertised under a short alias; the manifest keeps the name. */
@@ -208,9 +354,8 @@ export interface EnvironmentCatalogOptions {
/** Admit complete discovery before binding; six bytes per UTF-16 unit covers JSON escaping and UTF-8 expansion. */
export const environmentToolDefinitions = (
manifest: EnvironmentManifest,
evidenceId: string,
name: string = manifest.name,
source: EnvironmentSource,
name: string = source.manifest.name,
{
operation = ENVIRONMENT_SERVE_OPERATION,
dataPlane = "none",
@@ -254,15 +399,22 @@ export const environmentToolDefinitions = (
? "answers from the served environment state where the state backs this tool, otherwise replays retained records"
: "replays retained records";
const definitions: EnvironmentToolDefinition[] = [];
for (const tool of manifest.tools) {
if (!reserve(EMPTY_TOOL_BYTES) || !text(tool.name)) return refused();
for (const tool of source.manifest.tools) {
const annotations = environmentToolAnnotations(tool);
// Only MCP discovery carries the hints; charging them per tool over-charges the card harmlessly.
if (
!reserve(EMPTY_TOOL_BYTES + JSON.stringify({ annotations }).length) ||
!text(tool.name)
)
return refused();
const count =
tool.params.length +
tool.query_parameter_names.length +
(WRITE_METHODS.has(tool.method) ? 1 : 0);
// A lower bound rejects enormous argument lists before constructing their alias map.
if (!reserve(count * EMPTY_PROPERTY_BYTES)) return refused();
const description = `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${evidenceId}`;
// The response summary is bounded by its samples and charged with the whole description.
const description = `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${describeResponses(tool, source.exchanges)}; ${answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${source.evidenceId}`;
if (!text(description)) return refused();
const args = environmentToolArguments(tool);
const properties = new Map<string, EnvironmentToolProperty>();
@@ -289,6 +441,7 @@ export const environmentToolDefinitions = (
required,
additionalProperties: false,
},
annotations,
tool,
});
}
@@ -1,11 +1,9 @@
import { isAbsolute } from "node:path";
import { z } from "zod";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { EvmInterfacePort } from "./EvmInterfacePort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import type { Evidence } from "../../domain/evidence.js";
@@ -47,16 +45,6 @@ export class EvmInterfaceService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const inspected = await this.provider.inspect(input.data, options);
if (!inspected.ok) return inspected;
const diagnostics = diagnosticsSchema.safeParse(inspected.value.result);
@@ -8,6 +8,7 @@ import {
} from "../../domain/firmware/firmwareAnalysis.js";
import { AnalysisOutputError } from "../../domain/analysisErrorCore.js";
import { err, ok } from "../../domain/result.js";
import { createAnalysisProfile } from "../../domain/analysisProfile.js";
const fixture = (
operation: FirmwareOperation,
@@ -68,15 +69,35 @@ it.each(["inspect_firmware_regions", "extract_firmware"] as const)(
"retains %s Evidence after late cancellation and preserves provider failure precedence",
async (operation) => {
const { input, execution } = fixture(operation);
const profile = createAnalysisProfile(
{ ...execution.provider, version: "1" },
{
signature_database: "caller-selected snapshot",
},
);
const coverage = {
status: "unknown",
reason: "No call graph examined",
} as const;
const observation = {
...execution,
analysisProfile: profile,
relationshipCoverage: coverage,
};
const control = await new FirmwareAnalysisService({
execute: async () => ok(execution),
execute: async () => ok(observation),
}).execute(operation, input);
if (!control.ok) throw control.error;
expect(control.value).toMatchObject({
raw_result: { source: "controlled producer" },
analysis_profile: profile,
relationship_coverage: coverage,
});
const controller = new AbortController();
const result = await new FirmwareAnalysisService({
execute: async () => {
controller.abort();
return ok(execution);
return ok(observation);
},
}).execute(operation, input, { signal: controller.signal });
expect(result).toMatchObject({
@@ -1,4 +1,7 @@
import type { ExecutionOptions } from "../AnalysisProvider.js";
import {
createAnalysisExecutionEvidence,
type ExecutionOptions,
} from "../AnalysisProvider.js";
import type { FirmwareAnalysisPort } from "./FirmwareAnalysisPort.js";
import {
firmwareRequestSchema,
@@ -12,8 +15,7 @@ import {
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
import { jsonObjectSchema } from "../../domain/jsonValue.js";
import type { Evidence } from "../../domain/evidence.js";
import { projectInputIssues } from "../../domain/inputIssueProjection.js";
import { err, type Result } from "../../domain/result.js";
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
@@ -71,15 +73,12 @@ export class FirmwareAnalysisService {
),
);
return completeProviderEvidence(
createEvidence(execution.subject, execution.provider, {
createAnalysisExecutionEvidence(
operation,
parameters: jsonObjectSchema.parse(parsed.data.input),
result: execution.result,
rawResult: execution.rawResult,
locations: execution.locations,
limitations: execution.limitations,
confidence: "observed",
}),
parsed.data.input,
execution,
"observed",
),
options?.signal,
);
}
-12
View File
@@ -1,10 +1,8 @@
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { GoBinaryPort } from "./GoBinaryPort.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -53,16 +51,6 @@ export class GoBinaryService {
cause: input.error,
}),
);
if (!isAbsolute(input.data.path))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: ["path"],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
const { detail, ...providerInput } = input.data;
if (detail === "summary") {
const ready = requireAnalysisViewRetention(OPERATION, retention);
@@ -1,19 +1,21 @@
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { Evidence } from "../../domain/evidence.js";
import {
harmonyApplicationProjectionInputSchema,
projectHarmonyApplication,
} from "../../domain/harmony/harmonyApplication.js";
import type { Result } from "../../domain/result.js";
import { HARMONY_APPLICATION_PROVIDER } from "../InvestigationProviders.js";
import { projectInventoryEvidence } from "../InventoryProjectionEvidence.js";
import {
projectInventoryEvidence,
type InventoryProjectionEvidenceResult,
} from "../InventoryProjectionEvidence.js";
const OPERATION = "project_harmony_application_graph" as const;
/** Project authenticated HarmonyOS package inventory Evidence into application evidence. */
export const projectHarmonyApplicationEvidence = (
rawInput: unknown,
): Result<Evidence, AnalysisError> => {
): Result<InventoryProjectionEvidenceResult, AnalysisError> => {
return projectInventoryEvidence({
rawInput,
schema: harmonyApplicationProjectionInputSchema,
@@ -5,7 +5,6 @@ import type {
Il2cppSeedWriterPort,
Il2cppSeedReceipt,
} from "./Il2cppSeedWriterPort.js";
import { isAbsolute } from "node:path";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import {
requireAnalysisViewRetention,
@@ -111,19 +110,6 @@ export class Il2cppMetadataService {
cause: input.error,
}),
);
for (const field of ["path", "binary_path"] as const) {
const value = input.data[field];
if (value !== undefined && !isAbsolute(value))
return err(
new AnalysisInputError(OPERATION, undefined, [
{
path: [field],
reason: "invalid_format",
message: "Expected an absolute filesystem path on this host.",
},
]),
);
}
if (
input.data.binary_architecture !== undefined &&
input.data.binary_path === undefined
@@ -56,6 +56,9 @@ const parameters = (
target_id: input.target_id,
observation_ms: input.observation_ms,
include_script_sources: input.include_script_sources,
...(input.command_timeout_ms === undefined
? {}
: { command_timeout_ms: input.command_timeout_ms }),
}
: {}),
});
@@ -184,15 +184,12 @@ const staticRuntimeSource = (
evidence.normalized_result,
[...path, "normalized_result"],
);
const unmatchedLink = result.evidence_links.find(
(evidenceId) => !evidence.evidence_links.includes(evidenceId),
assertResultEvidenceLinksInEnvelope(
result.evidence_links,
evidence,
path,
"Runtime reconciliation result",
);
if (unmatchedLink !== undefined)
throw invalidEvidence(
[...path, "normalized_result"],
"evidence_links",
`Runtime reconciliation result references Evidence ${unmatchedLink} outside its envelope.`,
);
const applicationLayer = result.static_layers.find(
({ role }) => role === "application",
);
@@ -229,15 +226,12 @@ const managedApplicationSource = (
evidence.normalized_result,
[...path, "normalized_result"],
);
const unmatchedLink = result.evidence_links.find(
(evidenceId) => !evidence.evidence_links.includes(evidenceId),
assertResultEvidenceLinksInEnvelope(
result.evidence_links,
evidence,
path,
"Managed application graph result",
);
if (unmatchedLink !== undefined)
throw invalidEvidence(
[...path, "normalized_result"],
"evidence_links",
`Managed application graph result references Evidence ${unmatchedLink} outside its envelope.`,
);
return {
evidence,
graph: result.graph,
@@ -247,6 +241,25 @@ const managedApplicationSource = (
};
};
const assertResultEvidenceLinksInEnvelope = (
resultEvidenceLinks: readonly string[],
evidence: Evidence,
path: readonly (string | number)[],
resultLabel:
| "Runtime reconciliation result"
| "Managed application graph result",
): void => {
const unmatchedLink = resultEvidenceLinks.find(
(evidenceId) => !evidence.evidence_links.includes(evidenceId),
);
if (unmatchedLink !== undefined)
throw invalidEvidence(
[...path, "normalized_result"],
"evidence_links",
`${resultLabel} references Evidence ${unmatchedLink} outside its envelope.`,
);
};
const assertApplicationSubject = (
evidence: Evidence,
result: JavaScriptApplicationAnalysisResult,
@@ -188,6 +188,7 @@ const artifactFailureToOperationError = (
...(cause.partialObservation === undefined
? {}
: { partialObservation: cause.partialObservation }),
...(cause.admission === undefined ? {} : { admission: cause.admission }),
},
);
@@ -37,16 +37,13 @@ export const resolveSourceToBundleReference = async (
}),
);
if (!("kind" in reference)) return ok({ reference });
const read = await readJsonFileWithSource(reference.path, signal, resources);
if (!read.ok) {
return read.error instanceof AnalysisCancelledError
? err(
new AnalysisCancelledError("compare_source_to_bundle", {
cause: read.error.cause,
}),
)
: read;
}
const read = await readJsonFileWithSource(
reference.path,
signal,
resources,
"compare_source_to_bundle",
);
if (!read.ok) return read;
await setImmediate();
if (cancelled())
return err(
@@ -25,7 +25,7 @@ import {
type EvidenceLookup,
} from "../EvidenceInputResolver.js";
import type { ExecutionOptions } from "../AnalysisProvider.js";
import { FUNCTION_COMPARISON_PROVIDER } from "../InvestigationProviders.js";
import { FUNCTION_COMPARISON_PROVIDER } from "../../domain/comparisonEvidenceIdentity.js";
const OPERATION = "match_native_functions";
export interface NativeFunctionMatchingResult {
@@ -83,6 +83,7 @@ export const loadNativeMatchingBundleSources = async ({
paths[file]!,
admission.signal,
resources,
admission.operation,
);
if (!loaded.ok) throw loaded.error;
snapshots.push(await recordSnapshot(loaded.value, admission));
@@ -141,6 +142,7 @@ export const loadNativeMatchingBundleSources = async ({
paths[file]!,
admission.signal,
resources,
admission.operation,
);
if (!loaded.ok) throw loaded.error;
if ((await recordSnapshot(loaded.value, admission)) !== snapshots[file])
+36 -47
View File
@@ -9,7 +9,7 @@ import {
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import type { AnalysisErrorProjection } from "../../domain/analysisErrorProjection.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { createEvidence, parseEvidence } from "../../domain/evidence.js";
import { createEvidence } from "../../domain/evidence.js";
import { describeValidationFailure } from "../../domain/evidenceBundle.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import {
@@ -19,16 +19,15 @@ import {
import { processTraceSpecificationSchema } from "../../domain/process/processTraceSpecification.js";
import { processScenarioSchema } from "../../domain/process/processScenario.js";
import { compareProcessCaptures } from "../../domain/process/processComparison.js";
import { parseProcessCapture } from "../../domain/process/processCaptureParsing.js";
import { createProgressReporter } from "../ProgressReporter.js";
import { captureProcessScenario } from "../../process/capture/ProcessHarness.js";
import { ProcessCaptureResourceScope } from "../../process/capture/ProcessCaptureLifecycle.js";
import { ProcessCaptureError } from "../../process/capture/ProcessCaptureError.js";
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
import {
PROCESS_PROVIDER,
isProcessEvidenceProvider,
} from "../../domain/process/processEvidenceProvider.js";
import { createProcessCaptureEvidence } from "./ProcessEvidence.js";
createProcessCaptureEvidence,
parseProcessCaptureEvidence,
} from "./ProcessEvidence.js";
import { readJsonFile } from "../JsonFiles.js";
import { RegularFileCleanupFailure } from "../RegularFileRead.js";
@@ -147,8 +146,8 @@ export const compareProcessEvidenceFiles = async (
predicateType: "rea.process-comparison",
operation: "compare_process_captures",
parameters: {
left_evidence_id: left.id,
right_evidence_id: right.id,
left_evidence_id: left.record.evidence_id,
right_evidence_id: right.record.evidence_id,
left_normalization: left.capture.normalization,
right_normalization: right.capture.normalization,
...(traceSpecification === undefined
@@ -159,8 +158,8 @@ export const compareProcessEvidenceFiles = async (
confidence: "derived",
authority: "analyst-inference",
limitations: comparison.limitations,
locations: [...left.locations, ...right.locations],
evidenceLinks: [left.id, right.id],
locations: [...left.record.locations, ...right.record.locations],
evidenceLinks: [left.record.evidence_id, right.record.evidence_id],
});
} catch (cause: unknown) {
if (cause instanceof RegularFileCleanupFailure) throw cause;
@@ -169,45 +168,35 @@ export const compareProcessEvidenceFiles = async (
};
const parseCaptureEvidence = (input: unknown, side: "left" | "right") => {
let evidence;
try {
evidence = parseEvidence(input);
} catch (cause: unknown) {
if (cause instanceof z.ZodError)
throw new AnalysisInputError(
const parsed = parseProcessCaptureEvidence(input);
if (parsed.ok) return parsed.value;
const failure = parsed.error;
switch (failure.kind) {
case "invalid-evidence":
if (failure.cause instanceof z.ZodError)
throw new AnalysisInputError(
"compare_process_captures",
{ cause: failure.cause },
projectInputIssues(failure.cause.issues, input),
);
throw invalidCaptureEvidence(failure.cause, side);
case "wrong-workflow":
throw processInputFailure(
"compare_process_captures",
{ cause },
projectInputIssues(cause.issues, input),
"Capture evidence is not from the current process-capture workflow. Create new capture evidence, then try again.",
[side],
"invalid_value",
);
throw invalidCaptureEvidence(cause, side);
}
if (
evidence.operation !== "capture_process_scenario" ||
evidence.predicate_type !== "rea.process-capture"
) {
throw processInputFailure(
"compare_process_captures",
`Capture evidence is not from the current process-capture workflow. Create new capture evidence, then try again.`,
[side],
"invalid_value",
);
}
if (!isProcessEvidenceProvider(evidence.provider))
throw new AnalysisInputError("compare_process_captures", undefined, [
{
path: [side, "provider"],
reason: "invalid_value",
message: `The ${side} Evidence provider ${evidence.provider.id} v${evidence.provider.version} (${evidence.provider.name}) is not the current or supported legacy ${PROCESS_PROVIDER.id} v${PROCESS_PROVIDER.version} process capture provider.`,
},
]);
try {
return {
id: evidence.evidence_id,
capture: parseProcessCapture(evidence.normalized_result),
locations: evidence.locations,
};
} catch (cause: unknown) {
throw invalidCaptureEvidence(cause, side);
case "unsupported-provider":
throw new AnalysisInputError("compare_process_captures", undefined, [
{
path: [side, "provider"],
reason: "invalid_value",
message: `The ${side} Evidence provider ${failure.provider.id} v${failure.provider.version} (${failure.provider.name}) is not the current or supported legacy ${PROCESS_PROVIDER.id} v${PROCESS_PROVIDER.version} process capture provider.`,
},
]);
case "invalid-capture":
throw invalidCaptureEvidence(failure.cause, side);
}
};
Loaded 100 of 441 files, more files were not shown because too many files have changed in this diff. Show more