diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index f40d3113ad..82645590a9 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -9,7 +9,10 @@ provenance; inputs and recovered code are never executed. Recovery requires a caller-supplied Wakaru on Linux x64; see [the recovery guide](https://github.com/morluto/rea/blob/main/docs/javascript-recovery.md). Use `analyze_javascript_application` directly on the operator-supplied ASAR or -extracted tree. When size is unknown or large and the connected schema advertises +extracted tree. Exact `#alias` specifiers resolve through the importer's +enclosing package `imports` map to package-local files; nested `node_modules` +packages use their own map. Wildcard, recursive, and external-package import +targets stay unresolved with that limitation. When size is unknown or large and the connected schema advertises `detail`, request `detail: "summary"`. This returns an `inspect_analysis_view` Evidence record with a distinct ID and a `normalized_result.parent_evidence_id` for the complete retained application analysis. Source-map contents are part of diff --git a/README.md b/README.md index 9828ceaa40..e4a0cc28ba 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ **See a feature you like. Understand how it works, down to the binary level.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#what-you-can-analyze) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) @@ -245,7 +244,9 @@ Ghidra and IDA use your existing installations. See [provider setup](docs/instal Do I need to start Hopper first? REA starts Hopper when an operation needs it. On macOS, a first-run dialog may -ask you to choose demo mode or activate your license. See +ask you to choose demo mode or activate your license. Complete that prompt in the +same user session before your first CLI/MCP operation or unattended job. Open +Hopper manually if needed. See [Hopper startup and troubleshooting](docs/installation.md#launcher-paths-and-troubleshooting). diff --git a/README_ar.md b/README_ar.md index f7f143d209..05f698afc0 100644 --- a/README_ar.md +++ b/README_ar.md @@ -9,7 +9,6 @@ **وجدت ميزة تعجبك؟ افهم طريقة عملها حتى مستوى الشيفرة الثنائية.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_de.md b/README_de.md index f84ecea8b6..a4c12062ff 100644 --- a/README_de.md +++ b/README_de.md @@ -9,7 +9,6 @@ **Eine Funktion gefällt dir? Verstehe, wie sie funktioniert – bis hinunter auf die Binärebene.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_es.md b/README_es.md index 9e71abdd59..228ac8cac9 100644 --- a/README_es.md +++ b/README_es.md @@ -9,7 +9,6 @@ **¿Ves una función que te gusta? Entiende cómo funciona, hasta el nivel binario.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_fa.md b/README_fa.md index 03f1130139..2e0fef7930 100644 --- a/README_fa.md +++ b/README_fa.md @@ -9,7 +9,6 @@ **قابلیتی دیده‌اید که دوستش دارید؟ سازوکارش را تا سطح باینری درک کنید.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_fr.md b/README_fr.md index f3b0de61c5..06a8dba76c 100644 --- a/README_fr.md +++ b/README_fr.md @@ -9,7 +9,6 @@ **Repérez une fonctionnalité qui vous plaît. Comprenez son fonctionnement, jusqu'au niveau du binaire.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_id.md b/README_id.md index d992e99080..c2c372473a 100644 --- a/README_id.md +++ b/README_id.md @@ -9,7 +9,6 @@ **Melihat fitur yang Anda sukai. Pahami cara kerjanya hingga ke tingkat biner.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_it.md b/README_it.md index b9bbeb5a9d..dd3ee065e6 100644 --- a/README_it.md +++ b/README_it.md @@ -9,7 +9,6 @@ **Vedi una funzionalità che ti piace. Capisci come funziona, fino al livello binario.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#cosa-puoi-analizzare) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_ja.md b/README_ja.md index c9690cc7ff..501a844efc 100644 --- a/README_ja.md +++ b/README_ja.md @@ -9,7 +9,6 @@ **気に入った機能を見つけたら、その仕組みをバイナリのレベルまで理解する。** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_ko.md b/README_ko.md index 05b404e101..a79b04acc0 100644 --- a/README_ko.md +++ b/README_ko.md @@ -9,7 +9,6 @@ **마음에 드는 기능을 발견했다면, 바이너리 수준까지 작동 원리를 이해하세요.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_pl.md b/README_pl.md index 181e00eb08..798afb78c4 100644 --- a/README_pl.md +++ b/README_pl.md @@ -9,7 +9,6 @@ **Widzisz funkcję, która Ci się podoba? Dowiedz się, jak działa, aż do poziomu kodu binarnego.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_pt-BR.md b/README_pt-BR.md index 92e0c29c4b..2f0502eaaf 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -9,7 +9,6 @@ **Viu uma funcionalidade de que gostou? Entenda como ela funciona, até o nível binário.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_ru.md b/README_ru.md index 1de8a9d852..d1058f139b 100644 --- a/README_ru.md +++ b/README_ru.md @@ -9,7 +9,6 @@ **Нашли интересную функцию? Разберитесь, как она работает, вплоть до двоичного кода.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_th.md b/README_th.md index 96f9a5fa9e..55a1f8ab9a 100644 --- a/README_th.md +++ b/README_th.md @@ -9,7 +9,6 @@ **เห็นฟีเจอร์ที่ชอบแล้วอยากรู้ว่าทำงานอย่างไร? เข้าใจได้ลึกถึงระดับไบนารี** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_tr.md b/README_tr.md index 9b2c070373..fbc7799e4b 100644 --- a/README_tr.md +++ b/README_tr.md @@ -9,7 +9,6 @@ **Beğendiğiniz bir özellik mi gördünüz? İkili kod düzeyine kadar nasıl çalıştığını anlayın.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_uk.md b/README_uk.md index 6163011dc8..806980a7f3 100644 --- a/README_uk.md +++ b/README_uk.md @@ -9,7 +9,6 @@ **Побачили цікаву функцію? Зрозумійте, як вона працює, аж до рівня двійкового коду.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_vi.md b/README_vi.md index 51db95e443..ec7a6e0ac2 100644 --- a/README_vi.md +++ b/README_vi.md @@ -9,7 +9,6 @@ **Thấy một tính năng bạn thích? Hiểu cách nó hoạt động, đến tận mức mã nhị phân.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_zh-TW.md b/README_zh-TW.md index e0950b2188..5f4452597d 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -9,7 +9,6 @@ **看到喜歡的功能,弄清它的工作原理,深入到二進位層面。** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/README_zh.md b/README_zh.md index fe387a2af8..f0c765c294 100644 --- a/README_zh.md +++ b/README_zh.md @@ -9,7 +9,6 @@ **看到喜欢的功能,弄清它的工作原理,深入到二进制层面。** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) -[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) [![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js requirements](https://img.shields.io/badge/Node.js-requirements-339933?style=flat-square&logo=nodedotjs&logoColor=white)](#current-status) [![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) diff --git a/docs/installation.md b/docs/installation.md index aed4ac7c56..914dcc4b44 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -352,6 +352,12 @@ Other integrations, including Qwen Code, use `~/.agents/skills`. A mixed selection plans each distinct destination. Selecting the skill without a client uses the shared directory. Setup leaves existing skill copies in other locations untouched. +`rea setup --skill-client claude_code --yes --json` refreshes and verifies the +selected skill destination independently of MCP registration selection. The +final `doctor.scope.skill_clients` records which skill destinations were +checked; `doctor.scope.clients` continues to describe required registrations. +An unselected shared skill does not affect a client-only refresh. + `doctor --skill --json` verifies the selected copies against the bundled instructions and references. Consumers should use `identity.skill.state`, `installed_version`, and `installed_tool_count` for skill readiness. The @@ -561,8 +567,13 @@ need Xvfb, Python 3, X11 and XTEST; approved setup installs those dependencies on supported distributions. Add the curl installer's reported executable directory, commonly `~/.local/bin`, to your shell `PATH` if needed. -REA starts Hopper when needed. On macOS, its launcher can bring a window or -first-run dialog forward; choose demo mode or activate your existing license. +REA starts Hopper when needed. On macOS, complete its demo or license prompt in +the same user session before the first CLI/MCP operation or unattended job. Open +Hopper manually if needed; its launcher can also bring the dialog forward. +If startup reports `Cannot launch Hopper! Connection is invalid.` or waits for +bridge readiness, check Hopper for a pending demo, license, or loader dialog. +Complete the dialog and retry. If REA reports `cleanup_incomplete`, follow the +retained-resource guidance below before reopening the target. Hopper serializes analysis requests. Cancelling a wait can leave provider work running, which the session reports. Successful decompilation is cached until a relevant rename or comment changes it. @@ -591,6 +602,11 @@ if document closure is unconfirmed, REA retains the backing image and reports MCP-exit behavior. Startup deadlines report missing bridge readiness and preserve the launcher outcome; a successful helper exit does not prove that a loader dialog completed. +If an owned provider process exits, request failures preserve its observed exit +code or signal and bounded output in `details.launcher`. An exit can precede +output-stream closure; `output_closed: false` keeps that uncertainty explicit. +Later requests retain the known exit instead of reducing it to socket loss. +An external GUI's helper exit does not establish that the GUI itself exited. Cursor navigation returns the observed object start when Hopper snaps an interior address; adjacent-object navigation rejects unmapped inputs and document ends. Native API text rejects NUL characters and unpaired Unicode surrogates before diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 82e2cef99a..0023b7e665 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -231,6 +231,11 @@ unknown instead of silently treating them as absent or verified. Plain `.ts` artifact sources use TypeScript syntax without JSX, including angle-bracket type assertions. `.tsx` and `.jsx` retain JSX parsing. +Relative TypeScript imports can name their emitted `.js`, `.mjs`, or `.cjs` +counterparts while resolving to inventoried source files. This substitution +requires import or require semantics. Literal Electron preload, renderer, and +utility-process runtime paths retain the runtime filename selected by the caller. + Selected bounded text is then parsed as inert data to recover: - `package.json` metadata and declared main or renderer entrypoints; @@ -323,15 +328,44 @@ substitute version-dependent Electron defaults for omitted `webPreferences`. Dynamic option objects, bridge keys, API objects, and IPC channel expressions remain unknown and make coverage partial. -Electron API identity follows proven lexical module aliases. Bundled CommonJS -namespace imports are also followed through the interop helpers that bundlers -and compilers emit (`__toESM`, `__importStar`, `_interopRequireWildcard`), -including the `electron = __toESM(electron)` rewrap that Rolldown emits. This -step is inferred: a helper is recognized by its emitted name and a single -program-scope function definition, and its body is not evaluated, so a local -function that reuses one of these names is trusted to pass the module through. -Other reassigned bindings, dynamic selections, and visibly overwritten namespace -members are unresolved. +Electron API identity follows proven lexical module aliases. Namespace interop +also preserves identity when the complete Rolldown helper operations, lexical +roles, live getters, and captured intrinsic origins match, or when a stable +helper simply returns its module parameter. Harmless single-statement control +blocks preserve recognition; added operations still require proof. Literal +member and destructuring projections are retained from proven results. +Omitted or literal node-mode arguments preserve named exports; a module-valued +default projection requires a truthy literal node mode. Computed or extra +arguments remain unresolved. +Recognition uses structure rather than emitted names; other helper bodies and ambiguous +runtime fallbacks remain unresolved. Captured function identities survive later global-property +replacement, including copies through initialized aliases. Reads before an +alias initializer remain unresolved. Visible intrinsic writes before capture or +conversion are checked in their owning callable scope, including identifier leaves in nested/default/rest +destructuring and loop targets. Property keys and shadowed local bindings do not +overwrite the global intrinsic identity. Cross-call side effects, including +writes inside directly invoked local functions, are not evaluated. Other +reassignments, dynamic selections, and visibly overwritten namespace members +are unresolved. +Known Object/Reflect mutators remain recognizable through literal destructuring +of intrinsic objects and stable bound-function aliases. Calls retain bound +argument prefixes and the original bound receiver through `call`, `apply`, and +`Reflect.apply`, including generic `Function.prototype` invocation and binding +creation. Arguments follow each actual callable invocation rather than property +chain depth; rebinding retains the original bound receiver. +Visibly replaced invocation intrinsics produce unknown effects instead +of using their former call semantics; captured function aliases retain their +earlier identity. Unexamined targets remain unresolved. `Reflect.set` examines +its explicit receiver as well as its target, retaining unknown receiver effects. +Creating an unused binding does not itself mutate an intrinsic. Variable +redeclarations retain a single unconditional initializer +when they introduce no assignment or second initializer. +Stable identifier chains to fresh object, array, function, +undecorated class or RegExp syntax keep those targets separate from intrinsic +objects when initialized before use; opaque, decorated, reassigned or +not-yet-initialized targets remain unresolved. Writes during class initialization +are still retained. Function declarations are available before their textual +position; variable and class initializers must precede use. Direct and destructuring assignment, loop, update, and delete targets are checked through namespace aliases; unrelated member writes do not discard observed API identity. This static check does @@ -425,16 +459,40 @@ such as `./web3:app.js` still names a local artifact. Package `exports` fallback arrays are supported both at the top level and under the root `"."` entry. The resolver selects targets in declared order using the same conditional and invalid-entry handling as nested exports arrays. +Nonempty arrays containing only inactive conditions remain unmatched, allowing +the enclosing condition object to continue to a later active condition. +Empty arrays and explicit null targets keep their blocking outcome. +Refusal evidence distinguishes those source values and retains condition names +from unmatched nested targets rather than reporting array indices. For imports and requires, the active Node conditions include `module-sync`; earlier active conditions retain precedence over later ones. +Exact package-private `#alias` specifiers resolve against the importer's +enclosing `package.json` `imports` map — the nearest same-container package +scope, stopping at a `node_modules` directory so a nested package cannot inherit +an ancestor's aliases. Selected targets are package-local `./` files with no +extension or index fallback, using the same condition, array, and null selection +as exports subpaths. A missing key, explicit null, or out-of-scope alias keeps +an unresolved edge with that reason. Wildcard keys (`#internal/*`), recursive +`#` targets, and external-package import targets remain unresolved. + +Exact keys such as `#/x` are literal lookup identities. `#` and names ending +in `/` are invalid. Imports arrays skip invalid parent-relative, absolute, and +URL targets before selecting a later package-local file; an unsupported bare +package or recursive alias keeps its unresolved outcome rather than selecting +a later fallback. + ESM relative module paths and selected package exports targets use URL suffix handling and one percent-decoding pass. CommonJS relative paths and legacy package main fields retain literal filename punctuation. Exports targets must start with `./` and exclude `.`, `..`, and `node_modules` path segments, including their encoded forms, before URL normalization. Arrays skip invalid exports targets -in declared order; URL decoding and file lookup failures for a selected target -do not select a later entry. Rejected references retain the package metadata +in declared order, including URLs that escape the package root after normalization. +Selected package targets use the normalized URL path for lookup; tabs and +newlines in the URL text are removed by the URL parser. CommonJS package targets +reject encoded separators anywhere in the selected URL; ESM checks its pathname. +URL decoding and file lookup failures for a selected target do not select a +later entry. Rejected references retain the package metadata path, original target, failed constraint, and importing source location. HTML script references resolve to exact inventoried files after applying the diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 6157e7b03f..ad642b7a51 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -126,6 +126,24 @@ Member inspection binds build-local tokens, signatures, row offsets, method bodies, and decoded relationships to that artifact and MVID. Missing or malformed facts remain explicit in coverage and per-member status. +Metadata table streams may carry the CLR-compatible `ExtraData` extension +(`HeapSizes` flag `0x40`), which Microsoft readers honor although it is not +specified by ECMA-335. REA skips its fixed four-byte word after the row counts +before decoding table rows; the word's value is not a length. An incomplete +word is reported as a metadata coverage failure at its file offset, without +reading beyond the declared table stream. + +For unoptimized `#-` metadata, nonempty `FieldPtr` and `MethodPtr` tables +translate TypeDef list positions to physical member definitions. REA resolves +these indirections for member ownership and local attribute-constructor types; +field and method tokens still identify the original definition rows. +`field_list` and `method_list` retain the metadata's list coordinates, which +refer to the pointer table when present rather than necessarily contiguous +definition rows. Malformed or ambiguous pointers leave the owner unknown with +partial coverage while preserving readable member and attribute facts. +The retained inverse indexes are admitted against the existing representation +budget before allocation. + `exported_types` reports ExportedType rows as declared. Facade assemblies such as .NET `System.Runtime.dll` provide their types this way: `forwarder: true` marks a type forwarder, and `implementation_token` names the AssemblyRef, File, diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index c58db195f5..427189c0c9 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -450,7 +450,13 @@ Tool annotations describe effects and are hints, not authorization controls ([MCP ToolAnnotations](https://modelcontextprotocol.io/specification/2025-06-18/schema#toolannotations)). `readOnlyHint` includes session state: an analysis call that records additive Evidence is marked non-read-only even when it leaves the target unchanged. -`destructiveHint` describes possible data loss, not ordinary Evidence recording. +`destructiveHint` is true when a tool may discard data (overwriting or clearing +analyst metadata, replacing files, dropping session state), runs or drives code +REA does not own (`executesForeignCode`: process, LLDB, Electron, browser, and +native UI scenarios), or changes an attached Android device (`mutatesDevice`: +push, install, uninstall, start, and stop). Ordinary Evidence recording is not +destructive. The effect record in `binary_session` and the generated catalog +carries each flag separately. Effect metadata covers possible behavior across supported inputs: DMG inventory can launch `hdiutil` and create an owned temporary mount directory; extraction creates a fresh output directory on every call. Comparing supplied web captures diff --git a/docs/testing.md b/docs/testing.md index e8a02713a8..1473bbf350 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -139,8 +139,11 @@ directory, so default OS-home discovery remains unverified. Live model APIs and native Windows are also unverified. `verify:claude-client` requires installed Claude Code (verified with -`@anthropic-ai/claude-code@2.1.296`); select it with `REA_VERIFY_CLAUDE_COMMAND`. -The optional POSIX lane uses native `CLAUDE_CONFIG_DIR` discovery, an independent +`@anthropic-ai/claude-code@2.1.296` on POSIX and native Windows); select it +with `REA_VERIFY_CLAUDE_COMMAND`. On Windows, select +`claude.exe`: npm installs put only a `claude.cmd` shim on `PATH`, and the real +executable is under `node_modules/@anthropic-ai/claude-code/bin`. +The optional lane uses native `CLAUDE_CONFIG_DIR` discovery, an independent Git workspace, user settings and default built-in tools. It preserves caller preferences and backups, checks idempotent setup, waits through the native `WaitForMcpServers` tool when discovery is pending, and validates the complete @@ -149,8 +152,8 @@ native `Skill`, checks its full body and verifies named-schema JavaScript Eviden for a Unicode path. Use `-- chat` for ordinary chat with REA enabled, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package. The local Anthropic Messages/SSE fixture preserves native resource-hint envelopes in its -request artifacts. Usage is synthetic; live Anthropic API, native Windows and -bare-mode personal-skill activation remain unverified by this lane. +request artifacts. Usage is synthetic; live Anthropic API and bare-mode +personal-skill activation remain unverified by this lane. `verify:copilot-client` requires installed GitHub Copilot CLI (verified with `@github/copilot@1.0.95`); select it with `REA_VERIFY_COPILOT_COMMAND`. @@ -279,10 +282,12 @@ establish live DeepSeek API or native Windows coverage. Receipts include request artifacts, result digests and owned-process lineage. Full E2E tests invoke the production command dispatcher and real providers, -without fake launchers, runners or responses. `verify:keyed-archive` writes an -actual Foundation binary archive, runs the CLI and a separate stdio MCP -subprocess, checks parity and pagination, rejects an escaping path and checks -an XML graph golden. `verify:asset-catalog` compiles source-owned colors with +without fake launchers, runners or responses. `verify:keyed-archive` writes +Foundation binary, XML, and integer archives using one run-owned Swift module +cache. It checks CLI and separate stdio MCP parity, pagination, escaping-path +rejection, an XML graph golden, and numeric meaning for safe controls, unsafe +integers, boxed numbers, and integral-valued reals. `verify:asset-catalog` +compiles source-owned colors with `actool`, invokes real `assetutil`, then checks CLI/MCP results, exact catalog digest, every raw metadata field, pagination and malformed input rejection. Neither artifact workflow requires Hopper or launches it. Both run in macOS CI. diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index d1b7aa9334..30bc66ba75 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -13,10 +13,6 @@ import { exec } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs"; -assert( - process.platform !== "win32", - "Native Claude Code verifier requires POSIX; Windows is unverified", -); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const command = process.env.REA_VERIFY_CLAUDE_COMMAND ?? "claude"; @@ -46,8 +42,13 @@ try { await exec(command, ["--version"], { env: environment, timeout: 30_000 }) ).stdout.trim(); } catch (cause) { + // npm installs put only a claude.cmd shim on PATH, which Node does not spawn without a shell. + const windowsHint = + process.platform === "win32" + ? " On Windows, select claude.exe; npm installs provide it under node_modules/@anthropic-ai/claude-code/bin." + : ""; throw new Error( - `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND. ${String(cause)}`, + `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND.${windowsHint} ${String(cause)}`, ); } const target = join(lab, "fixture α测试"); diff --git a/scripts/verify/apple/keyed-archive-integers.mjs b/scripts/verify/apple/keyed-archive-integers.mjs new file mode 100644 index 0000000000..c95a56d3a8 --- /dev/null +++ b/scripts/verify/apple/keyed-archive-integers.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import { keyedArchiveResultSchema } from "../../../dist/domain/apple/keyedArchive.js"; + +const rowSchema = z.object({ + container: z.enum(["roots", "objects"]), + root: z.string(), + objectId: z.number().int().nonnegative().optional(), + path: z.array(z.union([z.string(), z.number().int().nonnegative()])), + kind: z.enum(["integer", "real"]), + literal: z.string(), +}); +const archiveSchema = z.object({ + filename: z.string(), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + rows: z.array(rowSchema), +}); +const oracleSchema = z.object({ xml: archiveSchema, binary: archiveSchema }); + +const formats = ["xml", "binary"]; +const requiredRows = [ + { root: "unsafeSigned", kind: "integer", literal: "9007199254740993" }, + { + root: "unsafeNegative", + kind: "integer", + literal: "-9223372036854775808", + }, + { root: "safeInteger", kind: "integer", literal: "9007199254740991" }, + { root: "safeNegative", kind: "integer", literal: "-42" }, + { root: "integralReal", kind: "real", literal: "9007199254740992" }, + { + root: "boxedUnsigned", + kind: "integer", + literal: "9007199254740993", + container: "objects", + }, + { + root: "boxedReal", + kind: "real", + literal: "9007199254740992", + container: "objects", + }, +]; + +const valueAt = (value, path) => { + for (const key of path) { + if (value === null || typeof value !== "object") return undefined; + value = value[key]; + } + return value; +}; + +const isUnsafeInteger = (row) => + row.kind === "integer" && + (BigInt(row.literal) > BigInt(Number.MAX_SAFE_INTEGER) || + BigInt(row.literal) < BigInt(Number.MIN_SAFE_INTEGER)); + +/** Verify the actual Foundation output and the production CLI/MCP result. */ +export async function verifyKeyedArchiveIntegerArchives(root) { + const oracle = oracleSchema.parse( + JSON.parse(await readFile(join(root, "oracle.json"), "utf8")), + ); + const archives = []; + + for (const format of formats) { + const archive = oracle[format]; + for (const required of requiredRows) { + assert.ok( + archive.rows.some( + (row) => + row.root === required.root && + row.kind === required.kind && + row.literal === required.literal && + (required.container === undefined || + row.container === required.container), + ), + `${format} Foundation oracle omitted ${required.root} ${required.literal}`, + ); + } + + const archivePath = join(root, archive.filename); + const digest = createHash("sha256") + .update(await readFile(archivePath)) + .digest("hex"); + assert.equal(digest, archive.sha256, `${format} archive digest`); + archives.push({ format, archive, archivePath, digest }); + } + return archives; +} + +/** Check every oracle numeric slot, preserving object-table identity lookups. */ +export function assertKeyedArchiveIntegerResult( + result, + format, + digest, + oracle, +) { + const parsed = keyedArchiveResultSchema.parse(result); + assert.equal(parsed.archive_sha256, digest); + assert.equal(parsed.target_sha256, digest); + assert.equal( + parsed.archive_format, + format === "xml" ? "xml-plist" : "binary-plist", + ); + + for (const row of oracle.rows) { + const unsafe = isUnsafeInteger(row); + const source = + row.container === "roots" + ? parsed.roots + : parsed.objects.find((object) => object.id === row.objectId)?.value; + const value = valueAt(source, row.path); + assert.notEqual( + value, + undefined, + `${row.root} numeric slot remains present`, + ); + + if (unsafe) { + const exact = + typeof value === "object" && + value !== null && + value.$plist_type === "integer" && + value.decimal === BigInt(row.literal).toString(); + const disclosed = parsed.limitations.some( + (reason) => + /integer|number/iu.test(reason) && + /precision|exact range|rounded/iu.test(reason), + ); + assert.ok( + exact || disclosed, + `${row.root} ${row.literal} must be exact or disclose numeric loss`, + ); + } else { + assert.equal( + typeof value, + "number", + `${row.root} retains its numeric type`, + ); + assert.equal(value, Number(row.literal), `${row.root} numeric value`); + } + } + return parsed; +} diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index f50eaacd15..471b76ef64 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -1,5 +1,5 @@ -import { execFile, spawnSync } from "node:child_process"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -11,6 +11,10 @@ import { artifactMcpResult, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { + assertKeyedArchiveIntegerResult, + verifyKeyedArchiveIntegerArchives, +} from "./keyed-archive-integers.mjs"; if (process.platform !== "darwin") throw new Error( @@ -170,26 +174,37 @@ try { ); }); } - const numericTests = spawnSync( - "npm", - [ - "run", - "test:focused", - "--", - "tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts", - ], - { - cwd: fileURLToPath(new URL("../../../", import.meta.url)), - stdio: "inherit", - timeout: 240_000, - }, - ); - if (numericTests.error) throw numericTests.error; - assert.equal( - numericTests.status, - 0, - "Native Foundation integer cases failed", - ); + const integers = join(root, "integers"); + await mkdir(integers); + await promisify(execFile)("/usr/bin/xcrun", [ + "swift", + "-module-cache-path", + join(root, "modules"), + fileURLToPath( + new URL( + "../../../tests/conformance/native/keyed-archive-integers.swift", + import.meta.url, + ), + ), + integers, + ]); + for (const fixture of await verifyKeyedArchiveIntegerArchives(integers)) { + const cliResult = assertKeyedArchiveIntegerResult( + await artifactCli("inspect-keyed-archive", fixture.archivePath), + fixture.format, + fixture.digest, + fixture.archive, + ); + await withArtifactMcp(fixture.archivePath, async (client) => { + const mcpResult = assertKeyedArchiveIntegerResult( + await artifactMcpResult(client, "inspect_keyed_archive"), + fixture.format, + fixture.digest, + fixture.archive, + ); + assert.deepEqual(mcpResult, cliResult); + }); + } report = { ok: true, mocked: false, @@ -197,6 +212,9 @@ try { stdio_mcp: true, xml_golden: true, uid_named_root: true, + numeric_formats: ["xml", "binary"], + numeric_cli: true, + numeric_stdio_mcp: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 104aa7c760..d11a582b21 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -20,6 +20,7 @@ import { clientRequirementsFor, NO_CLIENT_FEATURES, } from "./CapabilityClientRequirements.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const statusSchema = z.object({ open: z.boolean(), @@ -210,7 +211,7 @@ export const buildCapabilityInventory = ( reason: availability.reason, remediation: availability.remediation, }; - }).sort((left, right) => left.name.localeCompare(right.name)); + }).sort((left, right) => compareUnicodeCodePoints(left.name, right.name)); }; const availabilityFor = (context: AvailabilityContext): Availability => { diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index a6a16f96c8..bad6ee52f5 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -18,6 +18,7 @@ import { z } from "zod"; import { PRODUCT_IDENTITY } from "../identity.js"; import type { SetupClient } from "./SupportedClients.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; export type ClientConfigurationFormat = NonNullable; export type ClientServersKey = "mcp_servers" | "mcpServers" | "mcp" | "servers"; @@ -63,7 +64,7 @@ const normalizeConfigurationValue = (value: unknown): unknown => { return value; return Object.fromEntries( Object.entries(value) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([key, nested]) => [key, normalizeConfigurationValue(nested)]), ); }; diff --git a/src/application/ClientRegistrationEnvironment.ts b/src/application/ClientRegistrationEnvironment.ts index 0d537ba9b2..6048c2de15 100644 --- a/src/application/ClientRegistrationEnvironment.ts +++ b/src/application/ClientRegistrationEnvironment.ts @@ -8,6 +8,7 @@ import { type ClientConfigurationDocument, } from "./ClientConfigurationDocument.js"; import { PRODUCT_IDENTITY } from "../identity.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Read explicit REA overrides without dropping peers when one value is invalid. */ export const clientRegistrationEnvironment = ( @@ -47,7 +48,7 @@ export const clientRegistrationEnvironment = ( return ok( Object.fromEntries( Object.entries(environment).sort(([left], [right]) => - left.localeCompare(right), + compareUnicodeCodePoints(left, right), ), ), ); diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 8c3e5035b5..b40079c451 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -25,6 +25,7 @@ import { supportedClients, } from "./SupportedClients.js"; import type { SetupClient } from "./SupportedClients.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; interface ClientRegistrationStatusBase { readonly client: string; @@ -184,7 +185,7 @@ export const readClientRegistrationStatuses = async ( } } return statuses.sort((left, right) => - left.client.localeCompare(right.client), + compareUnicodeCodePoints(left.client, right.client), ); }; diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index 352724f37c..4251bbaf22 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -281,6 +281,42 @@ describe("doctor installation identity", () => { }); describe("doctor scoped readiness", () => { + it("checks explicitly selected skill destinations independently of registrations", async () => { + const selected: (readonly string[] | undefined)[] = []; + const result = await runDoctor( + undefined, + host({ + clientRegistrations: () => Promise.resolve([]), + installedSkillIdentity: (clients) => { + selected.push(clients); + return Promise.resolve({ + version: "stale", + toolCount: 0, + canonical: false, + }); + }, + }), + { + clients: [], + skillClients: ["claude_code", "claude_code"], + skill: true, + }, + ); + + expect(selected).toEqual([["claude_code"]]); + expect(result.scope).toMatchObject({ + mode: "explicit", + clients: [], + skill_clients: ["claude_code"], + skill: true, + }); + expect(result.healthy).toBe(false); + expect(result.scope_checks.map(({ name }) => name)).toEqual([ + "node", + "skill:identity", + ]); + }); + it("scopes effective readiness to selected clients without hiding drift", async () => { const registrations: readonly ClientRegistrationStatus[] = [ { diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index f9fc0575be..0f42474217 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -120,6 +120,7 @@ export interface DoctorReport { /** Caller-selected readiness boundary; omitted values retain audit semantics. */ export interface DoctorScope { readonly clients?: readonly string[]; + readonly skillClients?: readonly string[]; readonly providers?: readonly string[]; readonly skill?: boolean; } @@ -206,9 +207,10 @@ const collectDoctorIdentity = async ( const installationPaths = (await host.installationPaths?.()) ?? []; const registrations = (await host.clientRegistrations?.()) ?? []; const skillClientIds = - scope.mode === "audit-wide" + scope.skill_clients ?? + (scope.mode === "audit-wide" ? registrations.map(({ client }) => client) - : scope.clients; + : scope.clients); const installedSkillIdentity = await host.installedSkillIdentity?.(skillClientIds); const skillAligned = skillIdentityAligned(installedSkillIdentity); diff --git a/src/application/DoctorScope.ts b/src/application/DoctorScope.ts index a92cd2863a..0dd9dcd110 100644 --- a/src/application/DoctorScope.ts +++ b/src/application/DoctorScope.ts @@ -14,6 +14,7 @@ import type { export interface DoctorScopeReport { readonly mode: "audit-wide" | "explicit"; readonly clients: readonly string[]; + readonly skill_clients?: readonly string[]; readonly providers: readonly string[]; readonly skill: boolean; readonly target: string | null; @@ -40,6 +41,9 @@ export const normalizeDoctorScope = ( ): DoctorScopeReport => ({ mode: requested === undefined ? "audit-wide" : "explicit", clients: uniqueSorted(requested?.clients ?? []), + ...(requested?.skillClients === undefined + ? {} + : { skill_clients: uniqueSorted(requested.skillClients) }), providers: uniqueSorted(requested?.providers ?? []), skill: requested?.skill === true, target: target ?? null, diff --git a/src/application/ReconstructionObligationCandidates.ts b/src/application/ReconstructionObligationCandidates.ts index 60eaaed135..a269829dc2 100644 --- a/src/application/ReconstructionObligationCandidates.ts +++ b/src/application/ReconstructionObligationCandidates.ts @@ -18,6 +18,7 @@ import { semanticObligationPolicy, type ReconstructionObligationCandidatePolicy, } from "./ReconstructionObligationPolicies.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; type CandidateFields = Omit< ReconstructionObligation, @@ -54,7 +55,7 @@ export const deriveReconstructionObligationCandidates = ( addCandidate(candidates, reviewedCandidate(obligation, records)); return { candidates: [...candidates.values()].sort((left, right) => - left.obligation_id.localeCompare(right.obligation_id), + compareUnicodeCodePoints(left.obligation_id, right.obligation_id), ), limitations: [...limitations].sort(), }; @@ -430,7 +431,7 @@ const uniqueBy = ( key: (value: Value) => string, ): Value[] => [...new Map(values.map((value) => [key(value), value])).values()].sort( - (left, right) => key(left).localeCompare(key(right)), + (left, right) => compareUnicodeCodePoints(key(left), key(right)), ); const obligationId = (key: string): string => diff --git a/src/application/ReconstructionObligationLedgerDiagnostics.ts b/src/application/ReconstructionObligationLedgerDiagnostics.ts index d92fd21ef1..7d6d846371 100644 --- a/src/application/ReconstructionObligationLedgerDiagnostics.ts +++ b/src/application/ReconstructionObligationLedgerDiagnostics.ts @@ -3,6 +3,7 @@ import type { ReconstructionObligationManifest, } from "../domain/reconstructionObligationLedgerSchemas.js"; import type { Evidence } from "../domain/evidence.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; type Diagnostic = ReconstructionObligation["diagnostics"][number]; type Binding = NonNullable; @@ -50,7 +51,8 @@ export const uniqueObligationDiagnostics = ( ]), ).values(), ].sort((left, right) => - `${left.code}:${left.detail}`.localeCompare( + compareUnicodeCodePoints( + `${left.code}:${left.detail}`, `${right.code}:${right.detail}`, ), ); diff --git a/src/application/ReconstructionObligationLedgerEvaluation.ts b/src/application/ReconstructionObligationLedgerEvaluation.ts index c83c79dc9b..fe76263449 100644 --- a/src/application/ReconstructionObligationLedgerEvaluation.ts +++ b/src/application/ReconstructionObligationLedgerEvaluation.ts @@ -36,6 +36,7 @@ import { obligationReports, summarizeObligations, } from "./ReconstructionObligationLedgerProjection.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; type Diagnostic = ReconstructionObligation["diagnostics"][number]; type Binding = NonNullable; @@ -60,7 +61,7 @@ export const evaluateReconstructionObligationLedger = ({ }): ReconstructionObligationLedger => { const bundle = parseEvidenceBundle(bundleInput); const allCandidates = [...candidatesInput].sort((left, right) => - left.obligation_id.localeCompare(right.obligation_id), + compareUnicodeCodePoints(left.obligation_id, right.obligation_id), ); const orphanManifestIds = findOrphanManifestIds(allCandidates, manifest); const candidates = allCandidates; diff --git a/src/application/ReconstructionObligationOriginalCases.ts b/src/application/ReconstructionObligationOriginalCases.ts index 4d6cf3d383..46bb15be5b 100644 --- a/src/application/ReconstructionObligationOriginalCases.ts +++ b/src/application/ReconstructionObligationOriginalCases.ts @@ -2,6 +2,7 @@ import type { Evidence } from "../domain/evidence.js"; import type { ReconstructionObligation } from "../domain/reconstructionObligationLedgerSchemas.js"; import { evidenceSupportsOriginalAuthority } from "./ReconstructionObligationAuthority.js"; import type { ReconstructionObligationCandidate } from "./ReconstructionObligationCandidates.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; type Binding = NonNullable; type Diagnostic = ReconstructionObligation["diagnostics"][number]; @@ -21,7 +22,8 @@ export const originalCasesFor = ( ), ).values(), ].sort((left, right) => - `${left.kind}:${left.evidence_id}:${left.location}`.localeCompare( + compareUnicodeCodePoints( + `${left.kind}:${left.evidence_id}:${left.location}`, `${right.kind}:${right.evidence_id}:${right.location}`, ), ); diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index efde620679..c7432c68db 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -18,6 +18,7 @@ import type { ReferenceSourceRead, } from "../reference/ReferenceSourceReaderTypes.js"; import { PARSEABLE_REFERENCE_SOURCE_LANGUAGES } from "./ReferenceSourceImportTypes.js"; +import { typescriptEmittedSourceCandidates } from "./TypeScriptEmittedSourceCandidates.js"; export interface ParsedReferenceSourceEntries { readonly entries: HistoricalSourceGraphInput["entries"]; @@ -99,22 +100,9 @@ const resolveInternalSpecifier = ( `${normalized}${suffix}`, `${normalized}/index${suffix}`, ]); - // TypeScript source commonly names the extension emitted for NodeNext. // Retain present runtime files first; substitute only a missing counterpart. // Counterparts cover explicit filenames, not directory/index resolution. - if (/\.(?:ts|tsx|mts|cts)$/u.test(fromPath)) { - const sourceSuffixes = normalized.endsWith(".js") - ? [".ts", ".tsx", ".d.ts"] - : normalized.endsWith(".mjs") - ? [".mts", ".d.mts"] - : normalized.endsWith(".cjs") - ? [".cts", ".d.cts"] - : normalized.endsWith(".jsx") - ? [".tsx", ".d.ts"] - : []; - const stem = normalized.slice(0, -posix.extname(normalized).length); - candidates.push(...sourceSuffixes.map((suffix) => `${stem}${suffix}`)); - } + candidates.push(...typescriptEmittedSourceCandidates(fromPath, normalized)); const match = lookupPath.resolveCandidates(candidates); return match === undefined ? { to: normalized, resolution: "unresolved" } diff --git a/src/application/Setup.fixture.ts b/src/application/Setup.fixture.ts index f5460b78b0..a4c82678e1 100644 --- a/src/application/Setup.fixture.ts +++ b/src/application/Setup.fixture.ts @@ -159,6 +159,7 @@ export class FakeSetupHost implements SetupHost { scope === undefined ? environmentHealthy : (scope.clients?.length ?? 0) === 0 && + (scope.skillClients?.length ?? 0) === 0 && (scope.providers?.length ?? 0) === 0 && scope.skill !== true ? true @@ -169,6 +170,9 @@ export class FakeSetupHost implements SetupHost { scope: { mode: scope === undefined ? "audit-wide" : "explicit", clients: scope?.clients ?? [], + ...(scope?.skillClients === undefined + ? {} + : { skill_clients: scope.skillClients }), providers: scope?.providers ?? [], skill: scope?.skill === true, target: null, diff --git a/src/application/Setup.recovery.test.ts b/src/application/Setup.recovery.test.ts index ab480337a5..7a37d4afe2 100644 --- a/src/application/Setup.recovery.test.ts +++ b/src/application/Setup.recovery.test.ts @@ -232,6 +232,7 @@ describe("setup scoped readiness", () => { clients: ["codex"], providers: [], skill: true, + skillClients: ["codex"], }); }); diff --git a/src/application/Setup.ts b/src/application/Setup.ts index 6543748550..a9f89a30a7 100644 --- a/src/application/Setup.ts +++ b/src/application/Setup.ts @@ -125,7 +125,10 @@ const runSetupWorkflow = async ( : (options.clientIds ?? options.readinessScope?.clients ?? discovery.defaultClientIds); - let selectedSkillClientIds = options.skillClientIds ?? selectedClientIds; + let selectedSkillClientIds = + options.skillClientIds ?? + options.readinessScope?.skillClients ?? + selectedClientIds; let skillNeedsInstall = await host.skillNeedsInstall(selectedSkillClientIds); let skillSelected = options.installSkill === true || @@ -220,6 +223,7 @@ const runSetupWorkflow = async ( installHopper: planDiscovery.installHopper, installSkill, skillSelected, + selectedSkillClientIds, selectedClients: planned.selectedClients, clientsToConfigure: planned.clientsToConfigure, }; @@ -519,6 +523,7 @@ const resolvedReadinessScope = ( readonly selectedClients: readonly SetupClient[]; readonly installHopper: boolean; readonly skillSelected?: boolean; + readonly selectedSkillClientIds: readonly string[]; }, ): DoctorScope => ({ clients: selection.selectedClients.map(({ name }) => name), @@ -526,6 +531,9 @@ const resolvedReadinessScope = ( ? ["hopper"] : (requested?.providers ?? []), skill: selection.skillSelected === true || requested?.skill === true, + ...(selection.skillSelected === true || requested?.skill === true + ? { skillClients: selection.selectedSkillClientIds } + : {}), }); const summarizeDoctor = (report: DoctorReport): SetupDoctorSummary => ({ diff --git a/src/application/SetupPlan.ts b/src/application/SetupPlan.ts index 2871bd04b2..83402d1a43 100644 --- a/src/application/SetupPlan.ts +++ b/src/application/SetupPlan.ts @@ -15,6 +15,7 @@ import type { } from "./SetupTypes.js"; import type { SetupClient } from "./SupportedClients.js"; import type { DoctorReport } from "./Doctor.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Read-only setup facts shared by client selection and preflight. */ export interface SetupDiscovery { @@ -308,7 +309,7 @@ const clientConfigurationDetail = ( installHopper: boolean, ): string => { const entries = Object.entries(environment) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([name, value]) => `${name}=${value}`); const environmentDetail = entries.length === 0 diff --git a/src/application/SetupRegistrationEnvironment.ts b/src/application/SetupRegistrationEnvironment.ts index 7a46f56e1b..b9bf54f999 100644 --- a/src/application/SetupRegistrationEnvironment.ts +++ b/src/application/SetupRegistrationEnvironment.ts @@ -1,4 +1,5 @@ import type { DoctorProviderInspection } from "./Doctor.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Preserve adapter-published settings independently of runtime readiness. */ export const providerRegistrationEnvironment = ( @@ -9,5 +10,5 @@ export const providerRegistrationEnvironment = ( .flatMap(({ registrationEnvironment }) => Object.entries(registrationEnvironment), ) - .sort(([left], [right]) => left.localeCompare(right)), + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)), ); diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 915458047c..c4b0bc6e93 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -6,6 +6,7 @@ import { HermesClientPathError, resolveHermesClientHome, } from "./HermesClientHome.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** One supported client configuration location. */ export interface SetupClient { @@ -642,6 +643,6 @@ export const clientSkillDirectories = ( }); } return [...destinations.values()].sort((left, right) => - left.directory.localeCompare(right.directory), + compareUnicodeCodePoints(left.directory, right.directory), ); }; diff --git a/src/application/TypeScriptEmittedSourceCandidates.ts b/src/application/TypeScriptEmittedSourceCandidates.ts new file mode 100644 index 0000000000..663c7dada9 --- /dev/null +++ b/src/application/TypeScriptEmittedSourceCandidates.ts @@ -0,0 +1,29 @@ +/** Emitted NodeNext suffixes and the TypeScript source counterparts they name. */ +const EMITTED_SUFFIXES: ReadonlyArray< + readonly [suffix: string, sourceSuffixes: readonly string[]] +> = [ + [".js", [".ts", ".tsx", ".d.ts"]], + [".mjs", [".mts", ".d.mts"]], + [".cjs", [".cts", ".d.cts"]], + [".jsx", [".tsx", ".d.ts"]], +]; + +/** + * Return TypeScript source counterparts for an explicit emitted filename. + * Callers retain exact runtime-file precedence and must not use these for + * directory/index, URL, HTML, or package target resolution. + */ +export const typescriptEmittedSourceCandidates = ( + sourcePath: string, + candidate: string, +): readonly string[] => { + if (!/\.(?:ts|tsx|mts|cts)$/u.test(sourcePath)) return []; + const emitted = EMITTED_SUFFIXES.find(([suffix]) => + candidate.endsWith(suffix), + ); + if (emitted === undefined) return []; + // Strip the matched suffix by length. posix.extname would miss dotfile + // names such as ".js", whose whole basename is the emitted suffix itself. + const stem = candidate.slice(0, candidate.length - emitted[0].length); + return emitted[1].map((suffix) => `${stem}${suffix}`); +}; diff --git a/src/application/android/AndroidSearchEvidenceSummary.ts b/src/application/android/AndroidSearchEvidenceSummary.ts new file mode 100644 index 0000000000..42c0fb80f5 --- /dev/null +++ b/src/application/android/AndroidSearchEvidenceSummary.ts @@ -0,0 +1,120 @@ +import type { EvidenceWriter } from "../investigation/InvestigationRecordPort.js"; +import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { ok, type Result } from "../../domain/result.js"; + +/** + * Complete class-inventory pages dominate raw-call provenance for focused + * searches: one traversal embeds every examined class name regardless of the + * match count. The summary keeps the pagination audit inline (totals, page + * offsets and sizes, elided item counts) and links the retained record that + * still carries the complete inventory. + */ +const elideInventoryText = (text: string): string | undefined => { + let page: JsonValue; + try { + page = JSON.parse(text) as JsonValue; + } catch { + return undefined; + } + if ( + typeof page !== "object" || + page === null || + Array.isArray(page) || + !Array.isArray(page.items) + ) + return undefined; + const { items, ...audited } = page; + return JSON.stringify({ ...audited, items_omitted: items.length }); +}; + +/** Replace inventoried page bodies with their pagination audit. */ +const summarizeInventoryCall = (call: JsonValue): JsonValue => { + if ( + typeof call !== "object" || + call === null || + Array.isArray(call) || + call.operation !== "list_classes" + ) + return call; + const response = call.response; + if ( + typeof response !== "object" || + response === null || + Array.isArray(response) || + !Array.isArray(response.content) + ) + return call; + let elided = false; + const content = response.content.map((entry: JsonValue) => { + if ( + typeof entry !== "object" || + entry === null || + Array.isArray(entry) || + typeof entry.text !== "string" + ) + return entry; + const summarized = elideInventoryText(entry.text); + if (summarized === undefined) return entry; + elided = true; + return { ...entry, text: summarized }; + }); + return elided + ? { ...call, inventory_elided: true, response: { ...response, content } } + : call; +}; + +/** + * Retain the complete search Evidence — full raw provider calls included — + * in the session record, then deliver a bounded summary Evidence inline. + * Callers without a retaining session keep the complete inline delivery, so + * complete CLI JSON output is unchanged. Coverage metadata stays truthful in + * both copies: the examined inventory is never rewritten to matched-only. + */ +export const summarizeRetainedAndroidSearch = ( + evidence: Evidence, + recordEvidence: EvidenceWriter["recordEvidence"] | undefined, +): Result => { + if (recordEvidence === undefined) return ok(evidence); + const recorded = recordEvidence(evidence); + if (!recorded.ok) return recorded; + const subject = evidence.subject; + const raw = evidence.raw_result; + const summarized: JsonValue = + raw === null || + typeof raw !== "object" || + Array.isArray(raw) || + !Array.isArray(raw.calls) + ? raw + : { + ...raw, + complete_evidence_id: evidence.evidence_id, + calls: raw.calls.map(summarizeInventoryCall), + }; + return ok( + createEvidence( + subject === null + ? undefined + : { + path: subject.local_path, + sha256: subject.digest.sha256, + format: subject.format, + ...(subject.architecture === null + ? {} + : { architecture: subject.architecture }), + }, + evidence.provider, + { + operation: evidence.operation, + parameters: evidence.parameters, + result: evidence.normalized_result, + rawResult: summarized, + confidence: evidence.confidence, + limitations: evidence.limitations, + locations: evidence.locations, + evidenceLinks: [evidence.evidence_id], + }, + ), + ); +}; diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 2702a07061..43e6bbb9cd 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -25,6 +25,7 @@ import { evaluateAnalysisProviderCandidate, type AnalysisProviderCandidateEvaluation, } from "./AnalysisProviderEvaluation.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; /** Deterministic discovery truth for one configured deep provider. */ export interface AnalysisProviderCandidateStatus { @@ -109,12 +110,14 @@ export class AnalysisProviderRegistry { } this.#providers = Object.freeze( [...providers].sort((left, right) => - left.identity().id.localeCompare(right.identity().id), + compareUnicodeCodePoints(left.identity().id, right.identity().id), ), ); this.#byId = byId; this.#declaredOperations = Object.freeze( - [...declaredOperations].sort((left, right) => left.localeCompare(right)), + [...declaredOperations].sort((left, right) => + compareUnicodeCodePoints(left, right), + ), ); } diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index 9048c6de62..8fbe4aa2b4 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -11,6 +11,7 @@ import type { SessionProviderRoute, SessionProviderRouter, } from "./SessionProviderRouter.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; interface BinarySessionStatusInput { readonly target: BinaryTarget | undefined; @@ -39,7 +40,9 @@ export const binarySessionStatus = ({ }: BinarySessionStatusInput): JsonValue => { const capabilities = [...route.capabilities.values()] .filter(({ operation }) => operation !== "inventory_artifact") - .sort((left, right) => left.operation.localeCompare(right.operation)) + .sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ) .map((descriptor) => { const unavailable = runtimeUnavailability.get(descriptor.operation); return unavailable === undefined @@ -69,7 +72,9 @@ export const binarySessionStatus = ({ capabilities: candidate.capabilities .slice() .filter(({ operation }) => operation !== "inventory_artifact") - .sort((left, right) => left.operation.localeCompare(right.operation)) + .sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ) .map(capabilityStatus), })); const common = { @@ -102,7 +107,9 @@ const requestActivityStatus = ( ) => { const providers = snapshots .slice() - .sort((left, right) => left.provider.id.localeCompare(right.provider.id)) + .sort((left, right) => + compareUnicodeCodePoints(left.provider.id, right.provider.id), + ) .map(({ provider, active, queuedRequests }) => ({ provider: providerSummary(provider), active: @@ -150,7 +157,7 @@ const processLineageStatus = ( snapshots: snapshots .slice() .sort((left, right) => - left.provider.id.localeCompare(right.provider.id), + compareUnicodeCodePoints(left.provider.id, right.provider.id), ) .map(({ provider, observation }) => ({ provider: providerSummary(provider), diff --git a/src/application/binary/CompositeProvider.ts b/src/application/binary/CompositeProvider.ts index 941cf93752..b0a7e7c13a 100644 --- a/src/application/binary/CompositeProvider.ts +++ b/src/application/binary/CompositeProvider.ts @@ -10,6 +10,7 @@ import type { ProviderIdentity, } from "../AnalysisProvider.js"; import { closeAnalysisClient } from "./AnalysisClientCleanup.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; /** Synthetic identity for a deterministic provider set. */ const compositeProviderIdentity = ( @@ -53,8 +54,8 @@ export class CompositeProvider implements AnalysisProvider { this.#capabilities = Object.freeze( capabilities.sort( (left, right) => - left.operation.localeCompare(right.operation) || - left.provider.id.localeCompare(right.provider.id), + compareUnicodeCodePoints(left.operation, right.operation) || + compareUnicodeCodePoints(left.provider.id, right.provider.id), ), ); } @@ -122,13 +123,13 @@ export class CompositeProvider implements AnalysisProvider { [...clients.values()] .flatMap((client) => client.runtimeLineageSnapshots?.() ?? []) .sort((left, right) => - left.provider.id.localeCompare(right.provider.id), + compareUnicodeCodePoints(left.provider.id, right.provider.id), ), requestActivitySnapshots: () => [...clients.values()] .flatMap((client) => client.requestActivitySnapshots?.() ?? []) .sort((left, right) => - left.provider.id.localeCompare(right.provider.id), + compareUnicodeCodePoints(left.provider.id, right.provider.id), ), operationHealthSnapshot: () => { for (const client of clients.values()) { diff --git a/src/application/investigation/EvidenceLedger.ts b/src/application/investigation/EvidenceLedger.ts index 1b88c5737d..fb27d40c8f 100644 --- a/src/application/investigation/EvidenceLedger.ts +++ b/src/application/investigation/EvidenceLedger.ts @@ -24,6 +24,7 @@ import { type UpdateUnknownInput, } from "../../domain/residualUnknown.js"; import { err, ok, type Result } from "../../domain/result.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; type EvidenceLedgerFailure = EvidenceIntegrityError; type RecordResult = Result<"added" | "duplicate", EvidenceLedgerFailure>; @@ -321,7 +322,9 @@ export class EvidenceLedger { unknown.severity === filters.severity) && (filters.domain === undefined || unknown.domain === filters.domain), ) - .sort((left, right) => left.unknown_id.localeCompare(right.unknown_id)) + .sort((left, right) => + compareUnicodeCodePoints(left.unknown_id, right.unknown_id), + ) .map((unknown) => structuredClone(unknown)); } diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts index 9a79f84d14..8af7aa19d2 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts @@ -868,6 +868,365 @@ describe("package exports exact targets", () => { }); }); +describe("package import alias resolution", () => { + it.each(["import", "require"] as const)( + "resolves an exact #%s alias from the enclosing package imports map", + (moduleKind) => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./config.cjs" }, + }), + "main.js": "", + "config.cjs": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind, + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + declared_path: "#config", + }); + }, + ); + + it("resolves an alias from a nested file through the enclosing package root", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./config.cjs" }, + }), + "src/main.js": "", + "config.cjs": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "src/main.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + }); + }); + + it("uses a nested package imports map rather than the application root", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./root.cjs" }, + }), + "root.cjs": "", + "node_modules/fixture/package.json": JSON.stringify({ + imports: { "#config": "./local.cjs" }, + }), + "node_modules/fixture/index.js": "", + "node_modules/fixture/local.cjs": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "node_modules/fixture/index.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "node_modules/fixture/local.cjs", + }); + }); + + it("does not inherit an ancestor imports map across a nested package.json", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./root.cjs" }, + }), + "root.cjs": "", + "node_modules/fixture/package.json": "{}", + "node_modules/fixture/index.js": "", + }); + const outcome = resolve({ + declaredPath: "#config", + sourcePath: "node_modules/fixture/index.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain( + "node_modules/fixture/package.json", + ); + expect(outcome.limitations.join(" ")).not.toContain("root.cjs"); + }); + + it("does not apply a nested package map to the application root importer", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./root.cjs" }, + }), + "main.js": "", + "root.cjs": "", + "node_modules/fixture/package.json": JSON.stringify({ + imports: { "#config": "./local.cjs" }, + }), + "node_modules/fixture/local.cjs": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "root.cjs", + }); + }); +}); + +describe("package import alias target selection", () => { + it("selects import and require conditions from an imports target", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { + "#config": { + import: "./config.mjs", + require: "./config.cjs", + default: "./fallback.js", + }, + }, + }), + "main.js": "", + "config.mjs": "", + "config.cjs": "", + "fallback.js": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "import", + files, + }).resolved_path, + ).toBe("config.mjs"); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }).resolved_path, + ).toBe("config.cjs"); + }); + + it("resolves array import targets at their first string entry", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": [{ import: "./missing.mjs" }, "./config.cjs"] }, + }), + "main.js": "", + "config.cjs": "", + }); + expect( + resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + }); + }); + + it("does not add an extension or index to a selected import target", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./config" }, + }), + "main.js": "", + "config.cjs": "", + "config/index.js": "", + }); + const outcome = resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain( + "import targets are exact files", + ); + }); +}); + +describe("package import alias refusals", () => { + it("reports a missing exact alias without looking in another package", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./config.cjs" }, + }), + "main.js": "", + "config.cjs": "", + }); + const outcome = resolve({ + declaredPath: "#other", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain("#other"); + expect(outcome.limitations.join(" ")).toContain("#config"); + }); + + it("reports an explicit null import target as blocked", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": null }, + }), + "main.js": "", + "config.cjs": "", + }); + const outcome = resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain("explicit null"); + }); + + it("leaves wildcard import aliases unresolved", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#internal/*": "./src/*.js" }, + }), + "main.js": "", + "src/util.js": "", + }); + const outcome = resolve({ + declaredPath: "#internal/util", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "import", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toMatch(/wildcard/i); + }); + + it("leaves recursive and external-package import targets unresolved", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#loop": "#config", "#dep": "fixture" }, + }), + "main.js": "", + "node_modules/fixture/index.js": "", + }); + const recursive = resolve({ + declaredPath: "#loop", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + const external = resolve({ + declaredPath: "#dep", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(recursive).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(recursive.limitations.join(" ")).toMatch(/recursive/i); + expect(external).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(external.limitations.join(" ")).toMatch(/external-package/i); + }); + + it("rejects reserved hash specifiers", () => { + const files = aliasFiles({ + "package.json": JSON.stringify({ + imports: { "#config": "./config.cjs" }, + }), + "main.js": "", + "config.cjs": "", + }); + expect( + resolve({ + declaredPath: "#", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }); + + it("does not use a foreign-container enclosing package.json", () => { + const files = fileMap([ + file( + "package.json", + "other", + JSON.stringify({ imports: { "#config": "./config.cjs" } }), + ), + file("main.js", "root"), + file("config.cjs", "root"), + ]); + const outcome = resolve({ + declaredPath: "#config", + sourcePath: "main.js", + context: "module-specifier", + moduleKind: "require", + files, + }); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain("package scopes"); + }); +}); + describe("directory package entrypoint precedence", () => { it.each([ ["require", '{"main":"actual.cjs"}', "actual.cjs"], @@ -1105,6 +1464,159 @@ describe("contextual JavaScript module identity", () => { }); }); +describe("TypeScript NodeNext emitted-extension resolution", () => { + const container = "container"; + + it.each([ + ["./dep.js", "src/dep.ts"], + ["./ui/Button.js", "src/ui/Button.tsx"], + ["./cfg.mjs", "src/cfg.mts"], + ["./legacy.cjs", "src/legacy.cts"], + ] as const)( + "resolves %s to its TypeScript source counterpart", + (specifier, target) => { + const files = fileMap([ + file("src/main.ts", container), + file(target, container, "export const value = 1;"), + ]); + expect( + resolve({ + declaredPath: specifier, + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: target }); + }, + ); + + it.each([ + ["./dep.js", "src/dep.d.ts"], + ["./cfg.mjs", "src/cfg.d.mts"], + ["./legacy.cjs", "src/legacy.d.cts"], + ] as const)( + "resolves %s to a declaration counterpart", + (specifier, target) => { + const files = fileMap([ + file("src/main.ts", container), + file(target, container, "export const value = 1;"), + ]); + expect( + resolve({ + declaredPath: specifier, + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: target }); + }, + ); + + it("prefers a present runtime file and keeps the fallback off for JS callers", () => { + const files = fileMap([ + file("src/main.ts", container), + file("src/dep.js", container), + file("src/dep.ts", container), + file("src/js-main.js", container), + ]); + expect( + resolve({ + declaredPath: "./dep.js", + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "src/dep.js", + }); + expect( + resolve({ + declaredPath: "./dep.js", + sourcePath: "src/js-main.js", + context: "module-specifier", + moduleKind: "import", + files: fileMap([ + file("src/js-main.js", container), + file("src/dep.ts", container), + ]), + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + }); + + it("does not apply the mapping to URL resources or package export targets", () => { + const urlFiles = fileMap([ + file("src/main.ts", container), + file("src/asset.ts", container), + ]); + expect( + resolve({ + declaredPath: "asset.js", + sourcePath: "src/main.ts", + context: "url-reference", + files: urlFiles, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + + const packageFiles = fileMap([ + file("src/main.ts", container), + file( + "src/node_modules/fixture/package.json", + container, + '{"exports":"./dep.js"}', + ), + file("src/node_modules/fixture/dep.ts", container), + ]); + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files: packageFiles, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + }); + + it("does not substitute source counterparts for directory or index resolution", () => { + const files = fileMap([ + file("src/main.ts", container), + file("src/lib/index.ts", container), + ]); + expect( + resolve({ + declaredPath: "./lib.js", + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + }); + + it("resolves an emitted dotfile name within its own directory", () => { + const files = fileMap([ + file("src/main.ts", container), + file("src/.ts", container), + ]); + expect( + resolve({ + declaredPath: "./.js", + sourcePath: "src/main.ts", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "src/.ts", + }); + }); +}); + describe("url-reference JavaScript artifact path resolution", () => { it.each([ "app%00.js.map", @@ -1211,6 +1723,15 @@ const file = ( : { included: true, value: text }, }); +const aliasFiles = ( + entries: Readonly>, +): ReturnType => + fileMap( + Object.entries(entries).map(([path, text]) => + file(path, "root", text ?? ""), + ), + ); + describe("exact HTML resource resolution", () => { it.each([ ["extension", ["app.js"]], diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index 73d82e91b3..bd3bcdaaf9 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -14,6 +14,12 @@ import { percentDecodeUrlPath, stripQueryAndFragment, } from "../../domain/artifactPathSyntax.js"; +import { + classifyPackageImportSpecifier, + classifyPackageImportTargetString, + lookupExactPackageImport, +} from "../../domain/javascript/javascriptPackageImportAliases.js"; +import { typescriptEmittedSourceCandidates } from "../TypeScriptEmittedSourceCandidates.js"; type ArtifactPathResolutionContext = | "package-entrypoint" @@ -54,6 +60,7 @@ export interface ResolveArtifactPathInput { readonly context: ArtifactPathResolutionContext; readonly files: ReadonlyMap; readonly htmlBaseHref?: string | null; + /** Import/require semantics; runtime resource paths omit this provenance. */ readonly moduleKind?: "import" | "require"; } @@ -68,6 +75,21 @@ const EXTENSIONS = [ ".node", ]; +// A private URL namespace for logical artifact paths, including root packages. +// Encode inventory segments so filename punctuation remains literal in the base. +const PACKAGE_ARTIFACT_ROOT_URL = new URL("file:///rea-artifact/"); +const packageMetadataUrl = (path: string): URL => + new URL( + path.split("/").map(encodeURIComponent).join("/"), + PACKAGE_ARTIFACT_ROOT_URL, + ); + +interface PackageFileTarget { + readonly value: string; + /** Normalized URL retained until loader-specific validation and file lookup. */ + readonly url: URL; +} + type CandidateResolution = | { readonly resolvedPath: string; @@ -103,6 +125,13 @@ export const resolveArtifactPathByContext = ( }; const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { + // Package import keys are literal lookup identities, not artifact paths. + // The selected target is admitted separately by resolveExportTarget. + if ( + input.context === "module-specifier" && + input.declaredPath.startsWith("#") + ) + return resolvePackageImportAlias(input, input.declaredPath); const rejected = rejectDeclaration(input.declaredPath); if (rejected !== null) return rejected; const candidate = contextualCandidate(input); @@ -116,7 +145,16 @@ const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { `The exact HTML resource ${confined} was not found among the selected application files.`, ]) ); - return resolveCandidate(input, confined, new Set()); + return resolveCandidate( + input, + confined, + new Set(), + input.context === "module-specifier" && + input.moduleKind !== undefined && + (input.declaredPath.startsWith(".") || input.declaredPath.startsWith("/")) + ? typescriptEmittedSourceCandidates(input.sourcePath, confined) + : [], + ); }; const rejectDeclaration = (declared: string): CandidateResolution | null => { @@ -202,9 +240,9 @@ const bareModuleCandidate = ( declared: string, ): CandidateResolution => { const packageName = barePackageName(declared); - if (packageName === null || isBuiltin(declared) || declared.startsWith("#")) + if (packageName === null || isBuiltin(declared)) return unresolvedCandidate("external", [ - "The bare specifier is a Node builtin, package import map, or invalid package name.", + "The bare specifier is a Node builtin or invalid package name.", ]); if (input.moduleKind === "import" && packageName.includes("%")) return unresolvedCandidate("rejected", [ @@ -235,6 +273,125 @@ const bareModuleCandidate = ( ]); }; +/** + * Nearest same-container package.json, stopping at a `node_modules` directory + * so a nested package cannot inherit an ancestor's map. + */ +const enclosingPackageMetadata = ( + input: ResolveArtifactPathInput, +): + | Extract + | { readonly kind: "none" } => { + let directory = posix.dirname(input.sourcePath); + while (posix.basename(directory) !== "node_modules") { + const metadata = readPackageMetadata(input, directory); + if (metadata.kind !== "absent") return metadata; + if (directory === "." || directory === "") break; + directory = posix.dirname(directory); + } + return { kind: "none" }; +}; + +/** + * Resolve an exact `#alias` against the importer's enclosing `imports` map. + * Selected relative targets are exact files; patterns, recursive aliases, and + * external-package targets stay unresolved without falling through. + */ +const resolvePackageImportAlias = ( + input: ResolveArtifactPathInput, + specifier: string, +): CandidateResolution => { + const classified = classifyPackageImportSpecifier(specifier); + if (classified.kind === "invalid") + return unresolvedCandidate("rejected", [ + `The package import specifier ${JSON.stringify(specifier)} is invalid; "#" and specifiers that end with "/" are not valid import aliases.`, + ]); + if (classified.kind === "pattern") + return unresolvedCandidate("external", [ + `Wildcard and directory package import aliases remain unresolved; ${JSON.stringify(specifier)} is not an exact imports key.`, + ]); + const metadata = enclosingPackageMetadata(input); + if (metadata.kind === "none") + return unresolvedCandidate("external", [ + `No enclosing package.json defines package import alias ${specifier}; package import maps do not cross package scopes.`, + ]); + if (metadata.kind === "unavailable") + return unresolvedCandidate("unavailable", [metadata.limitation]); + const lookup = lookupExactPackageImport(metadata.imports, specifier); + if (lookup.kind === "undefined-map") + return unresolvedCandidate("external", [ + `Package metadata ${metadata.path} does not define package import alias ${specifier}.`, + ]); + if (lookup.kind === "missing-key") + return unresolvedCandidate("external", [ + `Package metadata ${metadata.path} does not define package import alias ${specifier}; it declares ${lookup.declared.join(", ") || "no imports keys"}.`, + ]); + if (lookup.kind === "pattern-unresolved") + return unresolvedCandidate("external", [ + `Wildcard package import aliases remain unresolved; package metadata ${metadata.path} has no exact imports key ${specifier}.`, + ]); + const selected = selectPackageImportTarget( + lookup.target, + input.moduleKind, + metadata.path, + ); + if (selected.status === "rejected") + return unresolvedCandidate("rejected", [ + `Package metadata ${metadata.path}: ${selected.limitation}`, + ]); + if (selected.status === "unmatched") + return unresolvedCandidate("external", [ + `Package metadata ${metadata.path} declares no imports target for alias ${specifier} under the active conditions; it declares ${selected.declared.join(", ") || "no conditions"}.`, + ]); + if (selected.status === "blocked") + return unresolvedCandidate("external", [ + `Package metadata ${metadata.path} blocks import alias ${specifier} with ${selected.cause === "null" ? "an explicit null imports target" : "an empty imports target array"}.`, + ]); + if (selected.status === "non-relative") { + const target = classifyPackageImportTargetString(selected.value); + return unresolvedCandidate("external", [ + target.kind === "recursive" + ? `Recursive package import aliases remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.` + : `External-package import targets remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.`, + ]); + } + return resolveExportTarget(input, metadata.path, selected, "imports"); +}; + +const selectPackageImportTarget = ( + value: unknown, + moduleKind: ResolveArtifactPathInput["moduleKind"], + packagePath: string, +): + | ({ readonly status: "value" } & PackageFileTarget) + | { readonly status: "non-relative"; readonly value: string } + | { readonly status: "blocked"; readonly cause: PackageTargetBlockCause } + | { readonly status: "unmatched"; readonly declared: readonly string[] } + | { readonly status: "rejected"; readonly limitation: string } => { + const selected = selectExportTarget( + value, + packageExportConditions(moduleKind), + packageMetadataUrl(packagePath), + "imports", + ); + if (selected.kind === "invalid" || selected.kind === "invalid-config") + return { status: "rejected", limitation: selected.limitation }; + if (selected.kind === "unmatched") + return { + status: "unmatched", + declared: selected.declared, + }; + if (selected.kind === "blocked") + return { status: "blocked", cause: selected.cause }; + if (selected.kind === "non-relative") + return { status: "non-relative", value: selected.value }; + return { + status: "value", + value: selected.value, + url: selected.url, + }; +}; + /** Self references use the nearest package scope before dependency lookup. */ const commonJsSelfCandidate = ( input: ResolveArtifactPathInput, @@ -242,35 +399,22 @@ const commonJsSelfCandidate = ( ): CandidateResolution | null => { if (input.moduleKind !== "require" || !input.files.has(input.sourcePath)) return null; - let directory = posix.dirname(input.sourcePath); - while (posix.basename(directory) !== "node_modules") { - const metadata = readPackageMetadata(input, directory); - if (metadata.kind === "unavailable") - return unresolvedCandidate("unavailable", [metadata.limitation]); - if (metadata.kind === "available") { - if ( - metadata.name === undefined || - metadata.exports === undefined || - (declared !== metadata.name && - !declared.startsWith(`${metadata.name}/`)) - ) - return null; - const subpath = - declared === metadata.name - ? "." - : `.${declared.slice(metadata.name.length)}`; - // Self resolution does not apply the CommonJS dependency-name matcher. - return resolvePackageExports( - input, - metadata.path, - subpath, - metadata.exports, - ); - } - if (directory === "." || directory === "") break; - directory = posix.dirname(directory); - } - return null; + const metadata = enclosingPackageMetadata(input); + if (metadata.kind === "none") return null; + if (metadata.kind === "unavailable") + return unresolvedCandidate("unavailable", [metadata.limitation]); + if ( + metadata.name === undefined || + metadata.exports === undefined || + (declared !== metadata.name && !declared.startsWith(`${metadata.name}/`)) + ) + return null; + const subpath = + declared === metadata.name + ? "." + : `.${declared.slice(metadata.name.length)}`; + // Self resolution does not apply the CommonJS dependency-name matcher. + return resolvePackageExports(input, metadata.path, subpath, metadata.exports); }; /** Package roots and exact subpaths share metadata and exports selection. */ @@ -313,7 +457,12 @@ const resolvePackageExports = ( subpath: string, rawExports: string | object, ): CandidateResolution => { - const exported = selectPackageExport(rawExports, subpath, input.moduleKind); + const exported = selectPackageExport( + rawExports, + subpath, + input.moduleKind, + packagePath, + ); if (exported.status === "rejected") return unresolvedCandidate("rejected", [ `Package metadata ${packagePath}: ${exported.limitation}`, @@ -324,26 +473,47 @@ const resolvePackageExports = ( ]); if (exported.status === "blocked") return unresolvedCandidate("external", [ - `Directory package metadata ${packagePath} blocks subpath ${subpath} with an explicit null exports target.`, + `Directory package metadata ${packagePath} blocks subpath ${subpath} with ${exported.cause === "null" ? "an explicit null exports target" : "an empty exports target array"}.`, ]); - return resolveExportTarget(input, packagePath, exported.value); + return resolveExportTarget(input, packagePath, exported); }; -/** An exports target is a URL naming one exact file for both Node loaders. */ +/** An exports or import target is a URL naming one exact file for both Node loaders. */ const resolveExportTarget = ( input: ResolveArtifactPathInput, packagePath: string, - target: string, + target: PackageFileTarget, + mapKind: "exports" | "imports" = "exports", ): CandidateResolution => { - const rejected = rejectDeclaration(target); - if (rejected !== null) return rejected; - const decoded = decodeModuleUrlPath(stripQueryAndFragment(target)); - if (typeof decoded !== "string") return decoded; - const candidate = confineCandidate(relativeCandidate(packagePath, decoded)); - if (typeof candidate !== "string") return candidate; + const withTargetSource = ( + outcome: CandidateResolution, + ): CandidateResolution => ({ + ...outcome, + limitations: outcome.limitations.map( + (reason) => + `Package metadata ${packagePath}, ${mapKind} target ${JSON.stringify(target.value)}: ${reason}`, + ), + }); + // Node's CommonJS finalizer checks the serialized URL, including suffixes; + // its ESM finalizer checks only the pathname. Neither error retries arrays. + if (input.moduleKind === "require" && /%(?:2f|5c)/iu.test(target.url.href)) + return withTargetSource( + unresolvedCandidate("rejected", [ + "The selected package URL contains encoded separator bytes.", + ]), + ); + const encodedPath = target.url.pathname.slice( + PACKAGE_ARTIFACT_ROOT_URL.pathname.length, + ); + const rejected = rejectDeclaration(encodedPath); + if (rejected !== null) return withTargetSource(rejected); + const decoded = decodeModuleUrlPath(encodedPath); + if (typeof decoded !== "string") return withTargetSource(decoded); + const candidate = confineCandidate(decoded); + if (typeof candidate !== "string") return withTargetSource(candidate); return ( resolveFileCandidates(input, [candidate]) ?? - exactExportsTargetNotFound(candidate) + withTargetSource(exactPackageTargetNotFound(candidate, mapKind)) ); }; @@ -492,6 +662,7 @@ type PackageMetadata = readonly path: string; readonly name: string | undefined; readonly exports: string | object | undefined; + readonly imports: object | undefined; readonly main: unknown; readonly module: unknown; } @@ -532,6 +703,7 @@ const readPackageMetadata = ( }; const name = Reflect.get(value, "name"); const exports = Reflect.get(value, "exports"); + const imports = Reflect.get(value, "imports"); return { kind: "available", path, @@ -543,6 +715,11 @@ const readPackageMetadata = ( (typeof exports === "object" && exports !== null) ? exports : undefined, + // Import maps must be non-array objects; other shapes are not a map. + imports: + typeof imports === "object" && imports !== null && !Array.isArray(imports) + ? imports + : undefined, main: Reflect.get(value, "main"), module: Reflect.get(value, "module"), }; @@ -553,8 +730,12 @@ const resolveCandidate = ( input: ResolveArtifactPathInput, candidate: string, packageChain: ReadonlySet, + emittedSourceCandidates: readonly string[] = [], ): CandidateResolution => - resolveFileCandidates(input, fileCandidates(candidate)) ?? + resolveFileCandidates(input, [ + ...fileCandidates(candidate), + ...emittedSourceCandidates, + ]) ?? resolveDirectory( input, candidate, @@ -644,10 +825,12 @@ const fileCandidates = (candidate: string): readonly string[] => [ const indexCandidates = (candidate: string): readonly string[] => EXTENSIONS.map((extension) => posix.join(candidate, `index${extension}`)); +type PackageTargetBlockCause = "null" | "empty-array"; + type PackageExportOutcome = - | { readonly status: "value"; readonly value: string } + | ({ readonly status: "value" } & PackageFileTarget) | { readonly status: "rejected"; readonly limitation: string } - | { readonly status: "blocked" } + | { readonly status: "blocked"; readonly cause: PackageTargetBlockCause } | { readonly status: "unmatched"; readonly declared: readonly string[] }; /** Only exact non-directory keys participate; wildcard expansion is unsupported. */ @@ -655,6 +838,7 @@ const selectPackageExport = ( value: unknown, subpath: string, moduleKind: ResolveArtifactPathInput["moduleKind"], + packagePath: string, ): PackageExportOutcome => { let target = value; let declared: string[] = []; @@ -681,35 +865,44 @@ const selectPackageExport = ( const selected = selectExportTarget( target, packageExportConditions(moduleKind), + packageMetadataUrl(packagePath), ); if (selected.kind === "invalid" || selected.kind === "invalid-config") return { status: "rejected", limitation: selected.limitation }; if (selected.kind === "unmatched") return { status: "unmatched", - declared: - typeof target === "object" && target !== null - ? Object.keys(target) - : declared, + declared: selected.declared, }; - return selected.kind === "blocked" - ? { status: "blocked" } - : { status: "value", value: selected.value }; + if (selected.kind === "blocked") + return { status: "blocked", cause: selected.cause }; + if (selected.kind === "non-relative") + return { + status: "rejected", + limitation: `The package exports target ${JSON.stringify(selected.value)} must start with "./".`, + }; + return { + status: "value", + value: selected.value, + url: selected.url, + }; }; type ExportTargetOutcome = - | { readonly kind: "value"; readonly value: string } - | { readonly kind: "blocked" } - | { readonly kind: "unmatched" } + | ({ readonly kind: "value" } & PackageFileTarget) + | { readonly kind: "non-relative"; readonly value: string } + | { readonly kind: "blocked"; readonly cause: PackageTargetBlockCause } + | { readonly kind: "unmatched"; readonly declared: readonly string[] } | { readonly kind: "invalid"; readonly limitation: string } | { readonly kind: "invalid-config"; readonly limitation: string }; const invalidExportTarget = ( value: unknown, reason: string, + mapKind: "exports" | "imports" = "exports", ): ExportTargetOutcome => ({ kind: "invalid", - limitation: `The package exports target ${JSON.stringify(value) ?? String(value)} ${reason}.`, + limitation: `The package ${mapKind === "imports" ? "import" : "exports"} target ${JSON.stringify(value) ?? String(value)} ${reason}.`, }); // Node checks raw target segments before URL normalization. Compare encoded @@ -729,50 +922,97 @@ const forbiddenExportSegment = (value: string): string | undefined => ); }); +const selectStringPackageTarget = ( + value: string, + packageUrl: URL, + mapKind: "exports" | "imports", +): ExportTargetOutcome => { + if (mapKind === "imports") { + const target = classifyPackageImportTargetString(value); + if (target.kind === "invalid") + return invalidExportTarget( + value, + "cannot be parent-relative, absolute, or a URL", + mapKind, + ); + if (target.kind !== "relative") return { kind: "non-relative", value }; + } else if (!value.startsWith("./")) + return invalidExportTarget(value, 'must start with "./"', mapKind); + const forbidden = forbiddenExportSegment(value); + if (forbidden !== undefined) + return invalidExportTarget( + value, + `contains forbidden path segment ${JSON.stringify(forbidden)} before URL normalization`, + mapKind, + ); + // Node constructs the URL before deciding whether this array entry is an + // invalid package target. Tabs/newlines and dot segments normalize here; + // percent decoding and existence checks belong after target selection. + const url = new URL(value, packageUrl); + if (!url.pathname.startsWith(new URL(".", packageUrl).pathname)) + return invalidExportTarget( + value, + "escapes the package root after URL normalization", + mapKind, + ); + return { + kind: "value", + value, + url, + }; +}; + +const selectArrayPackageTarget = ( + value: readonly unknown[], + conditions: ReadonlySet, + packageUrl: URL, + mapKind: "exports" | "imports", +): ExportTargetOutcome => { + if (value.length === 0) return { kind: "blocked", cause: "empty-array" }; + let lastRefusal: + | Extract + | undefined; + const declared = new Set(); + for (const entry of value) { + const nested = selectExportTarget(entry, conditions, packageUrl, mapKind); + // Node's array fallback catches invalid targets, not invalid configuration. + if (nested.kind === "invalid-config") return nested; + if (nested.kind === "unmatched") { + for (const condition of nested.declared) declared.add(condition); + continue; + } + if (nested.kind === "invalid" || nested.kind === "blocked") { + lastRefusal = nested; + continue; + } + return nested; + } + return lastRefusal ?? { kind: "unmatched", declared: [...declared] }; +}; + /** * Resolve a target in Node's declared order. Explicit null blocks an active * condition, whereas an unmatched nested object permits the next condition. * An array skips unmatched and invalid entries, stops at its first string, and - * preserves the final invalid-versus-null refusal if no string is selected. + * preserves the last invalid/null outcome if no string is selected. A nonempty + * array of unmatched conditions stays unmatched; an empty array blocks. */ const selectExportTarget = ( value: unknown, conditions: ReadonlySet, + packageUrl: URL, + mapKind: "exports" | "imports" = "exports", ): ExportTargetOutcome => { - if (typeof value === "string") { - if (!value.startsWith("./")) - return invalidExportTarget(value, 'must start with "./"'); - const forbidden = forbiddenExportSegment(value); - return forbidden === undefined - ? { kind: "value", value } - : invalidExportTarget( - value, - `contains forbidden path segment ${JSON.stringify(forbidden)} before URL normalization`, - ); - } - if (value === null) return { kind: "blocked" }; - if (Array.isArray(value)) { - let invalid: string | null = null; - for (const entry of value) { - const nested = selectExportTarget(entry, conditions); - // Node's array fallback catches invalid targets, not invalid configuration. - if (nested.kind === "invalid-config") return nested; - if (nested.kind === "invalid") { - invalid = nested.limitation; - continue; - } - if (nested.kind === "unmatched") continue; - invalid = null; - if (nested.kind === "value") return nested; - } - return invalid === null - ? { kind: "blocked" } - : { kind: "invalid", limitation: invalid }; - } + if (typeof value === "string") + return selectStringPackageTarget(value, packageUrl, mapKind); + if (value === null) return { kind: "blocked", cause: "null" }; + if (Array.isArray(value)) + return selectArrayPackageTarget(value, conditions, packageUrl, mapKind); if (typeof value !== "object") return invalidExportTarget( value, "must be a relative string, object, array, or null", + mapKind, ); for (const key of Object.keys(value)) { const numeric = Number(key); @@ -782,12 +1022,15 @@ const selectExportTarget = ( limitation: `The package exports configuration contains numeric condition key ${JSON.stringify(key)}.`, }; } + const declared = new Set(); for (const [condition, target] of Object.entries(value)) { + declared.add(condition); if (!conditions.has(condition)) continue; - const nested = selectExportTarget(target, conditions); + const nested = selectExportTarget(target, conditions, packageUrl, mapKind); if (nested.kind !== "unmatched") return nested; + for (const name of nested.declared) declared.add(name); } - return { kind: "unmatched" }; + return { kind: "unmatched", declared: [...declared] }; }; /** @@ -816,11 +1059,14 @@ const notFoundCandidate = (): CandidateResolution => ({ ], }); -const exactExportsTargetNotFound = (path: string): CandidateResolution => ({ +const exactPackageTargetNotFound = ( + path: string, + mapKind: "exports" | "imports", +): CandidateResolution => ({ resolvedPath: null, status: "not-found", limitations: [ - `The selected package exports target ${path} does not exist in the inventoried artifact container; exports targets are exact files, so no extension or directory index was tried.`, + `The selected package ${mapKind === "imports" ? "import" : "exports"} target ${path} does not exist in the inventoried artifact container; ${mapKind === "imports" ? "import" : "exports"} targets are exact files, so no extension or directory index was tried.`, ], }); diff --git a/src/artifacts/AsarArtifactReader.test.ts b/src/artifacts/AsarArtifactReader.test.ts new file mode 100644 index 0000000000..54eb78cd4b --- /dev/null +++ b/src/artifacts/AsarArtifactReader.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { orderAsarListing } from "./AsarArtifactReader.js"; + +describe("orderAsarListing", () => { + it("orders host-spelled listings by their portable archive path", () => { + const posix = orderAsarListing(["/a0", "/a", "/a/b"], "/"); + const windows = orderAsarListing(["\\a0", "\\a", "\\a\\b"], "\\"); + + // "/" (0x2F) precedes "0" (0x30); "\" (0x5C) would follow it. + expect(posix.map(({ path }) => path)).toEqual(["a", "a/b", "a0"]); + expect(windows.map(({ path }) => path)).toEqual(["a", "a/b", "a0"]); + expect(windows.map(({ listed }) => listed)).toEqual([ + "\\a", + "\\a\\b", + "\\a0", + ]); + }); + + it("keeps POSIX backslashes as literal filename characters", () => { + expect(orderAsarListing(["/a\\b"], "/")).toEqual([ + { listed: "/a\\b", path: "a\\b" }, + ]); + }); +}); diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index d08ebb0fe3..3e0f69a5b9 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -32,6 +32,7 @@ import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js"; import { admitAsarHeader } from "./AsarHeader.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** * Official Electron ASAR adapter with range-streamed member reads. @@ -64,7 +65,7 @@ export class AsarArtifactReader implements ArtifactReader { ) {} async *entries(signal?: AbortSignal): AsyncIterable { - let paths: string[]; + let paths: readonly AsarListedPath[]; try { await this.prepareContainer(undefined, signal); const snapshotPath = this.#snapshotPath(); @@ -82,9 +83,7 @@ export class AsarArtifactReader implements ArtifactReader { await headerHandle.close(); } abortIfNeeded(signal); - paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) => - left.localeCompare(right, "en"), - ); + paths = orderAsarListing(listPackage(snapshotPath, { isPack: false })); const archiveMetadata = await lstat(snapshotPath); if (!archiveMetadata.isFile() || archiveMetadata.isSymbolicLink()) throw new ArtifactReaderFailure( @@ -95,13 +94,8 @@ export class AsarArtifactReader implements ArtifactReader { } catch (cause: unknown) { throw asarFailure(this.path, "inventory", cause); } - for (const listed of paths) { + for (const { listed, path } of paths) { abortIfNeeded(signal); - // @electron/asar returns paths assembled with the host path module. - // Keep that spelling for its API calls, but expose archive paths with - // portable separators. On POSIX, backslashes can be literal filename - // characters and must remain untouched. - const path = toArtifactPath(listed); if (path.length === 0) continue; const providerPath = listed.startsWith(sep) ? listed.slice(sep.length) @@ -577,8 +571,30 @@ const waitForOperation = async ( }); }; -const toArtifactPath = (listed: string): string => { - const portable = sep === "\\" ? listed.replaceAll("\\", "/") : listed; +interface AsarListedPath { + readonly listed: string; + readonly path: string; +} + +/** + * Pair each host-spelled listing with its portable archive path, ordered by + * the portable path so Windows and POSIX hosts agree. + * + * @electron/asar returns paths assembled with the host path module. Keep that + * spelling for its API calls, but expose archive paths with portable + * separators. On POSIX, backslashes can be literal filename characters and + * must remain untouched. + */ +export const orderAsarListing = ( + listed: readonly string[], + separator: string = sep, +): AsarListedPath[] => + listed + .map((entry) => ({ listed: entry, path: toArtifactPath(entry, separator) })) + .sort((left, right) => compareUnicodeCodePoints(left.path, right.path)); + +const toArtifactPath = (listed: string, separator: string): string => { + const portable = separator === "\\" ? listed.replaceAll("\\", "/") : listed; return portable.replace(/^\/+|\/+$/gu, ""); }; diff --git a/src/artifacts/apple/MachoLoadCommandReader.ts b/src/artifacts/apple/MachoLoadCommandReader.ts index 561c2b84fb..8a0112fab9 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.ts @@ -526,12 +526,16 @@ const CPU_TYPE_X86 = 7; const CPU_TYPE_ARM = 12; const CPU_SUBTYPE_MASK = 0xff000000; const CPU_SUBTYPE_ARM64E = 2; +const CPU_SUBTYPE_ARM64E_X1 = 12; const CPU_SUBTYPE_X86_64_H = 8; const architectureName = (cpuType: number, cpuSubtype: number): string => { const subtype = cpuSubtype & ~CPU_SUBTYPE_MASK; if (cpuType === (CPU_TYPE_ARM | CPU_ARCH_ABI64)) - return subtype === CPU_SUBTYPE_ARM64E ? "arm64e" : "arm64"; + // Both pointer-authentication subtypes belong to the arm64e family. + return subtype === CPU_SUBTYPE_ARM64E || subtype === CPU_SUBTYPE_ARM64E_X1 + ? "arm64e" + : "arm64"; if (cpuType === (CPU_TYPE_ARM | CPU_ARCH_ABI64_32)) return "arm64_32"; if (cpuType === (CPU_TYPE_X86 | CPU_ARCH_ABI64)) return subtype === CPU_SUBTYPE_X86_64_H ? "x86_64h" : "x86_64"; diff --git a/src/artifacts/hostExecutableMarking.test.ts b/src/artifacts/hostExecutableMarking.test.ts index 7aff5bbdac..9f97186d24 100644 --- a/src/artifacts/hostExecutableMarking.test.ts +++ b/src/artifacts/hostExecutableMarking.test.ts @@ -13,6 +13,16 @@ describe("hostExecutableMarking", () => { expect( hostExecutableMarking("C:\\scratch\\SETUP.CMD", file(0o444), "win32"), ).toBe(true); + for (const path of [ + "tools/SETUP.EXE", + "run.Cmd", + "build.bat", + "legacy.com", + ]) + expect(hostExecutableMarking(path, file(0o666), "win32")).toBe(true); + expect(hostExecutableMarking("readonly.exe", file(0o444), "win32")).toBe( + true, + ); expect( hostExecutableMarking("C:\\scratch\\library.dll", file(0o666), "win32"), ).toBe(false); @@ -25,6 +35,11 @@ describe("hostExecutableMarking", () => { ).toBe(false); }); + it("does not treat other Windows filenames as executable even with execute bits", () => { + for (const path of ["notes.txt", "README", "archive.exe.bak"]) + expect(hostExecutableMarking(path, file(0o777), "win32")).toBe(false); + }); + it("keeps POSIX permission bits authoritative regardless of extension", () => { expect(hostExecutableMarking("/scratch/tool", file(0o755), "linux")).toBe( true, diff --git a/src/browser/CdpBrowserProvider.ts b/src/browser/CdpBrowserProvider.ts index 7138710775..642373af56 100644 --- a/src/browser/CdpBrowserProvider.ts +++ b/src/browser/CdpBrowserProvider.ts @@ -68,6 +68,7 @@ import { captureCdpScreenshot } from "./CdpScreenshot.js"; import { comparePngScreenshots } from "./PngVisualDiff.js"; import { CDP_BROWSER_PROVIDER_IDENTITY } from "./providerIdentities.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const CLEANUP_DOMAINS = ["Network", "Debugger", "Runtime", "Page"] as const; /** Passive browser observation through a selected user-owned CDP endpoint. */ @@ -408,7 +409,9 @@ const selectTargets = ( attached: target.attached, }); } - allowed.sort((left, right) => left.target_id.localeCompare(right.target_id)); + allowed.sort((left, right) => + compareUnicodeCodePoints(left.target_id, right.target_id), + ); return { allowed, excluded: { diff --git a/src/browser/CdpCaptureDocuments.ts b/src/browser/CdpCaptureDocuments.ts index 0b93059f2f..3c7fb98ed5 100644 --- a/src/browser/CdpCaptureDocuments.ts +++ b/src/browser/CdpCaptureDocuments.ts @@ -12,6 +12,7 @@ import { import { exclusionReasonForUrl } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; import { createDocumentSvgAncestry, domSvgLink } from "./CdpCaptureDomSvg.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; export type CapturedResource = Omit< WebPageInspection["resources"][number], @@ -346,7 +347,7 @@ const accessibilityStates = ( if (state !== undefined) states.set(name, state); } return [...states] - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([name, state]) => ({ name, value: state })); }; diff --git a/src/browser/CdpElectronInspection.ts b/src/browser/CdpElectronInspection.ts index ca49edabd1..5aa9d95b87 100644 --- a/src/browser/CdpElectronInspection.ts +++ b/src/browser/CdpElectronInspection.ts @@ -24,6 +24,7 @@ import { import { captureElectronScripts } from "./CdpElectronScripts.js"; import { captureElectronWorkers } from "./CdpElectronWorkers.js"; import { authorizedElectronFile } from "./ElectronFileScope.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; interface ElectronContext { readonly connection: CdpConnection; @@ -300,7 +301,7 @@ const captureResources = async ( pending.push(...recordsValue(tree.childFrames)); } return resources.sort((left, right) => - left.resource_key.localeCompare(right.resource_key), + compareUnicodeCodePoints(left.resource_key, right.resource_key), ); }; diff --git a/src/browser/CdpElectronProvider.ts b/src/browser/CdpElectronProvider.ts index 7d42682cb3..8257d49748 100644 --- a/src/browser/CdpElectronProvider.ts +++ b/src/browser/CdpElectronProvider.ts @@ -30,6 +30,7 @@ import { inspectCdpElectronPage } from "./CdpElectronInspection.js"; import { authorizedElectronFile } from "./ElectronFileScope.js"; import { CDP_ELECTRON_PROVIDER_IDENTITY } from "./providerIdentities.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Passive Electron provider for local file pages exposed by loopback CDP. */ export class CdpElectronProvider implements ElectronObservationPort { identity(): ProviderIdentity { @@ -73,7 +74,7 @@ export class CdpElectronProvider implements ElectronObservationPort { }); } allowed.sort((left, right) => - left.target_id.localeCompare(right.target_id), + compareUnicodeCodePoints(left.target_id, right.target_id), ); return ok( electronTargetListSchema.parse({ diff --git a/src/browser/CdpElectronScripts.ts b/src/browser/CdpElectronScripts.ts index 850cfd5f54..854b49413f 100644 --- a/src/browser/CdpElectronScripts.ts +++ b/src/browser/CdpElectronScripts.ts @@ -9,6 +9,7 @@ import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; import { requiredScriptSource } from "./CdpCaptureValues.js"; import type { ElectronScriptDraft } from "./CdpElectronScriptEvents.js"; import { authorizedElectronFile } from "./ElectronFileScope.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; interface ElectronScriptCaptureInput { readonly connection: CdpConnection; @@ -65,7 +66,7 @@ export const captureElectronScripts = async ( return { total, items: items.sort((left, right) => - left.script_key.localeCompare(right.script_key), + compareUnicodeCodePoints(left.script_key, right.script_key), ), }; }; diff --git a/src/browser/CdpElectronWorkers.ts b/src/browser/CdpElectronWorkers.ts index 080e1e46a2..b4e3d102f7 100644 --- a/src/browser/CdpElectronWorkers.ts +++ b/src/browser/CdpElectronWorkers.ts @@ -12,6 +12,7 @@ import { import type { CdpEndpointTarget } from "./CdpEndpoint.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; import { authorizedElectronFile } from "./ElectronFileScope.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Inventory local worker targets without attaching to them. */ export const captureElectronWorkers = async (input: { @@ -70,7 +71,7 @@ export const captureElectronWorkers = async (input: { }); } return workers.sort((left, right) => - left.target_id.localeCompare(right.target_id), + compareUnicodeCodePoints(left.target_id, right.target_id), ); }; diff --git a/src/browser/CdpPageCaptureScripts.ts b/src/browser/CdpPageCaptureScripts.ts index ffafa02e72..4108484340 100644 --- a/src/browser/CdpPageCaptureScripts.ts +++ b/src/browser/CdpPageCaptureScripts.ts @@ -13,6 +13,7 @@ import { } from "./CdpCaptureValues.js"; import type { CaptureContext } from "./CdpPageCapture.js"; import type { WebSourceMapRequest } from "./WebSourceMapFetcher.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; export interface CapturedScripts { readonly inventory: WebPageInspection["scripts"]; @@ -131,11 +132,15 @@ const keyAndBuildInventory = ( })); keyed.sort( (left, right) => - left.base.localeCompare(right.base) || - sourceDigest(left.item.source).localeCompare( + compareUnicodeCodePoints(left.base, right.base) || + compareUnicodeCodePoints( + sourceDigest(left.item.source), sourceDigest(right.item.source), ) || - (left.item.frame_id ?? "").localeCompare(right.item.frame_id ?? ""), + compareUnicodeCodePoints( + left.item.frame_id ?? "", + right.item.frame_id ?? "", + ), ); const totals = new Map(); for (const { base } of keyed) totals.set(base, (totals.get(base) ?? 0) + 1); diff --git a/src/browser/CdpWebMcpDiscovery.ts b/src/browser/CdpWebMcpDiscovery.ts index dd4b3fb6cf..b414bdd89c 100644 --- a/src/browser/CdpWebMcpDiscovery.ts +++ b/src/browser/CdpWebMcpDiscovery.ts @@ -7,7 +7,10 @@ import type { } from "../domain/webMcpDiscovery.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { digestCanonicalValue } from "../domain/canonicalDigest.js"; -import { compositeKey } from "../domain/unicodeCodePointOrder.js"; +import { + compareUnicodeCodePoints, + compositeKey, +} from "../domain/unicodeCodePointOrder.js"; import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js"; import type { CdpConnection, CdpEvent } from "./CdpConnection.js"; import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; @@ -521,7 +524,7 @@ const buildWebMcpResult = (options: WebMcpResultOptions): WebMcpDiscovery => { tools: { total: tools.size + completeness.droppedTotal, items: [...tools.values()].sort((left, right) => - left.tool_key.localeCompare(right.tool_key), + compareUnicodeCodePoints(left.tool_key, right.tool_key), ), }, completeness: completeness.snapshot(), diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index a6141ee18c..408ec68e09 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -6,9 +6,10 @@ import { z } from "zod"; import { PROMPT_CONTRACTS } from "./contracts/promptContracts.js"; import { TOOL_CONTRACTS } from "./contracts/toolContracts.js"; import { CLI_COMMAND_NAMES } from "./cliCommandNames.js"; +import { compareUnicodeCodePoints } from "./domain/unicodeCodePointOrder.js"; const sortedToolContracts = [...TOOL_CONTRACTS].sort((left, right) => - left.name.localeCompare(right.name), + compareUnicodeCodePoints(left.name, right.name), ); const createToolCatalog = () => @@ -39,7 +40,7 @@ const promptCatalog = PROMPT_CONTRACTS.map((contract) => ({ description: contract.description, arguments: contract.arguments, steps: contract.steps, -})).sort((left, right) => left.name.localeCompare(right.name)); +})).sort((left, right) => compareUnicodeCodePoints(left.name, right.name)); const digest = (value: unknown): string => { const encoded = canonicalize(value); diff --git a/src/cli/setupCommands.ts b/src/cli/setupCommands.ts index d036f708a4..d21be1c087 100644 --- a/src/cli/setupCommands.ts +++ b/src/cli/setupCommands.ts @@ -263,6 +263,9 @@ const setupRunOptions = ( ? { readinessScope: { clients: options.client, + ...(options.skillClient.length > 0 + ? { skillClients: options.skillClient } + : {}), providers: options.installHopper ? ["hopper"] : [], skill: options.skill ?? agentIntegrationSelected, }, diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts new file mode 100644 index 0000000000..f5043d2408 --- /dev/null +++ b/src/contracts/toolEffects.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; + +import { TOOL_CONTRACTS } from "./toolContracts.js"; +import { TOOL_EFFECTS, annotationsFromEffects } from "./toolEffects.js"; + +/** + * Descriptions that name running, launching, executing, starting, installing, + * or uninstalling something. A matching tool must declare foreign execution or + * device mutation unless it is listed below with the reason the wording does + * not describe that effect. + */ +const EXECUTION_WORDING = + /\b(?:run|runs|running|launch|launches|launching|execute|executes|executing|execution|start|starts|install|installs|uninstall|uninstalls)\b/iu; + +const EXECUTION_WORDING_EXCEPTIONS: Readonly> = { + inspect_native_load_image: "states that it does not execute the target", + resolve_native_call_targets: "states that it does not execute code", + inspect_pe_resources: "states that it does not execute the image", + inspect_binary_layout: + "parses the binary offline with operator-configured pwntools", + inspect_recorded_crash: "inspects a core file without launching its target", + inspect_macho: "runs host Mach-O inspection tools, not the target", + inspect_evm_interface: + "decodes bytecode in REA's bundled worker without executing it", + inspect_wasm_artifact: + "WABT validates and decompiles the module without executing it", + inspect_managed_artifact: "reads metadata without executing target code", + inspect_managed_members: "reads metadata without executing target code", + inspect_managed_native_boundaries: + "reads metadata without executing target code", + verify_managed_native_boundaries: "states it does not execute managed code", + project_managed_application_graph: "states it does not execute managed code", + inspect_android_package: "states it does not execute the APK", + search_android_classes: "states it does not execute the APK", + inspect_apktool_client: + "inspects the operator-selected apktool launcher; never installs it", + inspect_jeb_client: "talks to a caller-started JEB; never launches it", + open_jeb_project: "the engine analyzes the file; REA does not execute it", + inspect_adb_client: + "inspects the operator-selected adb client; never starts its server", + list_adb_devices: "may start the local adb server, which REA does not own", + inspect_firmware_regions: "states there is no target execution", + extract_firmware: "states that extracted code is never executed", + analyze_javascript_application: "reconstructs the graph without executing", + reconcile_javascript_runtime: "never invents execution claims", + observe_javascript_runtime: "never evaluates or resumes the target", + trace_application_feature: "never executes the application", + trace_javascript_semantics: "static reachability never claims execution", + compare_javascript_export_shapes: "states it does not execute JavaScript", + project_android_application_graph: "execution-free inventory", + project_apple_application_graph: "execution-free inventory", + project_harmony_application_graph: "execution-free inventory", + trace_web_source_location: "states there is no application execution", + trace_web_module_imports: + "runs only REA's resolver stub in operator-configured Chromium", + export_web_scripts: "states there is no refetch or execution", + observe_web_execution: + "instruments an externally driven page; it does not run code itself", + recover_javascript_sources: "states there is no application code execution", + goto_address: "start names an object boundary, not a launch", + prev_address: "start names an object boundary, not a launch", + trace_call_path: "start names the traversal origin", + observe_native_ui: "observes an already-running app without launching it", + inspect_adb_package: "reports install times without changing the device", + list_adb_processes: "lists running processes without changing the device", + inspect_web_event_listeners: "states it does not evaluate JavaScript", +}; + +const declaresForeignAuthority = (name: string): boolean => { + const effects = TOOL_EFFECTS[name]; + return ( + effects !== undefined && + (effects.executesForeignCode || effects.mutatesDevice) + ); +}; + +describe("tool effects", () => { + it("audits every public tool exactly once", () => { + expect(Object.keys(TOOL_EFFECTS).sort()).toEqual( + TOOL_CONTRACTS.map(({ name }) => name).sort(), + ); + }); + + it("keeps execution and device wording aligned with declared effects", () => { + const undeclared = TOOL_CONTRACTS.filter( + ({ name, description }) => + EXECUTION_WORDING.test(description) && + !declaresForeignAuthority(name) && + EXECUTION_WORDING_EXCEPTIONS[name] === undefined, + ).map(({ name }) => name); + expect(undeclared).toEqual([]); + const staleExceptions = Object.keys(EXECUTION_WORDING_EXCEPTIONS).filter( + (name) => { + const contract = TOOL_CONTRACTS.find( + (candidate) => candidate.name === name, + ); + return ( + contract === undefined || + !EXECUTION_WORDING.test(contract.description) || + declaresForeignAuthority(name) + ); + }, + ); + expect(staleExceptions).toEqual([]); + }); + + it("marks foreign execution and device mutation as destructive", () => { + const destructive = TOOL_CONTRACTS.filter( + ({ effects }) => effects.executesForeignCode || effects.mutatesDevice, + ); + expect(destructive.map(({ name }) => name).sort()).toEqual([ + "capture_browser_scenario", + "capture_electron_scenario", + "capture_native_ui_scenario", + "capture_process_scenario", + "install_adb_package", + "observe_native_calls", + "push_adb_file", + "start_adb_activity", + "start_adb_app", + "stop_adb_app", + "uninstall_adb_package", + ]); + for (const contract of destructive) + expect(contract.annotations, contract.name).toMatchObject({ + readOnlyHint: false, + destructiveHint: true, + }); + }); + + it("derives destructiveHint from each destructive effect alone", () => { + const base = TOOL_EFFECTS["binary_session"]; + if (base === undefined) throw new Error("binary_session is unaudited"); + expect(annotationsFromEffects(base)).toEqual({ + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + for (const flag of [ + "mayDiscardData", + "executesForeignCode", + "mutatesDevice", + ] as const) + expect(annotationsFromEffects({ ...base, [flag]: true }), flag).toEqual({ + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: false, + }); + }); + + it("publishes the advertised annotations derived from canonical effects", () => { + for (const contract of TOOL_CONTRACTS) + expect(contract.annotations, contract.name).toEqual( + annotationsFromEffects(contract.effects), + ); + }); +}); diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 4910e3c905..90edfaf18d 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -7,7 +7,17 @@ export interface ToolEffects { readonly launchesProcess: boolean; readonly accessesNetwork: boolean; readonly changesUiState: boolean; + /** Can overwrite, clear, or drop state that the caller cannot recover from REA. */ readonly mayDiscardData: boolean; + /** + * Runs or drives code REA does not own on the host: the analysis target, a + * caller-selected executable, or the application loaded into one. Running + * REA's own pinned helpers or operator-configured analysis engines does not + * count. + */ + readonly executesForeignCode: boolean; + /** Changes an attached Android device: packages, files, or running apps. */ + readonly mutatesDevice: boolean; readonly idempotent: boolean; } @@ -27,6 +37,8 @@ const effects = (overrides: Partial = {}): ToolEffects => ({ accessesNetwork: false, changesUiState: false, mayDiscardData: false, + executesForeignCode: false, + mutatesDevice: false, idempotent: true, ...overrides, }); @@ -178,6 +190,7 @@ export const TOOL_EFFECTS: Readonly> = { launchesProcess: true, writesFilesystem: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), capture_adb_screen: effects({ @@ -239,6 +252,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), uninstall_adb_package: effects({ @@ -246,6 +260,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), start_adb_app: effects({ @@ -253,6 +268,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), start_adb_activity: effects({ @@ -260,6 +276,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), stop_adb_app: effects({ @@ -267,6 +284,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, accessesNetwork: true, + mutatesDevice: true, idempotent: false, }), search_android_classes: effects({ @@ -294,6 +312,7 @@ export const TOOL_EFFECTS: Readonly> = { mutatesSession: true, launchesProcess: true, writesFilesystem: true, + mayDiscardData: true, }), inspect_native_load_image: evidence, inspect_native_dispatch_metadata: evidence, @@ -330,11 +349,31 @@ export const TOOL_EFFECTS: Readonly> = { resolve_containing_procedure: evidence, search_procedures: evidence, search_strings: evidence, - set_address_name: effects({ mutatesTarget: true, mutatesSession: true }), - set_addresses_names: effects({ mutatesTarget: true, mutatesSession: true }), - set_bookmark: effects({ mutatesTarget: true, mutatesSession: true }), - set_comment: effects({ mutatesTarget: true, mutatesSession: true }), - set_inline_comment: effects({ mutatesTarget: true, mutatesSession: true }), + set_address_name: effects({ + mutatesTarget: true, + mutatesSession: true, + mayDiscardData: true, + }), + set_addresses_names: effects({ + mutatesTarget: true, + mutatesSession: true, + mayDiscardData: true, + }), + set_bookmark: effects({ + mutatesTarget: true, + mutatesSession: true, + mayDiscardData: true, + }), + set_comment: effects({ + mutatesTarget: true, + mutatesSession: true, + mayDiscardData: true, + }), + set_inline_comment: effects({ + mutatesTarget: true, + mutatesSession: true, + mayDiscardData: true, + }), unset_bookmark: effects({ mutatesTarget: true, mutatesSession: true, @@ -365,6 +404,7 @@ export const TOOL_EFFECTS: Readonly> = { accessesNetwork: true, changesUiState: true, mayDiscardData: true, + executesForeignCode: true, idempotent: false, }), // The owned process can change files, show UI and use the network. @@ -374,6 +414,7 @@ export const TOOL_EFFECTS: Readonly> = { launchesProcess: true, accessesNetwork: true, changesUiState: true, + executesForeignCode: true, idempotent: false, }), inspect_macho: nativeEvidence, @@ -419,6 +460,7 @@ export const TOOL_EFFECTS: Readonly> = { accessesNetwork: true, changesUiState: true, mayDiscardData: true, + executesForeignCode: true, idempotent: false, }), list_electron_targets: browserEvidence, @@ -431,6 +473,7 @@ export const TOOL_EFFECTS: Readonly> = { accessesNetwork: true, changesUiState: true, mayDiscardData: true, + executesForeignCode: true, idempotent: false, }), list_javascript_runtime_targets: browserEvidence, @@ -479,6 +522,7 @@ export const TOOL_EFFECTS: Readonly> = { writesFilesystem: true, launchesProcess: true, accessesNetwork: true, + executesForeignCode: true, idempotent: false, }), compare_process_captures: sessionEvidence, @@ -506,9 +550,15 @@ export const annotationsFromEffects = ( value.launchesProcess || value.accessesNetwork || value.changesUiState || - value.mayDiscardData + value.mayDiscardData || + value.executesForeignCode || + value.mutatesDevice ), - destructiveHint: value.mayDiscardData, + // MCP defines destructive as possibly non-additive changes to the + // environment. Running foreign code or changing a device cannot be bounded + // to additive effects, so either one is destructive even without data loss. + destructiveHint: + value.mayDiscardData || value.executesForeignCode || value.mutatesDevice, idempotentHint: value.idempotent, openWorldHint: value.launchesProcess || value.accessesNetwork, }); diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 1c4d449c89..51a53aa4f4 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -293,6 +293,8 @@ const toolAvailabilityFacts = { accessesNetwork: z.boolean(), changesUiState: z.boolean(), mayDiscardData: z.boolean(), + executesForeignCode: z.boolean(), + mutatesDevice: z.boolean(), idempotent: z.boolean(), }), annotations: z.object({ diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 71dad30c3a..389d4870ac 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -51,9 +51,9 @@ export interface AnalysisCleanupObservation { /** Optional provider-neutral context that must survive typed error projection. */ export interface AnalysisErrorOptions extends ErrorOptions { - readonly capturedOutput?: AnalysisCapturedOutput; - readonly cleanup?: AnalysisCleanupObservation; - readonly partialObservation?: AnalysisPartialObservation; + readonly capturedOutput?: AnalysisCapturedOutput | undefined; + readonly cleanup?: AnalysisCleanupObservation | undefined; + readonly partialObservation?: AnalysisPartialObservation | undefined; } /** Base class for expected analysis, provider, and session failures. */ diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index b0e81e54ee..fa159394d3 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -265,9 +265,12 @@ const hopperProcessMessage = (error: HopperProcessError): string => { const startup = error.userMessage === undefined ? "" : ` ${error.userMessage}`; if (error.providerState === "exited") { + const signal = error.launcherFailure?.signal; const exit = error.exitCode === null - ? "no exit code was observed" + ? signal === undefined || signal === null + ? "no exit code was observed" + : `signal ${signal}` : `exit code ${String(error.exitCode)}`; const recovery = error.failureCode === undefined && stage !== "launch" diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index 635d5c0219..eb1de87005 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -572,7 +572,7 @@ describe("analysis error projection: caller contract", () => { describe("analysis error projection: operational diagnostics", () => { it("distinguishes an observed Hopper exit from an undetermined provider failure", () => { const exited = projectAnalysisError( - new HopperProcessError(7, undefined, "search_strings", 4), + new HopperProcessError(7, { operation: "search_strings", requestId: 4 }), ); expect(exited).toMatchObject({ code: "provider_unavailable", @@ -590,7 +590,10 @@ describe("analysis error projection: operational diagnostics", () => { expect(exited.remediation.action).toContain("Restart the owned provider"); const unknown = projectAnalysisError( - new HopperProcessError(null, undefined, "search_strings", 5), + new HopperProcessError(null, { + operation: "search_strings", + requestId: 5, + }), ); expect(unknown.details).toMatchObject({ stage: "analysis", diff --git a/src/domain/hopperErrors.ts b/src/domain/hopperErrors.ts index f2b37a8a5a..2ec3135c70 100644 --- a/src/domain/hopperErrors.ts +++ b/src/domain/hopperErrors.ts @@ -1,4 +1,7 @@ -import { AnalysisError } from "./analysisErrorBase.js"; +import { + AnalysisError, + type AnalysisErrorOptions, +} from "./analysisErrorBase.js"; import type { HopperProcessProviderState, ProviderFailureStage, @@ -73,10 +76,10 @@ export type HopperDiagnosticType = | "bridge_exception"; /** Hopper's JSON-RPC endpoint returned an expected remote error response. */ -export interface HopperRemoteErrorContext { - readonly diagnosticType?: HopperDiagnosticType; - readonly operation?: string; - readonly requestId?: number; +export interface HopperRemoteErrorContext extends AnalysisErrorOptions { + readonly diagnosticType?: HopperDiagnosticType | undefined; + readonly operation?: string | undefined; + readonly requestId?: number | undefined; } export class HopperRemoteError extends HopperError { @@ -90,7 +93,7 @@ export class HopperRemoteError extends HopperError { readonly safeMessage: string, context: HopperRemoteErrorContext = {}, ) { - super(`Hopper request failed (${String(code)}): ${safeMessage}`); + super(`Hopper request failed (${String(code)}): ${safeMessage}`, context); this.diagnosticType = context.diagnosticType ?? "remote"; this.operation = context.operation; this.requestId = context.requestId; @@ -98,35 +101,50 @@ export class HopperRemoteError extends HopperError { } /** The owned Hopper bridge stopped before its client was closed. */ +export interface HopperProcessErrorContext extends AnalysisErrorOptions { + readonly diagnostic?: HopperStartupFailureDiagnostic | undefined; + readonly operation?: string | undefined; + readonly requestId?: number | undefined; + readonly providerState?: HopperProcessProviderState | undefined; + readonly stage?: ProviderFailureStage | undefined; + readonly launcherFailure?: HopperLauncherOutcome | undefined; +} + export class HopperProcessError extends HopperError { readonly _tag = "HopperProcessError"; readonly failureCode: HopperStartupFailureCode | undefined; override readonly userMessage: string | undefined; readonly providerState: HopperProcessProviderState; readonly stage: ProviderFailureStage; + readonly diagnostic: HopperStartupFailureDiagnostic | undefined; + readonly operation: string | undefined; + readonly requestId: number | undefined; + readonly launcherFailure: HopperLauncherOutcome | undefined; constructor( readonly exitCode: number | null, - readonly diagnostic?: HopperStartupFailureDiagnostic, - readonly operation?: string, - readonly requestId?: number, - providerState?: HopperProcessProviderState, - stage?: ProviderFailureStage, - readonly launcherFailure?: HopperLauncherOutcome, + context: HopperProcessErrorContext = {}, ) { - super(`Hopper bridge stopped unexpectedly with code ${String(exitCode)}`); + super( + `Hopper bridge stopped unexpectedly with code ${String(exitCode)}`, + context, + ); + this.diagnostic = context.diagnostic; + this.operation = context.operation; + this.requestId = context.requestId; + this.launcherFailure = context.launcherFailure; const failure = hopperStartupFailure(exitCode); this.stage = - stage ?? + context.stage ?? (failure === undefined - ? operation === undefined + ? context.operation === undefined ? "connection" : "analysis" : "launch"); this.failureCode = failure?.code; this.userMessage = failure?.message; this.providerState = - providerState ?? (exitCode === null ? "unknown" : "exited"); + context.providerState ?? (exitCode === null ? "unknown" : "exited"); } } diff --git a/src/domain/javascript/electronModuleInterop.test.ts b/src/domain/javascript/electronModuleInterop.test.ts new file mode 100644 index 0000000000..cca3a20c89 --- /dev/null +++ b/src/domain/javascript/electronModuleInterop.test.ts @@ -0,0 +1,662 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { ROLLDOWN_MODULE_INTEROP } from "./javascriptModuleInterop.fixture.js"; + +const WINDOW = + 'new electron.BrowserWindow({webPreferences:{preload:"./preload.js"}});'; +const IMPORT = 'let electron = require("electron");'; +const WRAP = "electron = __toESM(electron, 1);"; +// The upstream runtime uses control-body braces that its bundled printer omits. +// https://github.com/rolldown/rolldown/blob/22a0ab13732d81f2559ad2b7064b688699324889/crates/rolldown/src/runtime/runtime-base.js +const BLOCK_MODULE_INTEROP = ROLLDOWN_MODULE_INTEROP.replace( + 'if (from && typeof from === "object" || typeof from === "function") for', + 'if (from && typeof from === "object" || typeof from === "function") { for', +) + .replace( + "if (!__hasOwnProp.call(to, key) && key !== except) __defProp", + "if (!__hasOwnProp.call(to, key) && key !== except) { __defProp", + ) + .replace(" });", " });\n }") + .replace(" return to;", " }\n return to;"); + +describe("Electron module interop identity", () => { + it("preserves named APIs through the released Rolldown namespace conversion", () => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + ${IMPORT} ${WRAP} ${WINDOW} + new electron.BrowserWindow({webPreferences:{preload:runtimePath}}); + electron.ipcMain.handle("ping", () => "pong"); + electron.contextBridge.exposeInMainWorld("api", {ping:true}); + `); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: "./preload.js" }, + { preload_path: null }, + ]); + expect(analysis.electron.ipc).toMatchObject([ + { channel: "ping", side: "main" }, + ]); + expect(analysis.electron.context_bridge_apis).toMatchObject([ + { api_key: "api" }, + ]); + }); + + it("follows lexical identities when helper and parameter names change", () => { + const source = `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}` + .replaceAll("__toESM", "namespaceOf") + .replaceAll("__copyProps", "copyNamed") + .replaceAll("mod", "inputModule") + .replaceAll("isNodeMode", "nodeMode") + .replaceAll("target", "namespaceTarget") + .replaceAll("from", "sourceObject"); + expect( + analyzeJavaScriptStaticSource(source).electron.browser_windows, + ).toMatchObject([{ preload_path: "./preload.js" }]); + }); + + it("preserves the same operations through control-body braces", () => { + expect( + analyzeJavaScriptStaticSource( + `${BLOCK_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}`, + ).electron.browser_windows, + ).toMatchObject([{ preload_path: "./preload.js" }]); + }); + + it("keeps additional operations inside a control block unresolved", () => { + const altered = BLOCK_MODULE_INTEROP.replace( + "{ __defProp", + "{ from = unrelated; __defProp", + ); + expect( + analyzeJavaScriptStaticSource(`${altered}\n${IMPORT} ${WRAP} ${WINDOW}`) + .electron.browser_windows, + ).toEqual([]); + }); +}); + +const BOUND_INTRINSIC_MUTATIONS: [string, string][] = [ + [ + "intrinsic changed through aliased prebound arguments", + `const objectAlias = Object; const key = "create"; const define = Object.defineProperty; const mutate = define.bind(null, objectAlias, key); mutate({value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed through nested prebinding", + `const first = Object.defineProperty.bind(null, Object); const second = first.bind(null, "create"); second({value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "a replaced bound mutator bind method is unknown", + `const first = Object.defineProperty.bind(null, Object); first.bind = customBind; const second = first.bind(null, "create"); second({value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed through a computed prebound call", + `const mutate = Object.defineProperty.bind(null, Object, "create"); mutate["call"](null, {value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed through a computed prebound apply", + `const mutate = Object.defineProperty.bind(null, Object, "create"); mutate["apply"](null, [{value:customCreate}]); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "a replaced bind intrinsic makes a prebound mutator unknown", + `Function.prototype.bind = customBind; const mutate = Object.defineProperty.bind(null, Object, "create"); mutate({value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], +]; + +describe("Electron module interop uncertainty", () => { + it.each([ + ...BOUND_INTRINSIC_MUTATIONS, + [ + "intrinsic changed through an unresolved mutator target", + `Object.defineProperty(getTarget(), "create", {value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via dynamic apply", + `const args = [Object, "getOwnPropertyNames", {value:() => []}]; Object.defineProperty.apply(null, args); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via spread call", + `const args = [Object, "getOwnPropertyNames", {value:() => []}]; Object.defineProperty.call(null, ...args); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via Reflect.apply", + `const args = [Object, "getOwnPropertyNames", {value:() => []}]; Reflect.apply(Object.defineProperty, null, args); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via spread Reflect.apply", + `const args = [null, [Object, "create", {value:customCreate}]]; Reflect.apply(Object.defineProperty, ...args); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via call", + `Object.defineProperty.call(null, Object, "getOwnPropertyNames", {value:() => []}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed via apply", + `Object.defineProperty.apply(null, [Object, "getOwnPropertyNames", {value:() => []}]); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed with defineProperty", + `Object.defineProperty(Object, "getOwnPropertyNames", {value:() => []}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed with Reflect.set", + `Reflect.set(Object, "getOwnPropertyNames", () => []); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed with an aliased mutator", + `const change = Object.defineProperty; change(Object, "create", {value:customCreate}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed with assign", + `Object.assign(Object, {getOwnPropertyNames:() => []}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "intrinsic changed with defineProperties", + `Object.defineProperties(Function.prototype, {bind:{value:customBind}}); ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "overwritten Object global", + `Object = customObject; ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "overwritten intrinsic through an alias", + `const objectAlias = Object; objectAlias.create = customCreate; ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "overwritten intrinsic through globalThis", + `globalThis.Object.create = customCreate; ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "helper called before initialization", + `${IMPORT} ${WRAP} ${ROLLDOWN_MODULE_INTEROP}`, + ], + [ + "fake helper", + `${ROLLDOWN_MODULE_INTEROP}\n__toESM = (mod) => ({BrowserWindow:Fake}); ${IMPORT} ${WRAP}`, + ], + [ + "changed getter", + `${ROLLDOWN_MODULE_INTEROP.replace("from[k]", "to[k]")}\n${IMPORT} ${WRAP}`, + ], + [ + "replaced copy helper", + `${ROLLDOWN_MODULE_INTEROP}\n__copyProps = () => ({}); ${IMPORT} ${WRAP}`, + ], + [ + "replaced intrinsic alias", + `${ROLLDOWN_MODULE_INTEROP}\n__defProp = () => ({}); ${IMPORT} ${WRAP}`, + ], + [ + "shadowed Object", + `const Object = customObject; ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "overwritten intrinsic", + `Object.create = customCreate; ${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP}`, + ], + [ + "overwritten alias call", + `${ROLLDOWN_MODULE_INTEROP}\n__hasOwnProp.call = customCall; ${IMPORT} ${WRAP}`, + ], + [ + "overwritten Function bind", + `${ROLLDOWN_MODULE_INTEROP}\nFunction.prototype.bind = customBind; ${IMPORT} ${WRAP}`, + ], + [ + "conditional conversion", + `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} if (flag) { ${WRAP} }`, + ], + [ + "compound assignment", + `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} electron ||= __toESM(electron, 1);`, + ], + [ + "other input", + `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} electron = __toESM(other, 1);`, + ], + [ + "later overwrite", + `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} electron = unrelated;`, + ], + [ + "overwritten constructor", + `${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} electron.BrowserWindow = Fake;`, + ], + ])("does not infer Electron identity after %s", (_name, setup) => { + expect( + analyzeJavaScriptStaticSource(`${setup}\n${WINDOW}`).electron + .browser_windows, + ).toEqual([]); + }); +}); + +describe("Electron module interop helper results", () => { + it.each([ + 'const {BrowserWindow} = __toESM(require("electron"), 1); new BrowserWindow({});', + 'const {BrowserWindow: Window} = __toESM(require("electron"), 1); new Window({});', + 'const {default: {BrowserWindow}} = __toESM(require("electron"), 1); new BrowserWindow({});', + 'const Window = __toESM(require("electron"), 1).BrowserWindow; new Window({});', + ])("preserves a proven result projection: %s", (source) => { + expect( + analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP}\n${source}`) + .electron.browser_windows, + ).toHaveLength(1); + }); + + it.each([ + 'const {[key]: Window} = __toESM(require("electron"), 1); new Window({});', + 'const {...electron} = __toESM(require("electron"), 1); new electron.BrowserWindow({});', + 'let {BrowserWindow} = __toESM(require("electron"), 1); BrowserWindow = Fake; new BrowserWindow({});', + ])("keeps an uncertain result projection unresolved: %s", (source) => { + expect( + analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP}\n${source}`) + .electron.browser_windows, + ).toEqual([]); + }); + + it.each([ + "", + ", 0", + ", false", + ", true", + ", 1", + ', "node"', + ', ""', + ", null", + ])("preserves named exports with a literal node mode: %s", (mode) => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + const fresh = __toESM(require("electron")${mode}); + new fresh.BrowserWindow({}); + const {BrowserWindow} = __toESM(require("electron")${mode}); + new BrowserWindow({}); + let electron = require("electron"); + electron = __toESM(electron${mode}); ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(3); + }); + + it.each(["", ", 0", ", false"])( + "does not assume a module-valued default with node mode: %s", + (mode) => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + const original = require("electron"); + original.__esModule = true; + original.default = {BrowserWindow: Fake}; + const {default: {BrowserWindow}} = __toESM(require("electron")${mode}); + new BrowserWindow({}); + const other = __toESM(require("electron")${mode}).default; + new other.BrowserWindow({}); + new (__toESM(require("electron")${mode}).default.BrowserWindow)({}); + `); + expect(analysis.electron.browser_windows).toEqual([]); + }, + ); + + it.each(["__toESM", "identity"])( + "rejects side-effecting extra arguments to %s", + (helper) => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + function identity(mod) { return mod; } + const electron = ${helper}(require("electron"), mutate()); ${WINDOW} + `); + expect(analysis.electron.browser_windows).toEqual([]); + }, + ); + + it.each([ + "const __toESM = () => ({BrowserWindow:FakeWindow});", + "const __toESM = async (mod) => mod;", + "function* __toESM(mod) { return mod; }", + "const __toESM = (this && this.__toESM) || ((mod) => mod);", + ])("keeps an unproven fresh result unresolved: %s", (helper) => { + const analysis = analyzeJavaScriptStaticSource(`${helper} + const electron = __toESM(require("electron"), 1); ${WINDOW} + new (__toESM(require("electron"), 1)).BrowserWindow({}); + `); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it("preserves a proven fresh namespace and a pure identity helper", () => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + const electron = __toESM(require("electron"), 1); ${WINDOW} + new (__toESM(require("electron"), 1)).BrowserWindow({}); + function passthrough(value) {return value;} + const plain = passthrough(require("electron")); + new plain.BrowserWindow({}); + `); + expect(analysis.electron.browser_windows).toHaveLength(3); + }); +}); + +describe("Electron module interop pattern writes", () => { + it.each([ + "({Object} = replacements);", + "[Function] = replacements;", + "({nested:{Object}} = replacements);", + "({replacement:Object = customObject} = replacements);", + "({...Object} = replacements);", + "[...Function] = replacements;", + "for ({Object} of replacements) {}", + "for ([Function] in replacements) {}", + ])("rejects an intrinsic replaced through %s", (write) => { + const analysis = analyzeJavaScriptStaticSource(` + ${write} ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it.each([ + "({Object:unrelated} = replacements);", + "function decoy(Object) {({Object} = replacements);}", + ])("preserves intrinsic identity for an unrelated target: %s", (write) => { + const analysis = analyzeJavaScriptStaticSource(` + ${write} ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); + + it("preserves captured operations after later property destructuring", () => { + const analysis = analyzeJavaScriptStaticSource(` + ${ROLLDOWN_MODULE_INTEROP} ({create:Object.create} = replacements); + ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); +}); + +describe("Electron module interop intrinsic captures", () => { + it.each([ + "var captured = Object.defineProperty;", + "const {defineProperty:captured} = Object;", + "var first = Object.defineProperty; const captured = first;", + ])("retains the original function capture through %s", (capture) => { + const runtime = ROLLDOWN_MODULE_INTEROP.replace( + "__defProp = Object.defineProperty", + "__defProp = captured", + ); + const analysis = analyzeJavaScriptStaticSource(` + ${capture} Object.defineProperty = custom; + ${runtime} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); + + it.each(["var", "const"])( + "keeps an intrinsic alias read before its %s initializer unresolved", + (kind) => { + const runtime = ROLLDOWN_MODULE_INTEROP.replace( + "__defProp = Object.defineProperty", + "__defProp = later", + ); + const analysis = analyzeJavaScriptStaticSource(` + ${runtime} ${kind} later = Object.defineProperty; + ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toEqual([]); + }, + ); + + it("requires the helper's intrinsic binding to initialize before conversion", () => { + const runtime = ROLLDOWN_MODULE_INTEROP.replace( + "var __defProp = Object.defineProperty;", + "", + ); + const analysis = analyzeJavaScriptStaticSource(` + var captured = Object.defineProperty; + ${runtime} ${IMPORT} ${WRAP} ${WINDOW} + var __defProp = captured; + `); + expect(analysis.electron.browser_windows).toEqual([]); + }); +}); + +describe("Electron module interop capture and scope", () => { + it.each(["metadata", "alias"])( + "recognizes a hoisted function target through %s", + (target) => { + const analysis = analyzeJavaScriptStaticSource(` + const alias = metadata; + Object.defineProperty(${target}, "x", {value:1}); + function metadata() {} + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }, + ); + + it.each([ + "var metadata = function () {};", + "const metadata = () => {};", + "class metadata {}", + ])("does not hoist fresh targets from %s", (declaration) => { + const analysis = analyzeJavaScriptStaticSource(` + Object.defineProperty(metadata, "x", {value:1}); + ${declaration} + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it.each([ + 'const {defineProperty:change} = Object; change(Object,"create",{value:customCreate});', + "const objectAlias = Object; const {assign:change} = objectAlias; change(Object,{create:customCreate});", + 'const {set:change} = Reflect; change(Object,"create",customCreate);', + 'const {Object:{defineProperty:change}} = globalThis; change(Object,"create",{value:customCreate});', + ])("rejects a projected intrinsic mutator alias: %s", (write) => { + const source = `${write} ${ROLLDOWN_MODULE_INTEROP} + ${IMPORT} ${WRAP} ${WINDOW} + `.replaceAll("__toESM", "namespaceOf"); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it.each([ + "const metadata = {};", + "const metadata = [];", + "const initial = {}; const metadata = initial;", + "const metadata = () => {};", + "const metadata = function () {};", + "const metadata = async () => {};", + "const metadata = function* () {};", + "const metadata = class {};", + "const metadata = /local/;", + "function metadata() {}", + "class metadata {}", + ])( + "keeps a fresh mutator target separate from intrinsic identity: %s", + (setup) => { + const analysis = analyzeJavaScriptStaticSource(` + ${setup} Object.defineProperty(metadata,"x",{value:1}); + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }, + ); + + it.each([ + "const metadata = getMetadata();", + "let metadata = {}; metadata = Object;", + "Object.defineProperty(metadata,'x',{value:1}); const metadata = {};", + "const metadata = {intrinsic:Object}; Object.defineProperty(metadata.intrinsic,'create',{value:customCreate});", + "const metadata = class {static {Object.create = customCreate;}};", + "@(() => Object) class metadata {}", + ])("keeps an unproven mutator target unresolved: %s", (setup) => { + const analysis = analyzeJavaScriptStaticSource(` + ${setup} Object.defineProperty(metadata,"x",{value:1}); + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it("does not reject unrelated intrinsic property definitions", () => { + const analysis = analyzeJavaScriptStaticSource(` + Object.defineProperty(Object, "unrelated", {value:1}); + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); + + it("keeps opaque targets in a generic helper separate from outer module identity", () => { + const analysis = analyzeJavaScriptStaticSource(` + function genericAssign() { Object.assign.apply(null, arguments); } + ${ROLLDOWN_MODULE_INTEROP} ${IMPORT} ${WRAP} ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); + + it("rejects an opaque intrinsic mutation in the namespace's own callable", () => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + function openWindow(args) { + Object.defineProperty.apply(null, args); + ${IMPORT} ${WRAP} ${WINDOW} + } + `); + expect(analysis.electron.browser_windows).toEqual([]); + }); + + it("preserves captured intrinsics across later vendor polyfills and callable writes", () => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + Object.create = customCreate; + ${IMPORT} ${WRAP} + function runLoop(key) { globalThis[key] = this; } + function polyfill() { Object.create = customCreate; } + ${WINDOW} + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); + + it("keeps a shadowed namespace separate from the converted module", () => { + const analysis = analyzeJavaScriptStaticSource(`${ROLLDOWN_MODULE_INTEROP} + ${IMPORT} ${WRAP} ${WINDOW} + function decoy(electron) { ${WINDOW} } + `); + expect(analysis.electron.browser_windows).toHaveLength(1); + }); +}); + +describe("Electron module interop generic invocations", () => { + it.each([ + 'const change = Function.prototype.call.bind(Object.defineProperty); change(null,Object,"create",{value:customCreate});', + 'const call = Function.prototype.call; const change = call.bind(Object.defineProperty); change(null,Object,"create",{value:customCreate});', + 'const change = Function.prototype.call.bind(Object.defineProperty,null,Object,"create"); change({value:customCreate});', + 'Function.prototype.call.apply(Object.defineProperty,[null,Object,"create",{value:customCreate}]);', + 'Reflect.apply(Function.prototype.call,Object.defineProperty,[null,Object,"create",{value:customCreate}]);', + 'const change = Function.prototype.apply.bind(Object.defineProperty); change(null,[Object,"create",{value:customCreate}]);', + 'Reflect.apply(Function.prototype.apply,Object.defineProperty,[null,[Object,"create",{value:customCreate}]]);', + 'const first = Function.prototype.call.bind(Object.defineProperty,null,Object,"create"); const change = first.bind(fake); change({value:customCreate});', + "const change = Function.prototype.call.bind(Object.defineProperty); change(null,...args);", + "const change = Function.prototype.apply.bind(Object.defineProperty); change(null,args);", + 'const change = Function.prototype.call.bind(Object.defineProperty,null,...args); change(metadata,"x",{value:1});', + 'Function.prototype.call.call.call(Object.defineProperty,null,Object,"create",{value:customCreate});', + 'Object.defineProperty.call.call(Object.defineProperty,null,Object,"create",{value:customCreate});', + 'Function.prototype.apply.apply(Object.defineProperty,[null,[Object,"create",{value:customCreate}]]);', + 'const change = Object.defineProperty.call.bind(Object.defineProperty); change(null,Object,"create",{value:customCreate});', + 'const change = Function.prototype.call.call.bind(Object.defineProperty,null,Object,"create"); change({value:customCreate});', + ])("follows the callable receiver of a generic invocation: %s", (write) => { + expect( + analyzeJavaScriptStaticSource( + `${write}\n${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}`, + ).electron.browser_windows, + ).toEqual([]); + }); + + it.each([ + 'const change = Function.prototype.call.bind(Object.defineProperty); change(null,metadata,"x",{value:1});', + 'const change = Function.prototype.call.bind(Object.defineProperty,null,metadata,"x"); change({value:1});', + 'Function.prototype.call.call(Object.defineProperty,null,metadata,"x",{value:1});', + 'const change = Function.prototype.apply.bind(Object.defineProperty); change(null,[metadata,"x",{value:1}]);', + 'Reflect.apply(Function.prototype.apply,Object.defineProperty,[null,[metadata,"x",{value:1}]]);', + 'Function.prototype.call.call.call(Object.defineProperty,null,metadata,"x",{value:1});', + 'Function.prototype.apply.apply(Object.defineProperty,[null,[metadata,"x",{value:1}]]);', + 'const change = Object.defineProperty.call.bind(Object.defineProperty); change(null,metadata,"x",{value:1});', + ])("preserves fresh targets through a generic invocation: %s", (write) => { + expect( + analyzeJavaScriptStaticSource( + `const metadata = {}; ${write}\n${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}`, + ).electron.browser_windows, + ).toHaveLength(1); + }); +}); + +describe("Electron module interop bound mutators", () => { + it.each([ + 'const change = Object.defineProperty.bind(null); change(Object,"create",{value:customCreate});', + 'const change = Object.defineProperty.bind(null,Object,"create"); change({value:customCreate});', + 'const first = Object.defineProperty.bind(null,Object); const change = first; change("create",{value:customCreate});', + 'const change = Object.defineProperty.bind(null,Object); change.call(null,"create",{value:customCreate});', + 'const change = Object.defineProperty.bind(null,Object); change.apply(null,["create",{value:customCreate}]);', + 'const change = Object.defineProperty.bind(null,Object); Reflect.apply(change,null,["create",{value:customCreate}]);', + 'Object.defineProperty.bind(null,Object,"create")({value:customCreate});', + 'const change = Object.defineProperty.bind(null,...args); change(Object,"create",{value:customCreate});', + 'const metadata = {}; Object.defineProperty.call(...args,metadata,"x",{value:1});', + 'const metadata = {}; Object.defineProperty.apply(...args,[metadata,"x",{value:1}]);', + 'const change = Function.prototype.call.bind(Object.defineProperty,null); change(Object,"create",{value:customCreate});', + 'const first = Function.prototype.call.bind(Object.defineProperty,null); const change = first.bind(customReceiver,Object,"create"); change({value:customCreate});', + 'const change = Function.prototype.apply.bind(Object.defineProperty,null); change([Object,"create",{value:customCreate}]);', + 'Function.prototype.call.call(Object.defineProperty,null,Object,"create",{value:customCreate});', + 'Function.prototype.apply.call(Object.defineProperty,null,[Object,"create",{value:customCreate}]);', + 'const uncurry = Function.prototype.call.bind(Function.prototype.call); uncurry(Object.defineProperty,null,Object,"create",{value:customCreate});', + 'const change = Function.prototype.bind.call(Object.defineProperty,null); change(Object,"create",{value:customCreate});', + 'const bind = Function.prototype.call.bind(Function.prototype.bind); const change = bind(Object.defineProperty,null); change(Object,"create",{value:customCreate});', + 'const change = Function.prototype.bind.apply(Object.defineProperty,[null,Object,"create"]); change({value:customCreate});', + 'const change = Reflect.apply(Function.prototype.bind,Object.defineProperty,[null,Object,"create"]); change({value:customCreate});', + 'const first = Object.defineProperty.bind(null,Object); const change = Function.prototype.bind.call(first,null,"create"); change({value:customCreate});', + 'Reflect.apply = Object.defineProperty; Reflect.apply(Object,"create",{value:customCreate});', + 'const apply = Reflect.apply; Reflect.apply = Object.defineProperty; Reflect.apply(Object,"create",{value:customCreate});', + 'const change = Reflect.set.bind(null,Object,"create"); change(customCreate);', + 'const change = Object.defineProperty.call; change.call(Object.defineProperty,null,Object,"create",{value:customCreate});', + 'const {apply:change} = Object.defineProperty; change.call(Object.defineProperty,null,[Object,"create",{value:customCreate}]);', + 'const change = Reflect.apply.call; change.call(Reflect.apply,null,Object.defineProperty,null,[Object,"create",{value:customCreate}]);', + 'Reflect.set({},"create",customCreate,Object);', + 'const metadata = {}; Reflect.set(metadata,"create",customCreate,Object);', + 'const change = Reflect.set.bind(null,{},"create",customCreate); change(Object);', + 'Reflect.apply(Reflect.set,null,[{},"create",customCreate,Object]);', + 'Reflect.set({},"create",customCreate,...receivers);', + 'Reflect.set({},"create",...valuesAndReceivers);', + "Reflect.set({},...keysValuesAndReceivers);", + 'Reflect.set({},"create",customCreate,receiver);', + ])("rejects intrinsic writes through a bound mutator: %s", (write) => { + expect( + analyzeJavaScriptStaticSource( + `${write}\n${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}`, + ).electron.browser_windows, + ).toEqual([]); + }); + + it.each([ + 'const change = Object.defineProperty.bind(null); change(metadata,"x",{value:1});', + 'const change = Object.defineProperty.bind(null,metadata,"x"); change({value:1});', + 'const first = Object.defineProperty.bind(null,metadata); const change = first.bind(null,"x"); change({value:1});', + 'const change = Function.prototype.call.bind(Object.defineProperty,null); change(metadata,"x",{value:1});', + 'const change = Function.prototype.apply.bind(Object.defineProperty,null); change([metadata,"x",{value:1}]);', + 'const change = Function.prototype.bind.call(Object.defineProperty,null,metadata,"x"); change({value:1});', + 'const bind = Function.prototype.call.bind(Function.prototype.bind); const change = bind(Object.defineProperty,null,metadata,"x"); change({value:1});', + 'const change = Function.prototype.bind.apply(Object.defineProperty,[null,metadata,"x"]); change({value:1});', + 'const change = Reflect.apply(Function.prototype.bind,Object.defineProperty,[null,metadata,"x"]); change({value:1});', + 'const apply = Reflect.apply; Reflect.apply = replacement; apply(Object.defineProperty,null,[metadata,"x",{value:1}]);', + 'const change = Object.defineProperty.bind(null,Object,"create");', + 'Object.defineProperty.call(null,metadata,"x",{value:1});', + 'Object.defineProperty.apply(null,[metadata,"x",{value:1}]);', + 'const change = Object.defineProperty.call; change.call(Object.defineProperty,null,metadata,"x",{value:1});', + 'const {apply:change} = Object.defineProperty; change.call(Object.defineProperty,null,[metadata,"x",{value:1}]);', + 'Reflect.set(metadata,"x",1,{});', + 'const receiver = {}; Reflect.set({},"x",1,receiver);', + ])("keeps a fresh bound target or unused binding separate: %s", (write) => { + expect( + analyzeJavaScriptStaticSource( + `const metadata = {}; ${write}\n${ROLLDOWN_MODULE_INTEROP}\n${IMPORT} ${WRAP} ${WINDOW}`, + ).electron.browser_windows, + ).toHaveLength(1); + }); +}); + +it("keeps unrelated exports after an aliased prebound key write", () => { + const analysis = analyzeJavaScriptStaticSource(` + const electron = require("electron"); + const key = "Unrelated"; + const mutate = Object.defineProperty.bind(null, electron, key); + mutate({value: custom}); + new electron.BrowserWindow({}); + `); + expect(analysis.electron.browser_windows).toHaveLength(1); +}); diff --git a/src/domain/javascript/electronStaticAnalysisIdentity.test.ts b/src/domain/javascript/electronStaticAnalysisIdentity.test.ts index 99a3805e88..0005a19772 100644 --- a/src/domain/javascript/electronStaticAnalysisIdentity.test.ts +++ b/src/domain/javascript/electronStaticAnalysisIdentity.test.ts @@ -77,7 +77,7 @@ describe("Electron API identity", () => { it("follows bundler namespace interop helpers without accepting other writes", () => { const analysis = analyzeJavaScriptStaticSource(` var __toESM = (mod) => mod; - var __importStar = (this && this.__importStar) || ((mod) => mod); + var __importStar = (mod) => mod; let electron = require("electron"); electron = __toESM(electron); new electron.BrowserWindow({ webPreferences: { preload: "./rolldown.js" } }); @@ -120,4 +120,53 @@ describe("Electron API identity", () => { expect(analysis.electron.browser_windows).toEqual([]); }); + + it("follows stable declaration aliases but not an assigned default parameter", () => { + const analysis = analyzeJavaScriptStaticSource(` + const direct = require("electron"); + const constAlias = direct; + new constAlias.BrowserWindow({webPreferences:{preload:"./const.js"}}); + + let letAlias = require("electron"); + new letAlias.BrowserWindow({webPreferences:{preload:"./let.js"}}); + + const { BrowserWindow } = require("electron"); + new BrowserWindow({webPreferences:{preload:"./destructured.js"}}); + + function withDefault(electron = require("./not-electron")) { + if (flag) electron = require("electron"); + new electron.BrowserWindow({webPreferences:{preload:"./default.js"}}); + } + `); + + expect( + analysis.electron.browser_windows.map(({ preload_path }) => preload_path), + ).toEqual(["./const.js", "./let.js", "./destructured.js"]); + }); + + it("preserves one initializer across variable redeclarations without accepting writes", () => { + const analysis = analyzeJavaScriptStaticSource(` + var electron; + var electron = require("electron"); + var electron; + new electron.BrowserWindow({webPreferences:{preload:"./redeclared.js"}}); + + var replaced; + var replaced = require("electron"); + replaced = fake; + new replaced.BrowserWindow({webPreferences:{preload:"./replaced.js"}}); + + var initializedTwice = require("electron"); + var initializedTwice = fake; + new initializedTwice.BrowserWindow({}); + + var conditional; + if (flag) { var conditional = require("electron"); } + new conditional.BrowserWindow({}); + `); + + expect( + analysis.electron.browser_windows.map(({ preload_path }) => preload_path), + ).toEqual(["./redeclared.js"]); + }); }); diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index f89eabb051..cb3a533fe4 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -86,16 +86,20 @@ export const semanticObjectPatternKeys = ( return key === null ? [] : [key]; }); -/** Visit member write targets in direct and destructuring assignment forms. */ -export function* semanticMemberWriteTargets( +/** Visit binding and member leaves in direct and destructuring write targets. */ +export function* semanticWriteTargets( root: t.Node, -): Generator { +): Generator { const pending = [root]; while (pending.length > 0) { const next = pending.pop(); if (next === undefined) break; const node = unwrapJavaScriptExpression(next).node; - if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) + if ( + t.isIdentifier(node) || + t.isMemberExpression(node) || + t.isOptionalMemberExpression(node) + ) yield node; else if (t.isRestElement(node)) pending.push(node.argument); else if (t.isAssignmentPattern(node)) pending.push(node.left); diff --git a/src/domain/javascript/javascriptElectronMemberWrites.ts b/src/domain/javascript/javascriptElectronMemberWrites.ts index 696bb68aaa..003b495e05 100644 --- a/src/domain/javascript/javascriptElectronMemberWrites.ts +++ b/src/domain/javascript/javascriptElectronMemberWrites.ts @@ -2,10 +2,9 @@ import * as t from "@babel/types"; import { semanticStaticPropertyKey, - semanticMemberWriteTargets, + semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; -import { SemanticPropertyPathCoverage } from "./javascriptSemanticPropertyPaths.js"; import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; /** Match Electron's supported module entry points. */ @@ -24,7 +23,7 @@ export const electronExportPath = ( /** Static identity recovery cannot establish the absence of Electron usage. */ export const ELECTRON_IDENTITY_LIMITATION = - "Electron API identity follows proven lexical module aliases, including results of namespace interop helpers (__toESM, __importStar, _interopRequireWildcard) recognized by name and a single program-scope function definition; helper bodies are not evaluated. Other reassigned bindings, dynamic selections, and visibly overwritten namespace members are unresolved; missing findings do not establish absence. Arbitrary call side effects and runtime registration are not evaluated."; + "Electron API identity follows proven lexical module aliases and structurally verified module-namespace interop. Other reassignments, dynamic selections, and visibly overwritten namespace members are unresolved; missing findings do not establish absence. Arbitrary call side effects and runtime registration are not evaluated."; /** Preserve exact property segments instead of flattening literal dotted keys. */ export const electronMemberPath = ( @@ -42,14 +41,98 @@ export const electronMemberPath = ( return { root, members: members.toReversed() }; }; -/** Index visible writes through proven Electron aliases, without executing them. */ -export function* collectElectronMemberWritesSteps( +/** A known intrinsic callable and its examined positional arguments. */ +export interface JavaScriptIntrinsicInvocation { + readonly kind: "known" | "opaque"; + /** The original bound receiver; rebinding cannot replace it. */ + readonly receiver?: t.Node; + readonly path: readonly string[]; + readonly arguments: + | readonly t.CallExpression["arguments"][number][] + | undefined; +} + +/** Index member and binding writes through caller-proven lexical identities. */ +export function* collectJavaScriptIdentityWritesSteps( program: t.Program, - origin: (root: t.Node) => readonly string[] | undefined, -): Generator { - const writes = new SemanticPropertyPathCoverage(); + origin: ( + root: t.Node, + kind: "binding" | "member" | "opaque-call", + ) => readonly string[] | undefined, + intrinsicInvocation: ( + node: t.CallExpression | t.OptionalCallExpression, + ) => JavaScriptIntrinsicInvocation | undefined, + staticString: (node: t.Node | undefined) => string | undefined, + freshObject: (node: t.Node) => boolean, + retain: (node: t.Node, path: readonly (string | null)[]) => void, +): Generator { + const objectMutators = new Set([ + "defineProperty", + "defineProperties", + "assign", + "setPrototypeOf", + ]); + const reflectMutators = new Set([ + "set", + "deleteProperty", + "defineProperty", + "setPrototypeOf", + ]); + const keyedMutators = new Set(["set", "deleteProperty", "defineProperty"]); yield* traverseJavaScriptAstSteps(program, { enter: (node) => { + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { + const invocation = intrinsicInvocation(node); + if (invocation?.kind === "opaque") { + const scope = origin(node, "opaque-call"); + if (scope !== undefined) retain(node, scope); + return; + } + const callee = invocation?.path; + const arguments_ = invocation?.arguments; + const method = callee?.at(-1); + const owner = callee?.length === 3 ? callee[1] : undefined; + if ( + (owner === "Object" && objectMutators.has(method ?? "")) || + (owner === "Reflect" && reflectMutators.has(method ?? "")) + ) { + // Reflect.set can write through its explicit receiver as well as + // dispatching through its target. Both identities need examination. + const targets = + owner === "Reflect" && + method === "set" && + arguments_ !== undefined && + (arguments_.length >= 4 || arguments_.some(t.isSpreadElement)) + ? [ + arguments_[0], + arguments_.some(t.isSpreadElement) + ? undefined + : arguments_[3], + ] + : [arguments_?.[0]]; + for (const target of targets) { + // A known mutator with an opaque argument list can overwrite any + // captured intrinsic. Do not turn an unexamined target into proof. + if (target === undefined || t.isSpreadElement(target)) { + const scope = origin(node, "opaque-call"); + if (scope !== undefined) retain(node, scope); + } + if (t.isNode(target) && !t.isSpreadElement(target)) { + const { root, members } = electronMemberPath(target); + const exported = origin(root, "member"); + if (exported !== undefined) { + const key = keyedMutators.has(method ?? "") + ? [staticString(arguments_?.[1]) ?? null] + : []; + retain(node, [...exported, ...members, ...key]); + } else if (!freshObject(target)) { + const scope = origin(node, "opaque-call"); + if (scope !== undefined) retain(node, scope); + } + } + } + } + } const target = t.isAssignmentExpression(node) ? node.left : t.isUpdateExpression(node) @@ -60,12 +143,16 @@ export function* collectElectronMemberWritesSteps( ? node.left : undefined; if (target === undefined) return; - for (const member of semanticMemberWriteTargets(target)) { - const { root, members } = electronMemberPath(member); - const exported = origin(root); - if (exported !== undefined) writes.retain([...exported, ...members]); + for (const leaf of semanticWriteTargets(target)) { + if (t.isIdentifier(leaf)) { + const exported = origin(leaf, "binding"); + if (exported !== undefined) retain(node, exported); + continue; + } + const { root, members } = electronMemberPath(leaf); + const exported = origin(root, "member"); + if (exported !== undefined) retain(node, [...exported, ...members]); } }, }); - return writes; } diff --git a/src/domain/javascript/javascriptIntrinsicInvocation.ts b/src/domain/javascript/javascriptIntrinsicInvocation.ts new file mode 100644 index 0000000000..20885069fd --- /dev/null +++ b/src/domain/javascript/javascriptIntrinsicInvocation.ts @@ -0,0 +1,115 @@ +import * as t from "@babel/types"; + +import type { JavaScriptIntrinsicInvocation } from "./javascriptElectronMemberWrites.js"; +import { + semanticStaticPropertyKey, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; + +const arrayArguments = (array: t.Node | undefined) => + t.isArrayExpression(array) && + array.elements.every((element) => t.isExpression(element)) + ? array.elements.filter((element): element is t.Expression => + t.isExpression(element), + ) + : undefined; + +const invocationArguments = ( + method: "call" | "apply", + arguments_: JavaScriptIntrinsicInvocation["arguments"], +) => + t.isSpreadElement(arguments_?.[0]) + ? undefined + : method === "apply" + ? arrayArguments(arguments_?.[1]) + : arguments_?.slice(1); + +/** + * Normalize known invocation syntax once, retaining bound argument prefixes. + * Binding a function alone is not a write; only its eventual call is examined. + */ +export const resolveJavaScriptIntrinsicInvocation = ( + node: t.CallExpression | t.OptionalCallExpression, + intrinsicPath: (node: t.Node) => readonly string[] | undefined, + boundIntrinsic: (node: t.Node) => JavaScriptIntrinsicInvocation | undefined, + intrinsicReadChanged: (node: t.Node, path: readonly string[]) => boolean, +): JavaScriptIntrinsicInvocation | undefined => { + const visited = new Set(); + let callee: t.Node = node.callee; + let arguments_: JavaScriptIntrinsicInvocation["arguments"] = node.arguments; + let receiver: t.Node | undefined; + let explicitReceiver = false; + for (;;) { + callee = unwrapJavaScriptExpression(callee).node; + if (visited.has(callee)) return undefined; + visited.add(callee); + let path = intrinsicPath(callee); + if (path !== undefined && intrinsicReadChanged(callee, path)) + return { kind: "opaque", path, arguments: undefined }; + if (path === undefined) { + if ( + t.isMemberExpression(callee) || + t.isOptionalMemberExpression(callee) + ) { + const method = semanticStaticPropertyKey( + callee.property, + callee.computed, + ); + if (method === "call" || method === "apply") { + const bound = boundIntrinsic(callee.object); + if ( + bound !== undefined && + intrinsicReadChanged(callee, [...bound.path, method]) + ) + return { kind: "opaque", path: bound.path, arguments: undefined }; + receiver = arguments_?.[0]; + explicitReceiver = true; + callee = callee.object; + arguments_ = invocationArguments(method, arguments_); + continue; + } + } + const bound = boundIntrinsic(callee); + if (bound === undefined) return undefined; + if (bound.kind === "opaque") return bound; + path = bound.path; + receiver = bound.receiver; + explicitReceiver = true; + arguments_ = + bound.arguments === undefined + ? undefined + : [...bound.arguments, ...(arguments_ ?? [])]; + } + const method = path.length > 3 ? path.at(-1) : undefined; + if (method === "call" || method === "apply") { + // These properties hold generic function values. Their receiver selects + // the callable; property-chain depth does not consume more arguments. + const callable: t.Node | undefined = explicitReceiver + ? receiver + : t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee) + ? callee.object + : undefined; + if (!t.isExpression(callable)) return undefined; + callee = callable; + receiver = arguments_?.[0]; + explicitReceiver = true; + arguments_ = invocationArguments(method, arguments_); + continue; + } + if (path.join(".") === "intrinsic.Reflect.apply") { + const callable = arguments_?.[0]; + if (!t.isExpression(callable)) return undefined; + callee = callable; + receiver = arguments_?.[1]; + explicitReceiver = true; + arguments_ = arrayArguments(arguments_?.[2]); + continue; + } + return { + kind: "known", + path, + arguments: arguments_, + ...(receiver === undefined ? {} : { receiver }), + }; + } +}; diff --git a/src/domain/javascript/javascriptModuleInterop.fixture.ts b/src/domain/javascript/javascriptModuleInterop.fixture.ts new file mode 100644 index 0000000000..1ed97af906 --- /dev/null +++ b/src/domain/javascript/javascriptModuleInterop.fixture.ts @@ -0,0 +1,24 @@ +// Rolldown runtime from the public T3 Code nightly 20261011.2967 main.cjs. +// The input is inert source: none of these helpers or Electron APIs is executed. +export const ROLLDOWN_MODULE_INTEROP = ` +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) { + key = keys[i]; + if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, { + get: ((k) => from[k]).bind(null, key), + enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable + }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule || !__hasOwnProp.call(mod, "default") ? __defProp(target, "default", { + value: mod, + enumerable: true +}) : target, mod)); +`; diff --git a/src/domain/javascript/javascriptModuleInterop.ts b/src/domain/javascript/javascriptModuleInterop.ts new file mode 100644 index 0000000000..35e1c9ef12 --- /dev/null +++ b/src/domain/javascript/javascriptModuleInterop.ts @@ -0,0 +1,774 @@ +import { parseExpression } from "@babel/parser"; +import * as t from "@babel/types"; + +import type { JavaScriptModuleOrigin } from "./javascriptSemanticIr.js"; +import { + isUnshadowedGlobal, + resolveSemanticBindingState, + semanticScopeId, + semanticStableInitializer, + type JavaScriptSemanticAnalysisState, + type JavaScriptSemanticBindingState, +} from "./javascriptSemanticState.js"; +import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; +import { + childNodes, + javaScriptAstChildKeys, +} from "./javascriptSemanticTraversal.js"; +import { + electronMemberPath, + type JavaScriptIntrinsicInvocation, +} from "./javascriptElectronMemberWrites.js"; +import { resolveJavaScriptIntrinsicInvocation } from "./javascriptIntrinsicInvocation.js"; +import { SemanticPropertyPathCoverage } from "./javascriptSemanticPropertyPaths.js"; +import { + readExactJavaScriptLiteral, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; + +// Match the complete Rolldown runtime operations, including its live getters. +// Names and formatting are irrelevant; lexical roles, operators and keys are not. +const TO_ESM = parseExpression(`(mod, isNodeMode, target) => ( + target = mod != null ? __create(__getProtoOf(mod)) : {}, + __copyProps(isNodeMode || !mod || !mod.__esModule || !__hasOwnProp.call(mod, "default") + ? __defProp(target, "default", {value: mod, enumerable: true}) : target, mod) +)`); +const COPY_PROPS = parseExpression(`(to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") + for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) { + key = keys[i]; + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { + get: ((k) => from[k]).bind(null, key), + enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable + }); + } + return to; +}`); +const INTRINSICS: Readonly> = { + __create: ["Object", "create"], + __getProtoOf: ["Object", "getPrototypeOf"], + __defProp: ["Object", "defineProperty"], + __getOwnPropNames: ["Object", "getOwnPropertyNames"], + __getOwnPropDesc: ["Object", "getOwnPropertyDescriptor"], + __hasOwnProp: ["Object", "prototype", "hasOwnProperty"], +}; + +interface IntrinsicDependency { + readonly path: readonly string[]; + readonly capturedAt?: t.Node; +} + +interface InteropOrigin { + readonly origin: JavaScriptModuleOrigin; + readonly dependencies: readonly IntrinsicDependency[]; + readonly defaultIsModule: boolean; +} + +interface InteropHelper { + readonly dependencies: readonly IntrinsicDependency[]; + readonly readyAfter: number; + readonly identity: boolean; +} + +/** Prove one inert namespace conversion without relaxing general assignment semantics. */ +export const createSemanticModuleInterop = ( + state: JavaScriptSemanticAnalysisState, +) => { + const helpers = new Map(); + const origins = new Map(); + const calls = new WeakMap(); + const intrinsicBindings = new Map(); + const writes: { + readonly scope: string; + readonly offset: number; + readonly coverage: SemanticPropertyPathCoverage; + }[] = []; + const invalidatedBindings = new Map(); + const invalidatedCalls = new WeakMap(); + const opaqueMutationPath = (node: t.Node): readonly string[] => { + let current: t.Node | undefined = node; + while (current !== undefined) { + if (t.isFunction(current)) + return ["opaque-intrinsic", semanticScopeId("function", current)]; + current = state.parentsByNode.get(current); + } + return ["intrinsic"]; + }; + + const intrinsicPath = (node: t.Node): readonly string[] | undefined => { + let current = node; + const visited = new Set(); + const pending: { id: string; suffixLength: number }[] = []; + let suffix: readonly string[] = []; + const finish = ( + path: readonly string[] | undefined, + ): readonly string[] | undefined => { + for (const binding of pending) + intrinsicBindings.set( + binding.id, + path === undefined + ? null + : path.slice(0, path.length - binding.suffixLength), + ); + return path; + }; + for (;;) { + const { root, members } = electronMemberPath(current); + if (members.some((member) => member === null)) return finish(undefined); + suffix = [...(members as readonly string[]), ...suffix]; + for (const global of ["Object", "Function", "Reflect"]) + if (isUnshadowedGlobal(root, state, global)) + return finish(["intrinsic", global, ...suffix]); + for (const global of ["globalThis", "global", "window", "self"]) + if (isUnshadowedGlobal(root, state, global)) + return finish(["intrinsic", ...suffix]); + if (!t.isIdentifier(root)) return finish(undefined); + const binding = resolveSemanticBindingState(state, root, root.name); + const cached = + binding === undefined + ? undefined + : intrinsicBindings.get(binding.bindingId); + if (cached !== undefined) + return finish(cached === null ? undefined : [...cached, ...suffix]); + const initializer = semanticStableInitializer(binding, state); + if ( + binding === undefined || + initializer === undefined || + initializer.projection.some((segment) => segment === null) || + visited.has(binding.bindingId) + ) + return finish(undefined); + visited.add(binding.bindingId); + pending.push({ id: binding.bindingId, suffixLength: suffix.length }); + suffix = [...initializer.projection.map(String), ...suffix]; + current = initializer.node; + } + }; + + // A nominal alias path is sufficient for conservative mutation lookup, but + // helper proof also needs each read to follow initialization. An alias copy + // retains the earlier function capture; a member read captures it anew. + const capturedIntrinsic = (node: t.Node): IntrinsicDependency | undefined => { + const path = intrinsicPath(node); + if (path === undefined) return undefined; + const visited = new Set(); + let current = node; + let capturedAt = node; + let suffixLength = 0; + for (;;) { + const { root, members } = electronMemberPath(current); + suffixLength += members.length; + if (!t.isIdentifier(root)) return undefined; + const binding = resolveSemanticBindingState(state, root, root.name); + if (binding === undefined) return { path, capturedAt }; + const initializer = semanticStableInitializer(binding, state); + if ( + initializer === undefined || + (initializer.node.end ?? Infinity) >= (root.start ?? -1) || + visited.has(binding.bindingId) + ) + return undefined; + visited.add(binding.bindingId); + if (suffixLength === 0) capturedAt = initializer.node; + suffixLength += initializer.projection.length; + current = initializer.node; + } + }; + + const staticString = (node: t.Node | undefined): string | undefined => { + if (node === undefined) return undefined; + const visited = new Set(); + let current = node; + for (;;) { + const literal = readExactJavaScriptLiteral(current); + if (literal.found) + return typeof literal.value === "string" ? literal.value : undefined; + const expression = unwrapJavaScriptExpression(current).node; + if (!t.isIdentifier(expression)) return undefined; + const binding = resolveSemanticBindingState( + state, + expression, + expression.name, + ); + const initializer = semanticStableInitializer(binding, state); + if ( + binding === undefined || + initializer === undefined || + initializer.projection.length !== 0 || + (initializer.node.end ?? Infinity) >= (expression.start ?? -1) || + visited.has(binding.bindingId) + ) + return undefined; + visited.add(binding.bindingId); + current = initializer.node; + } + }; + + const intrinsicReadChanged = ( + node: t.Node, + path: readonly string[], + ): boolean => { + const capture = capturedIntrinsic(node)?.capturedAt ?? node; + const scope = opaqueMutationPath(capture).join("."); + const method = path.at(-1); + const invocationMethod = + method === "bind" || method === "call" || method === "apply"; + return writes.some( + (write) => + (write.scope === scope || write.scope === "intrinsic") && + write.offset < (capture.start ?? -1) && + (write.coverage.covers(path) || + (invocationMethod && + write.coverage.covers([ + "intrinsic", + "Function", + "prototype", + method, + ]))), + ); + }; + + const resolveBoundIntrinsic = ( + node: t.Node, + ): JavaScriptIntrinsicInvocation | undefined => { + const visited = new Set(); + const argumentGroups: t.CallExpression["arguments"][] = []; + const bindReads: t.Node[] = []; + let opaqueArguments = false; + let receiver: t.Node | undefined; + let current = node; + for (;;) { + current = unwrapJavaScriptExpression(current).node; + if (t.isIdentifier(current)) { + const binding = resolveSemanticBindingState( + state, + current, + current.name, + ); + const initializer = semanticStableInitializer(binding, state); + if ( + binding === undefined || + initializer === undefined || + initializer.projection.length !== 0 || + (initializer.node.end ?? Infinity) >= (current.start ?? -1) || + visited.has(binding.bindingId) + ) + return undefined; + visited.add(binding.bindingId); + current = initializer.node; + continue; + } + if (!t.isCallExpression(current)) return undefined; + const { root, members } = electronMemberPath(current.callee); + let bindTarget = root; + let bindingArguments: JavaScriptIntrinsicInvocation["arguments"] = + current.arguments; + let captured = capturedIntrinsic(current.callee); + let terminal = captured?.path.at(-1) === "bind"; + let path: readonly string[] | undefined = captured?.path.slice(0, -1); + if (members.at(-1) === "bind") { + // A direct .bind may continue a bound callable, without discarding members. + if (!terminal && members.length !== 1) return undefined; + bindReads.push(captured?.capturedAt ?? current.callee); + } else { + // Reuse the invocation owner for generic call/apply/Reflect.apply of bind. + const invocation = resolveJavaScriptIntrinsicInvocation( + current, + intrinsicPath, + boundIntrinsic, + intrinsicReadChanged, + ); + if ( + invocation?.kind !== "known" || + invocation.path.join(".") !== "intrinsic.Function.prototype.bind" || + invocation.receiver === undefined + ) + return undefined; + bindTarget = invocation.receiver; + bindingArguments = invocation.arguments; + captured = capturedIntrinsic(bindTarget); + terminal = captured !== undefined; + path = captured?.path; + } + receiver = bindingArguments?.[0]; + const ownArguments = bindingArguments?.slice(1) ?? []; + opaqueArguments ||= + bindingArguments === undefined || t.isSpreadElement(receiver); + opaqueArguments ||= ownArguments.some((argument) => + t.isSpreadElement(argument), + ); + argumentGroups.push(ownArguments); + if (!terminal || path === undefined) { + current = bindTarget; + continue; + } + const boundPath = [...path, "bind"]; + const functionBind = ["intrinsic", "Function", "prototype", "bind"]; + opaqueArguments ||= bindReads.some((capture) => { + const scope = opaqueMutationPath(capture).join("."); + return writes.some( + (write) => + (write.scope === scope || write.scope === "intrinsic") && + write.offset < (capture.start ?? -1) && + (write.coverage.covers(boundPath) || + write.coverage.covers(functionBind)), + ); + }); + return { + kind: opaqueArguments ? "opaque" : "known", + path, + ...(receiver === undefined ? {} : { receiver }), + // Rebinding appends arguments to the earlier bound argument list. + arguments: opaqueArguments + ? undefined + : argumentGroups.toReversed().flat(), + }; + } + }; + + const activeBoundValues = new WeakSet(); + const boundIntrinsic = ( + node: t.Node, + ): JavaScriptIntrinsicInvocation | undefined => { + if (activeBoundValues.has(node)) return undefined; + activeBoundValues.add(node); + try { + return resolveBoundIntrinsic(node); + } finally { + activeBoundValues.delete(node); + } + }; + + const isFreshObject = (node: t.Node): boolean => { + const visited = new Set(); + let current = node; + for (;;) { + current = unwrapJavaScriptExpression(current).node; + if ( + t.isObjectExpression(current) || + t.isArrayExpression(current) || + t.isFunction(current) || + (t.isClass(current) && !current.decorators?.length) || + t.isRegExpLiteral(current) + ) + return true; + if (!t.isIdentifier(current)) return false; + const binding = resolveSemanticBindingState(state, current, current.name); + const initializer = unprojectedInitializer(binding, state); + if ( + binding === undefined || + initializer === undefined || + (!t.isFunctionDeclaration(initializer) && + (initializer.end ?? Infinity) >= (current.start ?? -1)) || + visited.has(binding.bindingId) + ) + return false; + visited.add(binding.bindingId); + current = initializer; + } + }; + + const helperProof = ( + binding: JavaScriptSemanticBindingState, + ): InteropHelper | undefined => { + const cached = helpers.get(binding.bindingId); + if (cached !== undefined) return cached ?? undefined; + helpers.set(binding.bindingId, null); + if (state.scopesById.get(binding.scopeId)?.kind !== "program") + return undefined; + const wrapper = unprojectedInitializer(binding, state); + if (wrapper === undefined) return undefined; + if (identityFunction(wrapper, state)) { + const proof = { + dependencies: [], + readyAfter: t.isFunctionDeclaration(wrapper) + ? -1 + : (wrapper.end ?? Infinity), + identity: true, + }; + helpers.set(binding.bindingId, proof); + return proof; + } + const wrapperRoles = canonicalRoles(wrapper, TO_ESM, state); + const copyBinding = wrapperRoles?.get("__copyProps"); + const copy = unprojectedInitializer(copyBinding, state); + const copyRoles = + copy === undefined ? undefined : canonicalRoles(copy, COPY_PROPS, state); + if ( + wrapperRoles === undefined || + copyRoles === undefined || + copy === undefined + ) + return undefined; + const dependencies: IntrinsicDependency[] = []; + let readyAfter = Math.max(wrapper.end ?? Infinity, copy.end ?? Infinity); + for (const roles of [wrapperRoles, copyRoles]) + for (const [role, dependency] of roles) { + const expected = INTRINSICS[role]; + if (expected === undefined) continue; + const initializer = unprojectedInitializer(dependency, state); + const captured = + initializer === undefined + ? undefined + : capturedIntrinsic(initializer); + if ( + initializer === undefined || + captured === undefined || + JSON.stringify(captured.path) !== + JSON.stringify(["intrinsic", ...expected]) + ) + return undefined; + readyAfter = Math.max(readyAfter, initializer.end ?? Infinity); + dependencies.push(captured); + if (role === "__hasOwnProp") + dependencies.push({ path: [...captured.path, "call"] }); + } + dependencies.push( + { path: ["intrinsic", "Function", "prototype", "bind"] }, + { path: ["intrinsic", "Function", "prototype", "call"] }, + ); + const proof = { + dependencies, + identity: false, + readyAfter, + }; + helpers.set(binding.bindingId, proof); + return proof; + }; + + const provenCall = (node: t.Node) => { + if (!t.isCallExpression(node) || !t.isIdentifier(node.callee)) + return undefined; + const helper = resolveSemanticBindingState( + state, + node.callee, + node.callee.name, + ); + const proof = helper === undefined ? undefined : helperProof(helper); + const input = node.arguments[0]; + const mode = node.arguments[1]; + const literalMode = + mode === undefined + ? { found: true, value: false } + : t.isExpression(mode) + ? readExactJavaScriptLiteral(mode) + : { found: false }; + if ( + proof === undefined || + proof.readyAfter >= (node.start ?? -1) || + !t.isExpression(input) || + node.arguments.length > 2 || + !literalMode.found + ) + return undefined; + return { + input, + // A truthy node mode forces the module-valued default. Other modes + // preserve named exports without proving the source's default property. + defaultIsModule: !proof.identity && Boolean(literalMode.value), + dependencies: proof.dependencies.map((dependency) => ({ + ...dependency, + capturedAt: dependency.capturedAt ?? node, + })), + }; + }; + + const callOrigin = (node: t.Node): InteropOrigin | undefined => { + const cached = calls.get(node); + if (cached !== undefined) return cached; + const { root, members } = electronMemberPath(node); + if (!members.every((member): member is string => member !== null)) + return undefined; + const converted = provenCall(root); + const required = + converted === undefined + ? undefined + : semanticRequireOrigin(converted.input, state)?.origin; + if ( + converted === undefined || + required === undefined || + required.importedPath.length !== 0 || + (members[0] === "default" && !converted.defaultIsModule) + ) + return undefined; + const result = { + origin: { ...required, importedPath: members }, + dependencies: converted.dependencies, + defaultIsModule: converted.defaultIsModule, + }; + calls.set(node, result); + return result; + }; + + const origin = ( + binding: JavaScriptSemanticBindingState, + ): InteropOrigin | undefined => { + const single = semanticStableInitializer(binding, state); + if (single !== undefined) { + if ( + !single.projection.every( + (key): key is string => typeof key === "string", + ) + ) + return undefined; + const converted = callOrigin(single.node); + if ( + converted?.origin.importedPath.length === 0 && + single.projection[0] === "default" && + !converted.defaultIsModule + ) + return undefined; + const result = + converted === undefined + ? undefined + : { + ...converted, + origin: { + ...converted.origin, + importedPath: [ + ...converted.origin.importedPath, + ...single.projection, + ], + }, + }; + if (result !== undefined) origins.set(binding.bindingId, result); + return result; + } + const [initial, converted] = binding.initializers; + if ( + binding.definitions.length !== 2 || + binding.definitions[0]?.kind !== "variable" || + binding.definitions[1]?.kind !== "assignment" || + binding.initializers.length !== 2 || + initial === undefined || + converted === undefined || + initial.projection.length !== 0 || + converted.projection.length !== 0 || + state.conditionalInitializers.has(initial.node) + ) + return undefined; + const call = converted.node; + const assignment = state.parentsByNode.get(call); + const statement = + assignment === undefined + ? undefined + : state.parentsByNode.get(assignment); + const declaration = state.parentsByNode.get(initial.node); + const declarationStatement = + declaration === undefined + ? undefined + : state.parentsByNode.get(declaration); + if ( + !t.isCallExpression(call) || + !t.isIdentifier(call.callee) || + !t.isAssignmentExpression(assignment, { operator: "=" }) || + assignment.right !== call || + !t.isIdentifier(assignment.left, { name: binding.name }) || + !t.isExpressionStatement(statement) || + !t.isVariableDeclarator(declaration) || + !t.isVariableDeclaration(declarationStatement) || + state.parentsByNode.get(statement) !== + state.parentsByNode.get(declarationStatement) || + (initial.node.start ?? Infinity) >= (call.start ?? -1) + ) + return undefined; + const convertedCall = provenCall(call); + const input = convertedCall?.input; + if ( + !t.isIdentifier(input) || + resolveSemanticBindingState(state, input, input.name) !== binding + ) + return undefined; + const required = semanticRequireOrigin(initial.node, state)?.origin; + if (required === undefined || required.importedPath.length !== 0) + return undefined; + if (convertedCall === undefined) return undefined; + const result = { + origin: required, + dependencies: convertedCall.dependencies, + defaultIsModule: convertedCall.defaultIsModule, + }; + origins.set(binding.bindingId, result); + return result; + }; + const invalidatedOrigin = (record: InteropOrigin | undefined): boolean => + record?.dependencies.some(({ path, capturedAt }) => { + if (capturedAt === undefined) return true; + const scope = opaqueMutationPath(capturedAt).join("."); + // Captured function values survive later replacement of global + // properties. Other callable bodies are not execution evidence. + return writes.some( + (write) => + (write.scope === scope || write.scope === "intrinsic") && + write.offset < (capturedAt.start ?? -1) && + write.coverage.covers(path), + ); + }) ?? false; + return { + origin, + callOrigin, + intrinsicPath, + intrinsicInvocation: (node: t.CallExpression | t.OptionalCallExpression) => + resolveJavaScriptIntrinsicInvocation( + node, + intrinsicPath, + boundIntrinsic, + intrinsicReadChanged, + ), + boundIntrinsic, + staticString, + isFreshObject, + opaqueMutationPath, + retainWrite: (node: t.Node, path: readonly (string | null)[]) => { + const coverage = new SemanticPropertyPathCoverage(); + coverage.retain(path[0] === "opaque-intrinsic" ? ["intrinsic"] : path); + writes.push({ + scope: opaqueMutationPath(node).join("."), + offset: node.start ?? Infinity, + coverage, + }); + }, + invalidated: (binding: JavaScriptSemanticBindingState): boolean => { + const cached = invalidatedBindings.get(binding.bindingId); + if (cached !== undefined) return cached; + const result = invalidatedOrigin(origins.get(binding.bindingId)); + invalidatedBindings.set(binding.bindingId, result); + return result; + }, + invalidatedCall: (node: t.Node): boolean => { + const cached = invalidatedCalls.get(node); + if (cached !== undefined) return cached; + const result = invalidatedOrigin(calls.get(node)); + invalidatedCalls.set(node, result); + return result; + }, + }; +}; + +const identityFunction = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if ( + !t.isFunction(node) || + node.async || + node.generator || + node.params.length !== 1 + ) + return false; + const parameter = node.params[0]; + const result = t.isBlockStatement(node.body) + ? node.body.body.length === 1 && t.isReturnStatement(node.body.body[0]) + ? node.body.body[0].argument + : undefined + : node.body; + return ( + t.isIdentifier(parameter) && + t.isIdentifier(result) && + resolveSemanticBindingState(state, parameter, parameter.name) !== + undefined && + resolveSemanticBindingState(state, parameter, parameter.name) === + resolveSemanticBindingState(state, result, result.name) + ); +}; + +const unprojectedInitializer = ( + binding: JavaScriptSemanticBindingState | undefined, + state: JavaScriptSemanticAnalysisState, +): t.Node | undefined => { + const initializer = semanticStableInitializer(binding, state); + return initializer?.projection.length === 0 ? initializer.node : undefined; +}; + +/** Match operations and lexical roles, allowing single-statement control blocks. */ +const canonicalRoles = ( + actual: t.Node, + pattern: t.Node, + state: JavaScriptSemanticAnalysisState, +): ReadonlyMap | undefined => { + const roles = new Map(); + const used = new Map(); + const pending: { actual: t.Node; pattern: t.Node; parent: t.Node | null }[] = + [{ actual, pattern, parent: null }]; + while (pending.length !== 0) { + const pair = pending.pop(); + if (pair === undefined) return undefined; + const { parent } = pair; + const controlBody = + (t.isIfStatement(parent) && + (parent.consequent === pair.pattern || + parent.alternate === pair.pattern)) || + (t.isForStatement(parent) && parent.body === pair.pattern); + const node = controlBody ? controlBodyOperation(pair.actual) : pair.actual; + const expected = controlBody + ? controlBodyOperation(pair.pattern) + : pair.pattern; + if (node.type !== expected.type) return undefined; + const literalKey = + parent !== null && + ((t.isMemberExpression(parent) && + !parent.computed && + parent.property === expected) || + (t.isObjectProperty(parent) && + !parent.computed && + parent.key === expected)); + if (t.isIdentifier(expected) && t.isIdentifier(node) && !literalKey) { + const binding = resolveSemanticBindingState(state, node, node.name); + if ( + binding === undefined || + (roles.has(expected.name) && roles.get(expected.name) !== binding) || + (used.has(binding.bindingId) && + used.get(binding.bindingId) !== expected.name) + ) + return undefined; + roles.set(expected.name, binding); + used.set(binding.bindingId, expected.name); + } + const visitorKeys = javaScriptAstChildKeys(expected); + for (const [key, field] of Object.entries( + t.NODE_FIELDS[expected.type] ?? {}, + )) { + if ( + visitorKeys.includes(key) || + (t.isIdentifier(expected) && key === "name" && !literalKey) + ) + continue; + const fallback: unknown = Reflect.get(field, "default"); + if ( + JSON.stringify(Reflect.get(node, key) ?? fallback) !== + JSON.stringify(Reflect.get(expected, key) ?? fallback) + ) + return undefined; + } + const children = childNodes(node), + expectedChildren = childNodes(expected); + if (children.length !== expectedChildren.length) return undefined; + for (const [index, child] of children.entries()) { + const expectedChild = expectedChildren[index]; + if (expectedChild !== undefined) + pending.push({ + actual: child, + pattern: expectedChild, + parent: expected, + }); + } + } + return roles; +}; + +// Braces around one control-body statement do not add operations. Keep other +// blocks, directives and extra statements intact; their semantics need proof. +// Lexical declarations/references still pass through canonicalRoles' bijection. +const controlBodyOperation = (node: t.Node): t.Node => { + while ( + t.isBlockStatement(node) && + node.directives.length === 0 && + node.body.length === 1 + ) { + const statement = node.body[0]; + if (statement === undefined) break; + node = statement; + } + return node; +}; diff --git a/src/domain/javascript/javascriptPackageImportAliases.test.ts b/src/domain/javascript/javascriptPackageImportAliases.test.ts new file mode 100644 index 0000000000..6f64c1d650 --- /dev/null +++ b/src/domain/javascript/javascriptPackageImportAliases.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "vitest"; + +import { + classifyPackageImportSpecifier, + classifyPackageImportTargetString, + lookupExactPackageImport, +} from "./javascriptPackageImportAliases.js"; + +describe("package import specifier classification", () => { + it("treats an exact hash alias as an exact key", () => { + expect(classifyPackageImportSpecifier("#config")).toEqual({ + kind: "exact", + key: "#config", + }); + }); + + it.each(["#internal/env", "#/hidden"])( + "treats %j as an exact key", + (specifier) => { + expect(classifyPackageImportSpecifier(specifier)).toEqual({ + kind: "exact", + key: specifier, + }); + }, + ); + + it.each(["#", "#/", "#utils/"])( + "rejects reserved specifier %j", + (specifier) => { + expect(classifyPackageImportSpecifier(specifier)).toEqual({ + kind: "invalid", + }); + }, + ); + + it.each(["#internal/*", "#*"])( + "leaves pattern specifier %j unresolved", + (specifier) => { + expect(classifyPackageImportSpecifier(specifier)).toEqual({ + kind: "pattern", + }); + }, + ); +}); + +describe("exact package import lookup", () => { + it("returns the exact target without falling through to another key", () => { + expect( + lookupExactPackageImport( + { "#config": "./config.cjs", "#other": "./other.cjs" }, + "#config", + ), + ).toEqual({ + kind: "found", + target: "./config.cjs", + declared: ["#config", "#other"], + }); + }); + + it("names declared exact keys when the requested alias is absent", () => { + expect( + lookupExactPackageImport({ "#config": "./config.cjs" }, "#missing"), + ).toEqual({ + kind: "missing-key", + declared: ["#config"], + }); + }); + + it("does not claim a miss when the map only has pattern keys", () => { + expect( + lookupExactPackageImport({ "#internal/*": "./src/*.js" }, "#internal/a"), + ).toEqual({ + kind: "pattern-unresolved", + declared: ["#internal/*"], + }); + }); + + it("still selects an exact key when the same map also has patterns", () => { + expect( + lookupExactPackageImport( + { "#config": "./config.cjs", "#internal/*": "./src/*.js" }, + "#config", + ), + ).toEqual({ + kind: "found", + target: "./config.cjs", + declared: ["#config", "#internal/*"], + }); + }); + + it.each([undefined, null, "./config.cjs", ["#config"], 7] as const)( + "treats non-object imports value %j as an undefined map", + (imports) => { + expect(lookupExactPackageImport(imports, "#config")).toEqual({ + kind: "undefined-map", + }); + }, + ); + + it("does not treat a pattern key as an exact match for itself", () => { + expect( + lookupExactPackageImport({ "#internal/*": "./src/*.js" }, "#internal/*"), + ).toEqual({ + kind: "pattern-unresolved", + declared: ["#internal/*"], + }); + }); +}); + +describe("package import target string classification", () => { + it("keeps package-local relative targets", () => { + expect(classifyPackageImportTargetString("./config.cjs")).toEqual({ + kind: "relative", + target: "./config.cjs", + }); + }); + + it("marks recursive hash aliases as recursive", () => { + expect(classifyPackageImportTargetString("#other")).toEqual({ + kind: "recursive", + target: "#other", + }); + }); + + it.each(["lodash", "pkg/subpath"])( + "marks %j as an external-package target", + (target) => { + expect(classifyPackageImportTargetString(target)).toEqual({ + kind: "external-package", + target, + }); + }, + ); + + it.each([ + "../outside.js", + "/outside.js", + "https://example.invalid/mod.js", + "file:///outside.js", + "node:fs", + ])("marks %j as an invalid import target", (target) => { + expect(classifyPackageImportTargetString(target)).toEqual({ + kind: "invalid", + target, + }); + }); +}); diff --git a/src/domain/javascript/javascriptPackageImportAliases.ts b/src/domain/javascript/javascriptPackageImportAliases.ts new file mode 100644 index 0000000000..9eab77e100 --- /dev/null +++ b/src/domain/javascript/javascriptPackageImportAliases.ts @@ -0,0 +1,98 @@ +/** + * Exact package-private import-alias lookup. Node resolves `#` specifiers + * against the importer's enclosing package `imports` map; this module owns the + * specifier and exact-key rules. Condition, array, and null target selection + * stay with the shared package-target machinery. Wildcard keys, recursive + * `#` targets, and external-package targets remain explicit unknowns. + */ + +/** How one `#` specifier participates in exact import-alias lookup. */ +export type PackageImportSpecifierClassification = + | { readonly kind: "invalid" } + | { readonly kind: "pattern" } + | { readonly kind: "exact"; readonly key: string }; + +/** Outcome of looking up one exact `#` key in a package `imports` object. */ +export type PackageImportExactLookup = + | { readonly kind: "undefined-map" } + | { + readonly kind: "missing-key"; + readonly declared: readonly string[]; + } + | { + readonly kind: "pattern-unresolved"; + readonly declared: readonly string[]; + } + | { + readonly kind: "found"; + readonly target: unknown; + readonly declared: readonly string[]; + }; + +/** How a selected string target is interpreted for package-private imports. */ +export type PackageImportTargetClassification = + | { readonly kind: "relative"; readonly target: string } + | { readonly kind: "invalid"; readonly target: string } + | { readonly kind: "recursive"; readonly target: string } + | { readonly kind: "external-package"; readonly target: string }; + +/** + * Classify a `#` module specifier before any package-scope walk. + * `#` and names ending in `/` are invalid. Other literal keys, including + * `#/…`, participate in exact lookup; wildcard expansion remains unsupported. + */ +export const classifyPackageImportSpecifier = ( + specifier: string, +): PackageImportSpecifierClassification => { + if (specifier === "#" || specifier.endsWith("/")) return { kind: "invalid" }; + if (specifier.includes("*")) return { kind: "pattern" }; + return { kind: "exact", key: specifier }; +}; + +/** True when an `imports` key is a pattern rather than an exact alias. */ +export const isPackageImportPatternKey = (key: string): boolean => + key.includes("*") || key.endsWith("/"); + +/** + * Look up one exact `#` key in a package `imports` value. + * A missing exact key is reported as missing when every declared key is also + * exact; if the map contains pattern keys, the miss stays pattern-unresolved + * rather than claiming the specifier is undefined. + */ +export const lookupExactPackageImport = ( + imports: unknown, + key: string, +): PackageImportExactLookup => { + if (typeof imports !== "object" || imports === null || Array.isArray(imports)) + return { kind: "undefined-map" }; + const declared = Object.keys(imports); + if (Object.hasOwn(imports, key) && !isPackageImportPatternKey(key)) + return { + kind: "found", + target: Reflect.get(imports, key), + declared, + }; + return declared.some(isPackageImportPatternKey) + ? { kind: "pattern-unresolved", declared } + : { kind: "missing-key", declared }; +}; + +/** + * Classify a selected string import target. Only package-local `./` paths are + * resolved as files; recursive `#` aliases and bare packages stay unknown. + * Parent-relative, absolute, and URL targets are invalid, allowing array + * selection to skip them before choosing a package-local fallback. + */ +export const classifyPackageImportTargetString = ( + target: string, +): PackageImportTargetClassification => { + if (target.startsWith("./")) return { kind: "relative", target }; + if ( + target.startsWith("../") || + target.startsWith("/") || + URL.canParse(target) + ) + return { kind: "invalid", target }; + if (target.startsWith("#")) return { kind: "recursive", target }; + return { kind: "external-package", target }; +}; diff --git a/src/domain/javascript/javascriptRpcEndpoints.test.ts b/src/domain/javascript/javascriptRpcEndpoints.test.ts index 05062b4c76..b086470f3e 100644 --- a/src/domain/javascript/javascriptRpcEndpoints.test.ts +++ b/src/domain/javascript/javascriptRpcEndpoints.test.ts @@ -195,6 +195,32 @@ describe("tRPC binding identity and bundled syntax", () => { ]); }); + it("retains one client initializer across variable redeclarations", () => { + expect( + rpcEndpoints(` + var c; + var c = createTRPCClient({}); + var c; + c.users.query(); + + var replaced; + var replaced = createTRPCClient({}); + replaced = pool; + replaced.posts.query(); + + var initializedTwice = createTRPCClient({}); + var initializedTwice = pool; + initializedTwice.comments.query(); + + var conditional; + if (flag) { var conditional = createTRPCClient({}); } + conditional.groups.query(); + `), + ).toEqual([ + { kind: "network", value: "users", mechanism: "trpc:procedure:query" }, + ]); + }); + it("recognizes surviving factory names inside bundled call wrappers", () => { expect( rpcEndpoints(` diff --git a/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts index 0fd5aa1942..f5ac65b7ac 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts @@ -179,6 +179,34 @@ describe("JavaScript semantic analysis: calls 2", () => { ).not.toBe(left.functionFingerprints[0]?.components.literalSetSha256); }); + it("keeps nested callable syntax out of its owner's fingerprint", () => { + const analyze = (nested: number, owned: number) => + analyzeJavaScriptSemantics(` + function outer() { + function inner() { return ${nested}; } + const object = { method() { return ${nested}; } }; + class Nested { method() { return ${nested}; } } + return ${owned}; + } + `); + const first = analyze(1, 2); + const changedNested = analyze(3, 2); + const changedOwner = analyze(1, 4); + const syntax = (ir: typeof first) => + ir.functionFingerprints.map(({ components }) => ({ + ast: components.normalizedAstSha256, + control: components.controlFlowSha256, + literals: components.literalSetSha256, + })); + const original = syntax(first); + expect(original).toHaveLength(4); + expect(syntax(changedNested)[0]).toEqual(original[0]); + for (const index of [1, 2, 3]) + expect(syntax(changedNested)[index]).not.toEqual(original[index]); + expect(syntax(changedOwner)[0]).not.toEqual(original[0]); + expect(syntax(changedOwner).slice(1)).toEqual(original.slice(1)); + }); + it("keeps fingerprint relations ordered and effects scoped to their owner", () => { const ordered = analyzeJavaScriptSemantics(` function first() {} diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index c4d8153c14..517ed58364 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -18,6 +18,7 @@ import { import { semanticObjectPatternKeys, semanticStaticPropertyKey, + semanticWriteTargets, } from "./javascriptAstValues.js"; import type { JavaScriptSemanticAnalysisState, @@ -31,16 +32,14 @@ import { resolveSemanticBindingFromScope, semanticBindingId, semanticScopeId, + semanticStableInitializer, semanticVariableScope, } from "./javascriptSemanticState.js"; import { completeSemanticSteps, traverseJavaScriptAstSteps, } from "./javascriptSemanticTraversal.js"; -import { - semanticNamespaceInteropOperand, - semanticRequireOrigin, -} from "./javascriptSemanticRequireOrigin.js"; +import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { collectSemanticReturns, @@ -49,7 +48,7 @@ import { import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js"; import { evaluateSemanticProvenance } from "./javascriptSemanticValues.js"; import { - collectElectronMemberWritesSteps, + collectJavaScriptIdentityWritesSteps, electronExportPath, electronMemberPath, ELECTRON_MODULE, @@ -64,6 +63,8 @@ import { } from "./javascriptSourceParser.js"; import { isJavaScriptRpcClientFactory } from "./javascriptRpcEndpoints.js"; +import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { SemanticPropertyPathCoverage } from "./javascriptSemanticPropertyPaths.js"; interface BindPatternInput { readonly pattern: t.Node; @@ -205,13 +206,10 @@ export function* classifyParsedJavaScriptRpcClientRootsSteps( if (binding === undefined) return; let client = clients.get(binding.bindingId); if (client === undefined) { - const [initializer] = binding.initializers; + const initializer = semanticStableInitializer(binding, state); client = initializer !== undefined && initializer.projection.length === 0 && - binding.initializers.length === 1 && - !binding.definitions.some(({ kind }) => kind === "assignment") && - !state.conditionalInitializers.has(initializer.node) && isJavaScriptRpcClientFactory(initializer.node); clients.set(binding.bindingId, client); } @@ -374,6 +372,9 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( const facts = new Map(); const state = createState(file.program); yield* collectDefinitionsSteps(file.program, state); + const interop = createSemanticModuleInterop(state); + const writes = new SemanticPropertyPathCoverage(); + let collectedWrites = false; const exportsByBinding = new Map(); const rootExport = (root: t.Node): readonly string[] | undefined => { if (t.isIdentifier(root)) { @@ -382,23 +383,45 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( if (binding === undefined) return undefined; let exported = exportsByBinding.get(binding.bindingId); if (exported === undefined) { - exported = electronExport(binding, state); + exported = electronExport(binding, state, interop); exportsByBinding.set(binding.bindingId, exported); } - return exported ?? undefined; + return collectedWrites && interop.invalidated(binding) + ? undefined + : (exported ?? undefined); } - const origin = semanticRequireOrigin( - semanticNamespaceInteropOperand(root, state) ?? root, - state, - )?.origin; + const wrapped = interop.callOrigin(root); + const origin = + wrapped === undefined + ? semanticRequireOrigin(root, state)?.origin + : collectedWrites && interop.invalidatedCall(root) + ? undefined + : wrapped.origin; return origin !== undefined && ELECTRON_MODULE.test(origin.specifier) ? electronExportPath(origin.importedPath) : undefined; }; - const writes = yield* collectElectronMemberWritesSteps( + yield* collectJavaScriptIdentityWritesSteps( file.program, - rootExport, + (root, kind) => { + if (kind === "opaque-call") return interop.opaqueMutationPath(root); + const exported = kind === "binding" ? undefined : rootExport(root); + return exported === undefined + ? (interop.intrinsicPath(root) ?? + (kind === "member" + ? interop.boundIntrinsic(root)?.path + : undefined)) + : ["electron", ...exported]; + }, + interop.intrinsicInvocation, + interop.staticString, + interop.isFreshObject, + (node, path) => { + if (path[0] === "electron") writes.retain(path); + else interop.retainWrite(node, path); + }, ); + collectedWrites = true; yield* traverseJavaScriptAstSteps(file.program, { enter: (node) => { if ( @@ -409,7 +432,10 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( return; const { root, members } = electronMemberPath(node.callee); const exported = rootExport(root); - if (exported !== undefined && !writes.covers([...exported, ...members])) + if ( + exported !== undefined && + !writes.covers(["electron", ...exported, ...members]) + ) facts.set(root.start ?? -1, exported); }, }); @@ -421,19 +447,16 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( const electronExport = ( binding: JavaScriptSemanticBindingState, state: JavaScriptSemanticAnalysisState, + interop: ReturnType, ): readonly string[] | null => { - const initializers = namespacePreservingInitializers(binding, state); - if (initializers === undefined) return null; - const directOrigin = bindingOrigin(binding, initializers, state); - const [initializer] = initializers; - if ( - directOrigin === undefined && - (initializer === undefined || - initializers.length !== 1 || - initializers.length !== binding.initializers.length || - state.conditionalInitializers.has(initializer.node)) - ) - return null; + const reassigned = binding.definitions.some( + ({ kind }) => kind === "assignment", + ); + const converted = interop.origin(binding); + if (reassigned && converted === undefined) return null; + const directOrigin = converted?.origin ?? bindingOrigin(binding, state); + const initializer = semanticStableInitializer(binding, state); + if (directOrigin === undefined && initializer === undefined) return null; const provenance = directOrigin === undefined ? evaluateSemanticProvenance(binding, state) @@ -448,66 +471,17 @@ const electronExport = ( : null; }; -/** - * Return the initializers that can change which module a binding names. - * Bundlers emit `name = __toESM(name)` after a CommonJS require; rewrapping - * the binding's own value keeps every member, so it is not a reassignment. - */ -const namespacePreservingInitializers = ( - binding: JavaScriptSemanticBindingState, - state: JavaScriptSemanticAnalysisState, -): JavaScriptSemanticBindingState["initializers"] | undefined => { - const initializers = binding.initializers.filter( - ({ node }) => !rewrapsOwnNamespace(binding, node, state), - ); - const rewraps = binding.initializers.length - initializers.length; - const assignments = binding.definitions.filter( - ({ kind }) => kind === "assignment", - ).length; - return assignments === rewraps ? initializers : undefined; -}; - -const rewrapsOwnNamespace = ( - binding: JavaScriptSemanticBindingState, - node: t.Node, - state: JavaScriptSemanticAnalysisState, -): boolean => { - const assignment = state.parentsByNode.get(node); - const operand = semanticNamespaceInteropOperand(node, state); - return ( - t.isAssignmentExpression(assignment) && - assignment.operator === "=" && - assignment.right === node && - t.isIdentifier(assignment.left, { name: binding.name }) && - t.isIdentifier(operand, { name: binding.name }) && - !semanticResolutionBlocked(state, operand, operand.name) && - resolveSemanticBindingState(state, operand, operand.name) === binding - ); -}; - const bindingOrigin = ( binding: JavaScriptSemanticBindingState, - initializers: JavaScriptSemanticBindingState["initializers"], state: JavaScriptSemanticAnalysisState, ): JavaScriptModuleOrigin | undefined => { if (binding.directOrigins.length > 0) return binding.directOrigins.length === 1 ? binding.directOrigins[0] : undefined; - const [initializer] = initializers; - // A var/let declaration permits writes without proving one occurred. - // Require one unconditional initializer and no actual binding writes. - if ( - initializer === undefined || - initializers.length !== 1 || - state.conditionalInitializers.has(initializer.node) - ) - return undefined; - const origin = semanticRequireOrigin( - semanticNamespaceInteropOperand(initializer.node, state) ?? - initializer.node, - state, - )?.origin; + const initializer = semanticStableInitializer(binding, state); + if (initializer === undefined) return undefined; + const origin = semanticRequireOrigin(initializer.node, state)?.origin; const projection = initializer.projection; return origin === undefined || !projection.every((key): key is string => typeof key === "string") @@ -650,10 +624,11 @@ function* collectDefinitionsSteps( for (const [index, { body, scope, parameters }] of functionBodies.entries()) { if (index % 64 === 0) yield; for (const parameter of parameters) - for (const identifier of assignedPatternIdentifiers( + for (const identifier of semanticWriteTargets( t.isTSParameterProperty(parameter) ? parameter.parameter : parameter, )) - copyParameterToBody({ identifier, body, scope, state }); + if (t.isIdentifier(identifier)) + copyParameterToBody({ identifier, body, scope, state }); } } @@ -801,16 +776,18 @@ const bindAssignment = ( state, ); else - for (const identifier of assignedPatternIdentifiers(node.left)) - addAssignment(identifier, node, scope, state); + for (const identifier of semanticWriteTargets(node.left)) + if (t.isIdentifier(identifier)) + addAssignment(identifier, node, scope, state); } else if (t.isUpdateExpression(node) && t.isIdentifier(node.argument)) addAssignment(node.argument, node, scope, state); else if (t.isForOfStatement(node) || t.isForInStatement(node)) for (const pattern of t.isVariableDeclaration(node.left) ? node.left.declarations.map(({ id }) => id) : [node.left]) { - for (const identifier of assignedPatternIdentifiers(pattern)) - addAssignment(identifier, node, scope, state); + for (const identifier of semanticWriteTargets(pattern)) + if (t.isIdentifier(identifier)) + addAssignment(identifier, node, scope, state); if (t.isForOfStatement(node)) // Keep the loop value unknown, but preserve references yielded into // each bound slot so writes/escapes can reach their iterable origins. @@ -825,30 +802,6 @@ const bindAssignment = ( } }; -const assignedPatternIdentifiers = ( - pattern: t.Node, -): readonly t.Identifier[] => { - const identifiers: t.Identifier[] = []; - const pending = [pattern]; - while (pending.length > 0) { - const node = pending.pop(); - if (node === undefined) continue; - if (t.isIdentifier(node)) identifiers.push(node); - else if (t.isRestElement(node)) pending.push(node.argument); - else if (t.isAssignmentPattern(node)) pending.push(node.left); - else if (t.isArrayPattern(node)) { - for (const element of node.elements) - if (element !== null) pending.push(element); - } else if (t.isObjectPattern(node)) { - for (const property of node.properties) - pending.push( - t.isRestElement(property) ? property.argument : property.value, - ); - } - } - return identifiers; -}; - const nestedScope = ( node: t.Node, parent: t.Node | null, diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts index 0c5b4b12dc..633cca1216 100644 --- a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts @@ -63,6 +63,35 @@ it("traverses a large tree in steps with the synchronous visit order (#1462)", ( expect(stepped).toEqual(synchronous); }); +it("skips child subtrees without changing sibling ancestry or exit callbacks", () => { + const parsed = parseJavaScriptSource("[1, [2], 3]", "selection.js"); + if (parsed === null) throw new Error("Expected parsed source"); + const entered: number[] = []; + const exited: number[] = []; + const parents: string[][] = []; + const steps = traverseJavaScriptAstSteps(parsed.program, { + skip: (node) => node.type === "ArrayExpression" && node.start === 4, + enter: (node, parent, readAncestors) => { + if (node.type !== "NumericLiteral") return; + entered.push(node.value); + expect(readAncestors().at(-1)).toBe(parent); + parents.push(readAncestors().map(({ type }) => type)); + }, + exit: (node) => { + if (node.type === "NumericLiteral") exited.push(node.value); + }, + }); + while (steps.next().done !== true) { + // Complete the same cooperative traversal used by fingerprint collection. + } + expect(entered).toEqual([1, 3]); + expect(exited).toEqual([1, 3]); + expect(parents).toEqual([ + ["Program", "ExpressionStatement", "ArrayExpression"], + ["Program", "ExpressionStatement", "ArrayExpression"], + ]); +}); + it("preserves receiver and Electron alias scope across traversal pauses (#1462)", () => { const padding = Array.from( { length: 400 }, diff --git a/src/domain/javascript/javascriptSemanticFingerprints.ts b/src/domain/javascript/javascriptSemanticFingerprints.ts index 3d4b828bea..8b6edc0365 100644 --- a/src/domain/javascript/javascriptSemanticFingerprints.ts +++ b/src/domain/javascript/javascriptSemanticFingerprints.ts @@ -73,7 +73,7 @@ export function* collectJavaScriptSemanticFingerprintsSteps( const node = state.callableNodesById.get(callable.callableId); if (node === undefined) continue; yield; - const syntax = yield* fingerprintSyntaxSteps(node, callable.callableId); + const syntax = yield* fingerprintSyntaxSteps(node); const promiseEffects = promiseEffectsByOwner.get(callable.callableId) ?? []; const eventEffects = eventEffectsByOwner.get(callable.callableId) ?? []; const timerEffects = timerEffectsByOwner.get(callable.callableId) ?? []; @@ -164,28 +164,19 @@ interface FingerprintSyntax { function* fingerprintSyntaxSteps( root: t.Node, - targetCallableId: string, ): Generator { const normalizedTokens: string[] = []; const controlFlowTokens: string[] = []; const literals: string[] = []; - const callableStack: string[] = []; yield* traverseJavaScriptAstSteps(root, { + skip: (node) => semanticCallableIdForNode(node) !== null, enter: (node, parent) => { - const callableId = semanticCallableIdForNode(node); - if (callableId !== null) callableStack.push(callableId); - if (callableStack.at(-1) !== targetCallableId) return; normalizedTokens.push(normalizedNodeToken(node, parent)); const control = controlFlowToken(node); if (control !== null) controlFlowTokens.push(control); const literal = semanticLiteral(node); if (literal !== null) literals.push(literal); }, - exit: (node) => { - const callableId = semanticCallableIdForNode(node); - if (callableId !== null && callableStack.at(-1) === callableId) - callableStack.pop(); - }, }); return { normalizedTokens, controlFlowTokens, literals }; } diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 62df4c38cb..35f3a0ca9c 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -33,7 +33,7 @@ import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; import { semanticStaticPropertyKey, - semanticMemberWriteTargets, + semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; @@ -401,14 +401,15 @@ export function* collectSemanticMemberMutationsSteps( markEscaped(expression.object, mutation); }; const markTarget = (node: t.Node, mutation: t.Node): void => { - for (const target of semanticMemberWriteTargets(node)) - markValue( - target.object, - [semanticStaticPropertyKey(target.property, target.computed)], - new Set(), - "write", - mutation, - ); + for (const target of semanticWriteTargets(node)) + if (!t.isIdentifier(target)) + markValue( + target.object, + [semanticStaticPropertyKey(target.property, target.computed)], + new Set(), + "write", + mutation, + ); }; yield* traverseJavaScriptAstSteps(program, { enter: (node, parent) => { diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index ec9ec9ae63..00f0805ba1 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -5,7 +5,6 @@ import { resolveSemanticBindingState, semanticResolutionBlocked, type JavaScriptSemanticAnalysisState, - type JavaScriptSemanticBindingState, } from "./javascriptSemanticState.js"; import { semanticStaticPropertyKey, @@ -59,73 +58,3 @@ export const semanticRequireOrigin = ( call: current, }; }; - -// esbuild/Rolldown, TypeScript, and Babel emit these helpers for namespace -// imports of CommonJS modules; each returns an object exposing every own -// member of its operand, so member paths through the result are unchanged. -// Helper bodies are not evaluated: recognition trusts a single program-scope -// function definition under one of these emitted names. -const NAMESPACE_INTEROP_HELPERS = new Set([ - "__importStar", - "__toESM", - "_interopRequireWildcard", -]); - -/** Return the module operand of a locally declared namespace interop helper. */ -export const semanticNamespaceInteropOperand = ( - node: t.Node, - state: JavaScriptSemanticAnalysisState, -): t.Expression | undefined => { - const call = unwrapJavaScriptExpression(node).node; - if ( - !t.isCallExpression(call) || - !t.isIdentifier(call.callee) || - !NAMESPACE_INTEROP_HELPERS.has(call.callee.name) || - semanticResolutionBlocked(state, call.callee, call.callee.name) - ) - return undefined; - const helper = resolveSemanticBindingState( - state, - call.callee, - call.callee.name, - ); - const [operand] = call.arguments; - return helper !== undefined && - emittedInteropHelper(helper, state) && - t.isExpression(operand) - ? operand - : undefined; -}; - -// Emitters declare each helper once at program scope as a function, an -// arrow/function expression, or `(this && this.name) || `. -const emittedInteropHelper = ( - helper: JavaScriptSemanticBindingState, - state: JavaScriptSemanticAnalysisState, -): boolean => { - const [initializer] = helper.initializers; - if ( - initializer === undefined || - helper.initializers.length !== 1 || - helper.definitions.length !== 1 || - (helper.kind !== "function" && helper.kind !== "variable") || - state.scopesById.get(helper.scopeId)?.kind !== "program" - ) - return false; - const value = unwrapJavaScriptExpression(initializer.node).node; - const emitted = - t.isLogicalExpression(value, { operator: "||" }) && - t.isLogicalExpression(unwrapJavaScriptExpression(value.left).node, { - operator: "&&", - }) - ? unwrapJavaScriptExpression(value.right).node - : value; - return ( - t.isFunctionDeclaration(emitted) || - t.isFunctionExpression(emitted) || - t.isArrowFunctionExpression(emitted) || - (t.isCallExpression(emitted) && - (t.isFunctionExpression(emitted.callee) || - t.isArrowFunctionExpression(emitted.callee))) - ); -}; diff --git a/src/domain/javascript/javascriptSemanticState.ts b/src/domain/javascript/javascriptSemanticState.ts index 2024d74867..e07caad996 100644 --- a/src/domain/javascript/javascriptSemanticState.ts +++ b/src/domain/javascript/javascriptSemanticState.ts @@ -75,6 +75,30 @@ export interface JavaScriptSemanticAnalysisState { readonly conditionalInitializers: WeakSet; } +/** + * Return one unconditional source initializer for a declared lexical binding. + * Repeated variable declarations without another initializer do not write it. + * This establishes a stable source fact only; it does not prove immutability or + * that the declaration has executed at a particular program point. + */ +export const semanticStableInitializer = ( + binding: JavaScriptSemanticBindingState | undefined, + state: JavaScriptSemanticAnalysisState, +): JavaScriptSemanticBindingState["initializers"][number] | undefined => { + const [initializer] = binding?.initializers ?? []; + const [definition] = binding?.definitions ?? []; + return binding !== undefined && + ((definition?.kind === "variable" && + binding.definitions.every(({ kind }) => kind === "variable")) || + (binding.definitions.length === 1 && + (definition?.kind === "function" || definition?.kind === "class"))) && + binding.initializers.length === 1 && + initializer !== undefined && + !state.conditionalInitializers.has(initializer.node) + ? initializer + : undefined; +}; + /** Return the active scope from a non-empty construction stack. */ export const currentSemanticScope = ( stack: readonly JavaScriptSemanticScopeState[], diff --git a/src/domain/javascript/javascriptSemanticTraversal.ts b/src/domain/javascript/javascriptSemanticTraversal.ts index 16960ba8fe..581d7d4645 100644 --- a/src/domain/javascript/javascriptSemanticTraversal.ts +++ b/src/domain/javascript/javascriptSemanticTraversal.ts @@ -1,7 +1,7 @@ import * as t from "@babel/types"; /** - * Visit every Babel AST node in deterministic source-tree order. + * Visit Babel AST nodes in deterministic source-tree order, excluding skipped subtrees. * * `readAncestors` returns a snapshot during `enter`, outermost-first and * ending with the direct parent. Callers @@ -17,6 +17,8 @@ export const traverseJavaScriptAst = ( /** Callbacks for one deterministic AST traversal. */ export interface JavaScriptAstVisitor { + /** Omit a child and its subtree without entering or exiting it. */ + readonly skip?: (node: t.Node) => boolean; readonly enter: ( node: t.Node, parent: t.Node | null, @@ -50,6 +52,7 @@ export function* traverseJavaScriptAstSteps( current.nextIndex += 1; if (visited % 1024 === 0) yield; visited += 1; + if (visitor.skip?.(child) === true) continue; pending.push(enter(child, current.node)); } else { visitor.exit?.(current.node, current.parent); @@ -82,8 +85,7 @@ interface TraversalFrame { * not be reintroduced (they diverge on loc/comment fields and ordering). */ export const childNodes = (node: t.Node): t.Node[] => { - const keys = t.VISITOR_KEYS[node.type]; - if (keys === undefined) return []; + const keys = javaScriptAstChildKeys(node); const children: t.Node[] = []; for (const key of keys) { const value: unknown = Reflect.get(node, key); @@ -98,3 +100,7 @@ export const childNodes = (node: t.Node): t.Node[] => { } return children; }; + +/** Child-bearing fields; shared by traversal and structural AST comparison. */ +export const javaScriptAstChildKeys = (node: t.Node): readonly string[] => + t.VISITOR_KEYS[node.type] ?? []; diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index 768f73c0bb..72c351f31e 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -249,10 +249,37 @@ const networkIdentity = ( ): string => `net_${digestCanonicalValue({ method: request.method, url: request.url, resource_type: request.resource_type })}`; +const networkResponseFields = [ + "status", + "mime_type", + "encoded_data_length", +] as const; +type NetworkResponseField = (typeof networkResponseFields)[number]; + +const sharedNetworkResponseFields = ( + before: WebPageInspection, + after: WebPageInspection, +): ReadonlyMap> => { + const shared = new Map>(); + for (const inspection of [before, after]) + for (const request of inspection.network.requests) { + const identity = networkIdentity(request); + const fields = shared.get(identity) ?? new Set(networkResponseFields); + for (const field of networkResponseFields) + if (request[field] === null) fields.delete(field); + shared.set(identity, fields); + } + return shared; +}; + const networkDimension = ( before: WebPageInspection, after: WebPageInspection, ): Dimension => { + const responseFields = sharedNetworkResponseFields(before, after); + const responseFieldsComplete = [...responseFields.values()].every( + (fields) => fields.size === networkResponseFields.length, + ); const selected = bodyShapesSelected(before) && bodyShapesSelected(after); const webSocketShapes = webSocketShapesSelected(before) && webSocketShapesSelected(after); @@ -313,14 +340,14 @@ const networkDimension = ( afterWebSockets, webSocketShapes, ); - return compareDimension( + const compared = compareDimension( new Map([ - ...networkMap(before, sources), + ...networkMap(before, sources, responseFields), ...webSocketMetadataMap(beforeWebSockets), ...beforeWebSocketPayload, ]), new Map([ - ...networkMap(after, sources), + ...networkMap(after, sources, responseFields), ...webSocketMetadataMap(afterWebSockets), ...afterWebSocketPayload, ]), @@ -336,21 +363,31 @@ const networkDimension = ( webSocketShapesComplete, "Network capture or selected HTTP/WebSocket payload-shape coverage is attach-window limited, unavailable, or incomplete.", ); + return compared.status === "unchanged" && !responseFieldsComplete + ? { + ...compared, + status: "unknown", + reason: + "HTTP response status, MIME type, or byte-count evidence is incomplete.", + } + : compared; }; const networkMap = ( inspection: WebPageInspection, sources: ReadonlyMap, + responseFields: ReadonlyMap>, ): ReadonlyMap => { const grouped = new Map(); for (const request of inspection.network.requests) { const identity = networkIdentity(request); const shapes = sources.get(identity); const values = grouped.get(identity) ?? []; + const response: Record = {}; + for (const field of responseFields.get(identity) ?? []) + response[field] = request[field]; values.push({ - status: request.status, - mime_type: request.mime_type, - encoded_data_length: request.encoded_data_length, + ...response, redirects: (request.redirects ?? []).map( ({ request_timestamp: _requestTimestamp, diff --git a/src/dotnet/ManagedDecodeBudget.test.ts b/src/dotnet/ManagedDecodeBudget.test.ts index 48ef121231..ebab4a99e9 100644 --- a/src/dotnet/ManagedDecodeBudget.test.ts +++ b/src/dotnet/ManagedDecodeBudget.test.ts @@ -38,6 +38,18 @@ describe("managed projection admission", () => { }); describe("managed decoding bounds", () => { + it("bounds new row identities without charging lookup replay or merging tables", () => { + const budget = new ManagedDecodeBudget(); + budget.reserve(64 * 1024 * 1024 - 512, null); + budget.reserveRow(2, 1, 100); + budget.reserveRow(12, 1, 200); + + expect(() => budget.reserveRow(2, 1, 100)).not.toThrow(); + expect(() => budget.reserveRow(2, 2, 300)).toThrow(ManagedReaderFailure); + // A rejected row has no retained reservation; replay cannot bypass capacity. + expect(() => budget.reserveRow(2, 2, 300)).toThrow(ManagedReaderFailure); + }); + it("keeps concurrent decoding bounds within one budget until released", () => { const budget = new ManagedDecodeBudget(); const bound = 40 * 1024 * 1024; diff --git a/src/dotnet/ManagedDecodeBudget.ts b/src/dotnet/ManagedDecodeBudget.ts index 14b6915669..d02fce0bc7 100644 --- a/src/dotnet/ManagedDecodeBudget.ts +++ b/src/dotnet/ManagedDecodeBudget.ts @@ -15,6 +15,7 @@ interface CachedString { export class ManagedDecodeBudget { #remaining = MAX_METADATA_REPRESENTATION_BYTES; #transient = 0; + readonly #rows = new Map>(); readonly #strings = new Map(); readonly #digests = new WeakMap>(); @@ -23,6 +24,18 @@ export class ManagedDecodeBudget { this.#remaining -= bytes; } + /** Retain one bookkeeping reservation per physical row, independently of lookup replay. */ + reserveRow(table: number, row: number, offset: number): void { + let rows = this.#rows.get(table); + if (rows?.has(row) === true) return; + this.reserve(256, offset); + if (rows === undefined) { + rows = new Set(); + this.#rows.set(table, rows); + } + rows.add(row); + } + /** Bound a decoding allocation that stays live until `releaseTransient`; only retained output is reserved. */ admitTransient(bytes: number, offset: number | null): void { this.#admit(bytes, offset); diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index 7d54d5f075..54dd49976d 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -19,6 +19,10 @@ import { readMetadataString, sha256Bytes, } from "./ManagedMetadataHeaps.js"; +import { + memberListIndexSize, + memberListTable, +} from "./ManagedMemberPointers.js"; export type ManagedType = ManagedMemberInspection["types"][number]; export type ManagedExportedType = NonNullable< @@ -93,8 +97,8 @@ const readTypeRange = ( row: number, issues: ManagedParseIssue[], ): TypeRange => { - const methods = layout.table(6); - const fields = layout.table(4); + const methods = memberListTable(layout, 6); + const fields = memberListTable(layout, 4); const cursor = metadataRowCursor(bytes, layout, 2, row); cursor.readUInt32(); const nameIndex = cursor.readIndex(layout.stringIndexSize); @@ -114,8 +118,8 @@ const readTypeRange = ( issues, ) ?? null; cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); - const fieldStart = cursor.readIndex(layout.tableIndexSize(4)); - const methodStart = cursor.readIndex(layout.tableIndexSize(6)); + const fieldStart = cursor.readIndex(memberListIndexSize(layout, 4)); + const methodStart = cursor.readIndex(memberListIndexSize(layout, 6)); let nextField = (fields?.rowCount ?? 0) + 1; let nextMethod = (methods?.rowCount ?? 0) + 1; const typeTable = layout.table(2); @@ -125,8 +129,8 @@ const readTypeRange = ( next.readIndex(layout.stringIndexSize); next.readIndex(layout.stringIndexSize); next.readIndex(layout.codedIndexSize("TypeDefOrRef")); - nextField = next.readIndex(layout.tableIndexSize(4)); - nextMethod = next.readIndex(layout.tableIndexSize(6)); + nextField = next.readIndex(memberListIndexSize(layout, 4)); + nextMethod = next.readIndex(memberListIndexSize(layout, 6)); } return { token: metadataToken(2, row), @@ -209,8 +213,8 @@ export const parseTypes = ( readonly issues: readonly ManagedParseIssue[]; } => { const typeTable = layout.table(2); - const fields = layout.table(4); - const methods = layout.table(6); + const fields = memberListTable(layout, 4); + const methods = memberListTable(layout, 6); const items: ManagedType[] = []; const issues: ManagedParseIssue[] = []; for (let row = 1; row <= (typeTable?.rowCount ?? 0); row += 1) { @@ -233,8 +237,8 @@ export const parseTypes = ( const extendsRaw = cursor.readIndex( layout.codedIndexSize("TypeDefOrRef"), ); - const fieldStart = cursor.readIndex(layout.tableIndexSize(4)); - const methodStart = cursor.readIndex(layout.tableIndexSize(6)); + const fieldStart = cursor.readIndex(memberListIndexSize(layout, 4)); + const methodStart = cursor.readIndex(memberListIndexSize(layout, 6)); const nextRange = ranges[row] ?? null; const fieldRange = rowRange( fields, diff --git a/src/dotnet/ManagedMemberPointerBudget.test.ts b/src/dotnet/ManagedMemberPointerBudget.test.ts new file mode 100644 index 0000000000..dcde2670b4 --- /dev/null +++ b/src/dotnet/ManagedMemberPointerBudget.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + attachManagedDecodeBudget, + ManagedDecodeBudget, +} from "./ManagedDecodeBudget.js"; +import { memberListRowLookup } from "./ManagedMemberPointers.js"; +import { readManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { buildManagedMemberPointerFixture } from "./ManagedPe.fixture.js"; +import { readManagedPeLayout } from "./ManagedPeReader.js"; +import { ManagedReaderFailure } from "./ManagedReaderFailure.js"; + +const representationCapacity = 64 * 1024 * 1024; + +const pointerFixture = () => { + const bytes = buildManagedMemberPointerFixture(); + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture has no CLI metadata"); + const layout = readManagedMetadataLayout( + bytes, + pe.rvaToOffset(pe.cli.metadata.rva, pe.cli.metadata.size, "cli.metadata"), + pe.cli.metadata.size, + ); + // A fresh owner lets the test set exact remaining capacity independently of + // the metadata version/string admission already exercised by the parser. + const budget = new ManagedDecodeBudget(); + attachManagedDecodeBudget(layout, budget); + const indexBytes = + ((layout.table(6)?.rowCount ?? 0) + 1) * Uint32Array.BYTES_PER_ELEMENT + + 256; + return { bytes, layout, budget, indexBytes }; +}; + +describe("managed member pointer index capacity", () => { + it("rejects insufficient capacity before allocating the inverse index", () => { + const { bytes, layout, budget, indexBytes } = pointerFixture(); + budget.reserve(representationCapacity - indexBytes + 1, null); + let allocations = 0; + let failure: unknown; + // Proxy forwards static BYTES_PER_ELEMENT too: losing it would produce a + // NaN admission failure and falsely pass a before-allocation assertion. + vi.stubGlobal( + "Uint32Array", + new Proxy(Uint32Array, { + construct(target, arguments_, newTarget) { + allocations += 1; + return Reflect.construct(target, arguments_, newTarget); + }, + }), + ); + try { + memberListRowLookup(bytes, layout, 6); + } catch (cause: unknown) { + failure = cause; + } finally { + vi.unstubAllGlobals(); + } + + expect(failure).toBeInstanceOf(ManagedReaderFailure); + expect(failure).toMatchObject({ + issue: { + code: "resource-limit", + scope: "metadata.representation", + offset: layout.table(5)?.offset, + }, + }); + expect(allocations).toBe(0); + }); + + it("reuses one layout's index without allocating or charging it again", () => { + const { bytes, layout, budget, indexBytes } = pointerFixture(); + budget.reserve(representationCapacity - indexBytes, null); + let allocations = 0; + let observed: number[]; + vi.stubGlobal( + "Uint32Array", + new Proxy(Uint32Array, { + construct(target, arguments_, newTarget) { + allocations += 1; + return Reflect.construct(target, arguments_, newTarget); + }, + }), + ); + try { + const first = memberListRowLookup(bytes, layout, 6); + const repeated = memberListRowLookup(bytes, layout, 6); + observed = [first(1), first(2), repeated(1), repeated(2)]; + } finally { + vi.unstubAllGlobals(); + } + + expect(observed).toEqual([2, 1, 2, 1]); + expect(allocations).toBe(1); + expect(() => budget.reserve(1, null)).toThrow(ManagedReaderFailure); + }); +}); diff --git a/src/dotnet/ManagedMemberPointers.test.ts b/src/dotnet/ManagedMemberPointers.test.ts new file mode 100644 index 0000000000..35764177f7 --- /dev/null +++ b/src/dotnet/ManagedMemberPointers.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "vitest"; + +import { inspectManagedArtifactBytes } from "./ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; +import { readManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { readManagedPeLayout } from "./ManagedPeReader.js"; +import { + buildManagedMemberPointerFixture, + managedPeFixtureTarget, +} from "./ManagedPe.fixture.js"; + +const metadataLayout = (bytes: Buffer) => { + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture omitted CLI metadata"); + return readManagedMetadataLayout( + bytes, + pe.rvaToOffset(pe.cli.metadata.rva, pe.cli.metadata.size, "cli.metadata"), + pe.cli.metadata.size, + ); +}; + +describe("managed member pointer ownership", () => { + it.each([ + { table: "fields" as const, prefix: "0x040000" }, + { table: "methods" as const, prefix: "0x060000" }, + ])("maps reordered pointer slots to physical $table", ({ table, prefix }) => { + const bytes = buildManagedMemberPointerFixture(); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + expect(result.types.map((type) => type.full_name)).toEqual([ + "", + "Fixture.A", + "Fixture.B", + ]); + expect( + result[table].map(({ token, declaring_type, declaring_type_token }) => ({ + token, + declaring_type, + declaring_type_token, + })), + ).toEqual([ + { + token: `${prefix}01`, + declaring_type: "Fixture.B", + declaring_type_token: "0x02000003", + }, + { + token: `${prefix}02`, + declaring_type: "Fixture.A", + declaring_type_token: "0x02000002", + }, + ]); + }); + + it("resolves a local custom-attribute constructor through MethodPtr", () => { + const bytes = buildManagedMemberPointerFixture(); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + + expect(artifact.metadata.stream_names).toContain("#-"); + expect(artifact.coverage).toEqual({ state: "complete", issues: [] }); + expect(artifact.attributes).toEqual([ + expect.objectContaining({ + parent_token: "0x20000001", + constructor_token: "0x06000001", + type_name: "Fixture.B", + }), + ]); + expect(artifact.attributes[0]?.type_name).toBe( + members.methods[0]?.declaring_type, + ); + }); + + it.each([ + { + label: "identity pointer tables", + fieldPointers: [1, 2], + methodPointers: [1, 2], + streamName: "#-" as const, + }, + { + label: "ordinary direct lists", + fieldPointers: null, + methodPointers: null, + streamName: "#~" as const, + }, + { + label: "empty pointer tables with direct lists", + fieldPointers: [], + methodPointers: [], + streamName: "#-" as const, + }, + ])("preserves $label", ({ fieldPointers, methodPointers, streamName }) => { + const bytes = buildManagedMemberPointerFixture({ + fieldPointers, + methodPointers, + streamName, + }); + const target = managedPeFixtureTarget(bytes); + const members = inspectManagedMembersBytes(bytes, target); + const artifact = inspectManagedArtifactBytes(bytes, target); + + expect(members.coverage).toEqual({ state: "complete", issues: [] }); + expect(members.fields.map((field) => field.declaring_type)).toEqual([ + "Fixture.A", + "Fixture.B", + ]); + expect(members.methods.map((method) => method.declaring_type)).toEqual([ + "Fixture.A", + "Fixture.B", + ]); + expect(artifact.attributes[0]?.type_name).toBe("Fixture.A"); + }); +}); + +describe("invalid managed member pointers", () => { + describe.each([ + { + table: "fields" as const, + pointers: "fieldPointers" as const, + scope: "metadata.FieldPtr", + prefix: "0x040000", + names: ["firstField", "secondField"], + independent: "methods" as const, + }, + { + table: "methods" as const, + pointers: "methodPointers" as const, + scope: "metadata.MethodPtr", + prefix: "0x060000", + names: [".ctor", ".ctor"], + independent: "fields" as const, + }, + ])( + "invalid $table pointers", + ({ table, pointers, scope, prefix, names, independent }) => { + it.each([ + { label: "zero row", rows: [0, 1], owners: [null, null] }, + { label: "out-of-range row", rows: [3, 1], owners: [null, null] }, + { label: "duplicate row", rows: [1, 1], owners: [null, null] }, + { label: "omitted row", rows: [1], owners: ["Fixture.A", null] }, + ])( + "preserves member facts and independent ownership for $label", + ({ rows, owners }) => { + const bytes = buildManagedMemberPointerFixture({ [pointers]: rows }); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.coverage.state).toBe("partial"); + expect(result.coverage.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "invalid-row", + scope, + offset: expect.any(Number), + }), + ]), + ); + expect( + result[table].map( + ({ token, name, declaring_type, declaring_type_token }) => ({ + token, + name, + declaring_type, + declaring_type_token, + }), + ), + ).toEqual( + owners.map((owner, index) => ({ + token: `${prefix}0${index + 1}`, + name: names[index], + declaring_type: owner, + declaring_type_token: owner === null ? null : "0x02000002", + })), + ); + expect( + result[independent].map((member) => member.declaring_type), + ).toEqual(["Fixture.B", "Fixture.A"]); + expect( + result.coverage.issues.some( + (issue) => + issue.scope === + (table === "fields" + ? "metadata.MethodPtr" + : "metadata.FieldPtr"), + ), + ).toBe(false); + }, + ); + }, + ); + + it.each([ + { label: "zero row", methodPointers: [2, 0] }, + { label: "duplicate row", methodPointers: [2, 2] }, + { label: "omitted constructor row", methodPointers: [2] }, + ])( + "preserves attribute facts with an unknown constructor owner for $label", + ({ methodPointers }) => { + const controlBytes = buildManagedMemberPointerFixture(); + const control = inspectManagedArtifactBytes( + controlBytes, + managedPeFixtureTarget(controlBytes), + ).attributes[0]; + if (control === undefined) + throw new Error("Fixture omitted custom attribute"); + expect(control.type_name).toBe("Fixture.B"); + const bytes = buildManagedMemberPointerFixture({ methodPointers }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.coverage.state).toBe("partial"); + expect(result.coverage.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "invalid-row", + scope: "metadata.MethodPtr", + }), + ]), + ); + expect(result.attributes).toEqual([ + { ...control, row_offset: expect.any(Number), type_name: null }, + ]); + }, + ); + + it.each([ + { name: "FieldPtr", table: 3 }, + { name: "MethodPtr", table: 5 }, + ])("rejects a table stream ending inside a $name row", ({ table }) => { + const bytes = buildManagedMemberPointerFixture(); + const layout = metadataLayout(bytes); + const pointers = layout.table(table); + if (pointers === undefined) + throw new Error("Fixture omitted pointer table"); + const versionLength = bytes.readUInt32LE(layout.rootOffset + 12); + const firstStreamHeader = + ((layout.rootOffset + 16 + versionLength + 3) & ~3) + 4; + const streamNameEnd = bytes.indexOf(0, firstStreamHeader + 8); + expect(bytes.toString("ascii", firstStreamHeader + 8, streamNameEnd)).toBe( + "#-", + ); + const tablesOffset = + layout.rootOffset + bytes.readUInt32LE(firstStreamHeader); + bytes.writeUInt32LE( + pointers.offset + pointers.rowSize - 1 - tablesOffset, + firstStreamHeader + 4, + ); + + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(result.coverage).toMatchObject({ + state: "unavailable", + issues: [ + expect.objectContaining({ + code: "invalid-metadata-root", + scope: "metadata.table-data", + }), + ], + }); + expect(result.fields).toEqual([]); + expect(result.methods).toEqual([]); + }); +}); diff --git a/src/dotnet/ManagedMemberPointers.ts b/src/dotnet/ManagedMemberPointers.ts new file mode 100644 index 0000000000..3517d0a111 --- /dev/null +++ b/src/dotnet/ManagedMemberPointers.ts @@ -0,0 +1,101 @@ +import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; +import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { + managedFailure, + ManagedReaderFailure, +} from "./ManagedReaderFailure.js"; + +type MemberTable = 4 | 6; +type ListRowLookup = (definitionRow: number) => number; +const lookups = new WeakMap< + ManagedMetadataLayout, + Map +>(); + +/** TypeDef list coordinates refer to the nonempty pointer table when present. */ +export const memberListTable = ( + layout: ManagedMetadataLayout, + table: MemberTable, +) => { + const pointers = layout.table(table - 1); + return pointers !== undefined && pointers.rowCount > 0 + ? pointers + : layout.table(table); +}; + +/** List columns must accommodate either physical definition or pointer rows. */ +export const memberListIndexSize = ( + layout: ManagedMetadataLayout, + table: MemberTable, +): 2 | 4 => + layout.tableIndexSize(table - 1) === 4 || layout.tableIndexSize(table) === 4 + ? 4 + : 2; + +/** Resolve physical definition rows to list positions without changing token identity. */ +export const memberListRowLookup = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + table: MemberTable, +): ListRowLookup => { + const pointers = layout.table(table - 1); + if (pointers === undefined || pointers.rowCount === 0) return (row) => row; + let byTable = lookups.get(layout); + const cached = byTable?.get(table); + if (cached instanceof ManagedReaderFailure) throw cached; + if (cached !== undefined) return cached; + + const definitionCount = layout.table(table)?.rowCount ?? 0; + // The layout already admitted the source table extent. Reserve retained index + // storage and its fixed bookkeeping before allocating either container. + managedDecodeBudget(layout).reserve( + (definitionCount + 1) * Uint32Array.BYTES_PER_ELEMENT + 256, + pointers.offset, + ); + if (byTable === undefined) { + byTable = new Map(); + lookups.set(layout, byTable); + } + const positions = new Uint32Array(definitionCount + 1); + try { + const width = layout.tableIndexSize(table); + for (let row = 1; row <= pointers.rowCount; row += 1) { + const offset: number = pointers.offset + (row - 1) * pointers.rowSize; + const definition = + width === 2 ? bytes.readUInt16LE(offset) : bytes.readUInt32LE(offset); + if (definition === 0 || definition > definitionCount) + throw managedFailure( + "invalid-row", + `metadata.${pointers.name}`, + `Pointer row ${String(row)} names an out-of-range definition row ${String(definition)}`, + offset, + ); + if (positions[definition] !== 0) + throw managedFailure( + "invalid-row", + `metadata.${pointers.name}`, + `Definition row ${String(definition)} has more than one pointer row; its declaring type is ambiguous`, + offset, + ); + positions[definition] = row; + } + const lookup: ListRowLookup = (definition) => { + const position = positions[definition] ?? 0; + if (position === 0) + throw managedFailure( + "invalid-row", + `metadata.${pointers.name}`, + `Definition row ${String(definition)} has no pointer row identifying its declaring type`, + pointers.offset, + ); + return position; + }; + byTable.set(table, lookup); + return lookup; + } catch (cause: unknown) { + // Invalid metadata must not rebuild and charge the same retained index for + // every attribute/member that asks for its owner. + if (cause instanceof ManagedReaderFailure) byTable.set(table, cause); + throw cause; + } +}; diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index 896a6bc217..bc982d81f7 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -31,6 +31,7 @@ import { } from "./ManagedMetadataHeaps.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; import { methodBody } from "./ManagedMethodBodyReader.js"; +import { memberListRowLookup } from "./ManagedMemberPointers.js"; export const parseFields = ( bytes: Buffer, @@ -46,6 +47,10 @@ export const parseFields = ( const issues: ManagedParseIssue[] = []; const table = layout.table(4); const declaringType = createDeclaringTypeLookup(ranges, "field"); + const listRow = readManagedValue( + () => memberListRowLookup(bytes, layout, 4), + issues, + ); for (let row = 1; row <= (table?.rowCount ?? 0); row += 1) { readManagedValue(() => { const cursor = metadataRowCursor(bytes, layout, 4, row); @@ -62,7 +67,11 @@ export const parseFields = ( cursor.readIndex(layout.blobIndexSize), layout.blob.size, ); - const declared = declaringType(row); + const position = + listRow === undefined + ? undefined + : readManagedValue(() => listRow(row), issues); + const declared = position === undefined ? null : declaringType(position); const token = metadataToken(4, row); fields.push({ token, @@ -237,6 +246,10 @@ export const parseMethods = ({ const issues: ManagedParseIssue[] = []; const table = layout.table(6); const declaringType = createDeclaringTypeLookup(ranges, "method"); + const listRow = readManagedValue( + () => memberListRowLookup(bytes, layout, 6), + issues, + ); for (let row = 1; row <= (table?.rowCount ?? 0); row += 1) { readManagedValue(() => { const cursor = metadataRowCursor(bytes, layout, 6, row); @@ -256,7 +269,11 @@ export const parseMethods = ({ layout.blob.size, ); cursor.readIndex(layout.tableIndexSize(8)); - const declared = declaringType(row); + const position = + listRow === undefined + ? undefined + : readManagedValue(() => listRow(row), issues); + const declared = position === undefined ? null : declaringType(position); const token = metadataToken(6, row); methods.push({ token, diff --git a/src/dotnet/ManagedMetadataExtraData.test.ts b/src/dotnet/ManagedMetadataExtraData.test.ts new file mode 100644 index 0000000000..38434390b9 --- /dev/null +++ b/src/dotnet/ManagedMetadataExtraData.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, it } from "vitest"; + +import { inspectManagedArtifactBytes } from "./ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; +import { readManagedPeLayout } from "./ManagedPeReader.js"; +import { + buildManagedPeFixture, + managedPeFixtureTarget, +} from "./ManagedPe.fixture.js"; + +const fixtureTablesStream = (bytes: Buffer) => { + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture omitted CLI metadata"); + const root = pe.rvaToOffset( + pe.cli.metadata.rva, + pe.cli.metadata.size, + "cli.metadata", + ); + const versionLength = bytes.readUInt32LE(root + 12); + const streamHeader = ((root + 16 + versionLength + 3) & ~3) + 4; + expect( + bytes.toString( + "ascii", + streamHeader + 8, + bytes.indexOf(0, streamHeader + 8), + ), + ).toBe("#~"); + const tablesOffset = root + bytes.readUInt32LE(streamHeader); + const valid = bytes.readBigUInt64LE(tablesOffset + 8); + let count = 0; + for (let table = 0; table < 64; table += 1) + if ((valid & (1n << BigInt(table))) !== 0n) count += 1; + return { streamHeader, tablesOffset, extraDataOffset: 24 + count * 4 }; +}; + +describe("CLR metadata tables ExtraData extension", () => { + it.each([ + { label: "absent extension", metadataExtraData: undefined }, + { label: "zero word", metadataExtraData: 0 }, + { label: "nonzero word", metadataExtraData: 0x1234_5678 }, + { label: "maximum unsigned word", metadataExtraData: 0xffff_ffff }, + ])( + "preserves metadata and member identities with $label", + ({ metadataExtraData }) => { + const plain = buildManagedPeFixture(); + const plainTarget = managedPeFixtureTarget(plain); + const baseline = inspectManagedArtifactBytes(plain, plainTarget); + const baselineMembers = inspectManagedMembersBytes(plain, plainTarget); + const bytes = buildManagedPeFixture( + metadataExtraData === undefined ? {} : { metadataExtraData }, + ); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + const shift = metadataExtraData === undefined ? 0 : 4; + + expect(artifact.coverage).toEqual({ state: "complete", issues: [] }); + expect(members.coverage).toEqual({ state: "complete", issues: [] }); + if (baseline.module === null || baseline.assembly === null) + throw new Error("Fixture omitted module or assembly"); + expect(artifact.module).toEqual({ + ...baseline.module, + row_offset: baseline.module.row_offset + shift, + }); + expect(artifact.assembly).toEqual({ + ...baseline.assembly, + row_offset: baseline.assembly.row_offset + shift, + }); + expect(artifact.metadata.table_row_counts).toEqual( + baseline.metadata.table_row_counts, + ); + expect(artifact.target_frameworks).toEqual(baseline.target_frameworks); + expect(members.types).toEqual( + baselineMembers.types.map((type) => ({ + ...type, + row_offset: type.row_offset + shift, + })), + ); + expect(members.fields).toEqual( + baselineMembers.fields.map((field) => ({ + ...field, + row_offset: field.row_offset + shift, + })), + ); + expect(members.methods).toEqual( + baselineMembers.methods.map((method) => ({ + ...method, + row_offset: method.row_offset + shift, + })), + ); + expect(members.call_edges).toEqual(baselineMembers.call_edges); + expect(members.field_accesses).toEqual(baselineMembers.field_accesses); + }, + ); +}); + +describe("CLR ExtraData index widths and boundaries", () => { + it("combines ExtraData with actual four-byte string heap indexes", () => { + const fieldName = "f".repeat(65_536); + const plain = buildManagedPeFixture({ fieldName }); + const baseline = inspectManagedMembersBytes( + plain, + managedPeFixtureTarget(plain), + ); + const bytes = buildManagedPeFixture({ + fieldName, + metadataExtraData: 0x1234_5678, + }); + const { tablesOffset } = fixtureTablesStream(bytes); + expect(bytes[tablesOffset + 6]).toBe(0x41); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(baseline.coverage).toEqual({ state: "complete", issues: [] }); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + expect(result.module?.name).toBe("Fixture.dll"); + expect(result.module?.mvid).toBe(baseline.module?.mvid); + expect(result.types.map((type) => type.full_name)).toEqual([ + "Fixture.Program", + ]); + expect(result.fields.map(({ token, name }) => ({ token, name }))).toEqual([ + { token: "0x04000001", name: fieldName }, + ]); + // Main is placed after the large field name, so its index exceeds 0xffff. + expect(result.methods.map(({ token, name }) => ({ token, name }))).toEqual([ + { token: "0x06000001", name: "Main" }, + ]); + }); + + it("honors ExtraData in an unoptimized table stream without pointer tables", () => { + const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 }); + const baselineTarget = managedPeFixtureTarget(bytes); + const baseline = inspectManagedArtifactBytes(bytes, baselineTarget); + const baselineMembers = inspectManagedMembersBytes(bytes, baselineTarget); + const { streamHeader } = fixtureTablesStream(bytes); + bytes.write("#-", streamHeader + 8, "ascii"); + const target = managedPeFixtureTarget(bytes); + const result = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + expect(members.coverage).toEqual({ state: "complete", issues: [] }); + expect(result.metadata.stream_names).toContain("#-"); + expect(result.metadata.stream_names).not.toContain("#~"); + expect(result.module).toEqual(baseline.module); + expect(result.assembly).toEqual(baseline.assembly); + expect(members.types).toEqual(baselineMembers.types); + expect(members.fields).toEqual(baselineMembers.fields); + expect(members.methods).toEqual(baselineMembers.methods); + }); + + it("reports a truncated row count before interpreting ExtraData", () => { + const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 }); + const { streamHeader, extraDataOffset } = fixtureTablesStream(bytes); + bytes.writeUInt32LE(extraDataOffset - 1, streamHeader + 4); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.coverage).toMatchObject({ + state: "unavailable", + issues: [ + expect.objectContaining({ + code: "invalid-metadata-root", + scope: "metadata.table-row-count", + }), + ], + }); + expect(result.types).toEqual([]); + expect(result.methods).toEqual([]); + }); + + it.each([0, 1, 3])( + "rejects an ExtraData word with only %i declared bytes", + (availableBytes) => { + const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 }); + const { streamHeader, tablesOffset, extraDataOffset } = + fixtureTablesStream(bytes); + expect(bytes[tablesOffset + 6]).toBe(0x40); + expect(bytes.readUInt32LE(tablesOffset + extraDataOffset)).toBe( + 0x1234_5678, + ); + // Only the declared tables-stream extent is shortened; the PE and heaps stay intact. + bytes.writeUInt32LE(extraDataOffset + availableBytes, streamHeader + 4); + + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(result.coverage).toMatchObject({ + state: "unavailable", + issues: [ + expect.objectContaining({ + code: "invalid-metadata-root", + scope: "metadata.tables-extra-data", + offset: tablesOffset + extraDataOffset, + }), + ], + }); + expect(result.module).toBeNull(); + expect(result.types).toEqual([]); + expect(result.methods).toEqual([]); + }, + ); +}); diff --git a/src/dotnet/ManagedMetadataHeaps.ts b/src/dotnet/ManagedMetadataHeaps.ts index ca062b8684..51b85576c5 100644 --- a/src/dotnet/ManagedMetadataHeaps.ts +++ b/src/dotnet/ManagedMetadataHeaps.ts @@ -245,8 +245,9 @@ export const metadataRowCursor = ( "Metadata table is absent", start, ); - // Reserve per-row object/index bookkeeping before readers retain decoded facts. - managedDecodeBudget(layout).reserve(256, start); + // Temporary cursors may revisit a row while resolving owners or references. + // Bind fixed retained bookkeeping to that row's identity, not each lookup. + managedDecodeBudget(layout).reserveRow(table, row, start); return new MetadataRowCursor( bytes, start, diff --git a/src/dotnet/ManagedMetadataInventory.ts b/src/dotnet/ManagedMetadataInventory.ts index 9540047dea..c31059d575 100644 --- a/src/dotnet/ManagedMetadataInventory.ts +++ b/src/dotnet/ManagedMetadataInventory.ts @@ -155,7 +155,7 @@ const readAttributes = ( } => { const attributes = readRows(layout.table(12), (row) => readManagedValue( - () => readCustomAttribute(bytes, layout, row, heapExtent(layout)), + () => readCustomAttribute(bytes, layout, row, heapExtent(layout), issues), issues, ), ); diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 32b5f8d251..3bcaada9d5 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -1,4 +1,9 @@ import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; +import { + memberListIndexSize, + memberListRowLookup, + memberListTable, +} from "./ManagedMemberPointers.js"; import type { ManagedArtifactInspection, @@ -17,7 +22,7 @@ import { readMetadataGuid, readMetadataString, } from "./ManagedMetadataHeaps.js"; -import { managedFailure } from "./ManagedReaderFailure.js"; +import { managedFailure, readManagedValue } from "./ManagedReaderFailure.js"; import type { ManagedResourceDirectory } from "./ManagedMetadataInventory.js"; type ModuleIdentity = NonNullable; @@ -242,21 +247,24 @@ const declaringTypeForMethod = ( heapExtent: number, ): string | null => { const types = layout.table(2); - const methods = layout.table(6); + const methods = memberListTable(layout, 6); if (types === undefined || methods === undefined) return null; + if (methodRow < 1 || methodRow > (layout.table(6)?.rowCount ?? 0)) + return null; + const position = memberListRowLookup(bytes, layout, 6)(methodRow); for (let row = 1; row <= types.rowCount; row += 1) { const cursor = metadataRowCursor(bytes, layout, 2, row); cursor.readUInt32(); const nameIndex = cursor.readIndex(layout.stringIndexSize); const namespaceIndex = cursor.readIndex(layout.stringIndexSize); cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); - cursor.readIndex(layout.tableIndexSize(4)); - const methodStart = cursor.readIndex(layout.tableIndexSize(6)); + cursor.readIndex(memberListIndexSize(layout, 4)); + const methodStart = cursor.readIndex(memberListIndexSize(layout, 6)); const methodEnd = row === types.rowCount ? methods.rowCount + 1 : methodListForType(bytes, layout, row + 1); - if (methodRow < methodStart || methodRow >= methodEnd) continue; + if (position < methodStart || position >= methodEnd) continue; const name = readMetadataString(bytes, layout, nameIndex, heapExtent); const namespace = readMetadataString( bytes, @@ -279,8 +287,8 @@ const methodListForType = ( cursor.readIndex(layout.stringIndexSize); cursor.readIndex(layout.stringIndexSize); cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); - cursor.readIndex(layout.tableIndexSize(4)); - return cursor.readIndex(layout.tableIndexSize(6)); + cursor.readIndex(memberListIndexSize(layout, 4)); + return cursor.readIndex(memberListIndexSize(layout, 6)); }; const attributeTypeName = ( @@ -377,6 +385,7 @@ export const readCustomAttribute = ( layout: ManagedMetadataLayout, row: number, heapExtent: number, + issues: ManagedParseIssue[], ): CustomAttribute => { const cursor = metadataRowCursor(bytes, layout, 12, row); const parentRaw = cursor.readIndex( @@ -432,7 +441,11 @@ export const readCustomAttribute = ( : `CustomAttribute constructor coded index 0x${typeRaw.toString(16)} is invalid: ${typeReason}`, typeOffset, ); - const typeName = attributeTypeName(bytes, layout, typeRaw, heapExtent); + const typeName = + readManagedValue( + () => attributeTypeName(bytes, layout, typeRaw, heapExtent), + issues, + ) ?? null; return { parent_token: parent, constructor_token: metadataCodedToken( diff --git a/src/dotnet/ManagedMetadataLayout.ts b/src/dotnet/ManagedMetadataLayout.ts index 5a34e25122..41a3c77561 100644 --- a/src/dotnet/ManagedMetadataLayout.ts +++ b/src/dotnet/ManagedMetadataLayout.ts @@ -358,6 +358,18 @@ const buildMetadataTables = ({ if ((valid & (1n << BigInt(index))) === 0n) continue; tableCursor += 4; } + // CLR readers honor an optional extra-data word after the row counts. + // Its value is opaque; it is not a length or part of the first table row. + if (((tableBytes[6] ?? 0) & 0x40) !== 0) { + if (tableCursor > tableBytes.length - 4) + throw managedFailure( + "invalid-metadata-root", + "metadata.tables-extra-data", + "Metadata tables extra-data word is truncated", + tablesOffset + tableCursor, + ); + tableCursor += 4; + } for (let index = 0; index < METADATA_TABLE_NAMES.length; index += 1) { const count = rowCounts[index] ?? 0; if ((valid & (1n << BigInt(index))) === 0n) continue; diff --git a/src/dotnet/ManagedMetadataRowSizes.ts b/src/dotnet/ManagedMetadataRowSizes.ts index 738defb44e..900d30e338 100644 --- a/src/dotnet/ManagedMetadataRowSizes.ts +++ b/src/dotnet/ManagedMetadataRowSizes.ts @@ -15,7 +15,7 @@ const ROW_SIZE_FACTORIES: readonly RowSizeFactory[] = [ ({ stringSize: s, guidSize: g }) => 2 + s + g * 3, ({ codedSize: c, stringSize: s }) => c("ResolutionScope") + s * 2, ({ tableSize: t, codedSize: c, stringSize: s }) => - 4 + s * 2 + c("TypeDefOrRef") + t(4) + t(6), + 4 + s * 2 + c("TypeDefOrRef") + Math.max(t(3), t(4)) + Math.max(t(5), t(6)), ({ tableSize: t }) => t(4), ({ stringSize: s, blobSize: b }) => 2 + s + b, ({ tableSize: t }) => t(6), diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index 9f1c4cdde5..8f8ae95fb5 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -3,6 +3,8 @@ import { createHash } from "node:crypto"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; interface ManagedPeFixtureOptions { + readonly additionalEmptyTypes?: number; + readonly customAttributeRowCount?: number; readonly cliFlags?: number; readonly corruptMetadataSignature?: boolean; readonly customAttributeTypeRaw?: number; @@ -18,8 +20,10 @@ interface ManagedPeFixtureOptions { readonly ilBody?: Buffer; readonly malformedAssemblyReferenceRows?: readonly number[]; readonly malformedCustomAttributeRows?: readonly number[]; + readonly metadataExtraData?: number; readonly metadataValidMaskExtra?: bigint; readonly methodName?: string; + readonly methodFlags?: number; readonly methodSignature?: Buffer; readonly moduleRowCount?: number; readonly assemblyRowCount?: number; @@ -187,6 +191,7 @@ const typeDefRow = ({ fieldListSize, methodListSize, extendsRaw = (1 << 2) | 1, + flags = 0x0010_0001, }: { readonly name: number; readonly namespace: number; @@ -197,9 +202,10 @@ const typeDefRow = ({ readonly fieldListSize: MetadataIndexSize; readonly methodListSize: MetadataIndexSize; readonly extendsRaw?: number; + readonly flags?: number; }): Buffer => Buffer.concat([ - u32(0x0010_0001), + u32(flags), metadataIndex(name, stringSize), metadataIndex(namespace, stringSize), metadataIndex(extendsRaw, typeDefOrRefSize), @@ -228,6 +234,7 @@ interface MethodDefRowInput { readonly rva?: number; readonly flags?: number; readonly implFlags?: number; + readonly parameterList?: number; } const methodDefRow = ({ @@ -239,6 +246,7 @@ const methodDefRow = ({ rva = 0, flags = 0x0016, implFlags = 0, + parameterList = 0, }: MethodDefRowInput): Buffer => Buffer.concat([ u32(rva), @@ -246,7 +254,7 @@ const methodDefRow = ({ u16(flags), metadataIndex(name, stringSize), metadataIndex(signature, blobSize), - metadataIndex(0, parameterSize), + metadataIndex(parameterList, parameterSize), ]); const memberRefRow = ({ @@ -328,22 +336,30 @@ const assemblyRefRow = ( metadataIndex(0, blobSize), ]); -const customAttributeRow = ( - value: number, - options: ManagedPeFixtureOptions, - parentIndexSize: 2 | 4, - typeIndexSize: MetadataIndexSize, - blobSize: MetadataIndexSize, -): Buffer => +const customAttributeRow = ({ + value, + options, + parentIndexSize, + typeIndexSize, + blobSize, + row, +}: { + readonly value: number; + readonly options: ManagedPeFixtureOptions; + readonly parentIndexSize: 2 | 4; + readonly typeIndexSize: MetadataIndexSize; + readonly blobSize: MetadataIndexSize; + readonly row: number; +}): Buffer => Buffer.concat([ parentIndexSize === 2 ? u16( - options.malformedCustomAttributeRows?.includes(1) + options.malformedCustomAttributeRows?.includes(row) ? 0xffff : (1 << 5) | 14, ) : u32( - options.malformedCustomAttributeRows?.includes(1) + options.malformedCustomAttributeRows?.includes(row) ? 0xffff_ffff : (1 << 5) | 14, ), @@ -405,6 +421,7 @@ const metadataRoot = ( strings: Buffer, guid: Buffer, blob: Buffer, + tableStreamName: "#~" | "#-" = "#~", ): Buffer => { const version = Buffer.from("v4.0.30319\0\0", "utf8"); const header = Buffer.concat([ @@ -418,7 +435,7 @@ const metadataRoot = ( u16(4), ]); const streamHeaderSize = - metadataStreamHeader(0, 0, "#~").length + + metadataStreamHeader(0, 0, tableStreamName).length + metadataStreamHeader(0, 0, "#Strings").length + metadataStreamHeader(0, 0, "#GUID").length + metadataStreamHeader(0, 0, "#Blob").length; @@ -431,7 +448,7 @@ const metadataRoot = ( offset = align4(offset + guid.length); const blobOffset = offset; const headers = Buffer.concat([ - metadataStreamHeader(tablesOffset, tables.length, "#~"), + metadataStreamHeader(tablesOffset, tables.length, tableStreamName), metadataStreamHeader(stringsOffset, strings.length, "#Strings"), metadataStreamHeader(guidOffset, guid.length, "#GUID"), metadataStreamHeader(blobOffset, blob.length, "#Blob"), @@ -450,12 +467,13 @@ const tablesStream = ( rows: ReadonlyMap, extraValidMask: bigint, heapSizes: number, + extraData?: number, ): Buffer => { let valid = extraValidMask; for (const table of rows.keys()) valid |= 1n << BigInt(table); const header = Buffer.alloc(24); header.writeUInt8(2, 4); - header.writeUInt8(heapSizes, 6); + header.writeUInt8(heapSizes | (extraData === undefined ? 0 : 0x40), 6); header.writeUInt8(1, 7); header.writeBigUInt64LE(valid, 8); const counts: Buffer[] = []; @@ -470,7 +488,12 @@ const tablesStream = ( if (tableRows === undefined) continue; data.push(...tableRows); } - return Buffer.concat([header, ...counts, ...data]); + return Buffer.concat([ + header, + ...counts, + ...(extraData === undefined ? [] : [u32(extraData)]), + ...data, + ]); }; interface FixtureMetadataRowContext { @@ -484,6 +507,7 @@ interface FixtureMetadataRowContext { readonly attributeNamespace: number; readonly constructorName: number; readonly typeName: number; + readonly additionalTypeNames: readonly number[]; readonly typeNamespace: number; readonly fieldName: number; readonly methodName: number; @@ -536,15 +560,16 @@ const customAttributeRows = ( ): readonly Buffer[] => options.targetFramework === null ? [] - : [ - customAttributeRow( - indexes.attributeBlob, + : Array.from({ length: options.customAttributeRowCount ?? 1 }, (_, index) => + customAttributeRow({ + value: indexes.attributeBlob, options, - widths.hasCustomAttribute, - widths.customAttributeType, - heapSizes.blobs, - ), - ]; + parentIndexSize: widths.hasCustomAttribute, + typeIndexSize: widths.customAttributeType, + blobSize: heapSizes.blobs, + row: index + 1, + }), + ); const exportedTypeTable = ( options: ManagedPeFixtureOptions, @@ -577,12 +602,17 @@ const fixtureIndexWidths = ({ const rowCounts = new Map([ [0, options.moduleRowCount ?? 1], [1, 1], - [2, 1], + [2, indexes.additionalTypeNames.length + 1], [4, 1], [6, 1], [8, 0], [10, 1], - [12, options.targetFramework === null ? 0 : 1], + [ + 12, + options.targetFramework === null + ? 0 + : (options.customAttributeRowCount ?? 1), + ], ...(indexes.pinvokeModuleName === null ? [] : ([[26, 1]] as const)), ...(indexes.pinvokeImportName === null ? [] : ([[28, 1]] as const)), [32, options.assemblyRowCount ?? 1], @@ -648,9 +678,9 @@ const fixtureMetadataRows = ({ ], [ 2, - [ + [...indexes.additionalTypeNames, indexes.typeName].map((name) => typeDefRow({ - name: indexes.typeName, + name, namespace: indexes.typeNamespace, fieldList: 1, methodList: 1, @@ -662,7 +692,7 @@ const fixtureMetadataRows = ({ ? {} : { extendsRaw: options.extendsRaw }), }), - ], + ), ], [ 4, @@ -685,7 +715,9 @@ const fixtureMetadataRows = ({ blobSize: heapSizes.blobs, parameterSize: widths.table(8), rva: methodRva, - flags: options.pinvoke === undefined ? 0x0016 : 0x2016, + flags: + options.methodFlags ?? + (options.pinvoke === undefined ? 0x0016 : 0x2016), }), ], ], @@ -774,6 +806,10 @@ const buildManagedFixtureMetadata = ( const attributeNamespace = strings.add("System.Runtime.Versioning"); const constructorName = strings.add(".ctor"); const typeName = strings.add(options.typeName ?? "Program"); + const additionalTypeNames = Array.from( + { length: options.additionalEmptyTypes ?? 0 }, + (_, index) => strings.add(`Empty${String(index)}`), + ); const typeNamespace = strings.add(options.typeNamespace ?? "Fixture"); const fieldName = strings.add(options.fieldName ?? "counter"); const methodName = strings.add(options.methodName ?? "Main"); @@ -825,6 +861,7 @@ const buildManagedFixtureMetadata = ( attributeNamespace, constructorName, typeName, + additionalTypeNames, typeNamespace, fieldName, methodName, @@ -845,7 +882,12 @@ const buildManagedFixtureMetadata = ( (guidSize === 4 ? 0x02 : 0) | (blobSize === 4 ? 0x04 : 0); const metadata = metadataRoot( - tablesStream(rows, options.metadataValidMaskExtra ?? 0n, heapFlags), + tablesStream( + rows, + options.metadataValidMaskExtra ?? 0n, + heapFlags, + options.metadataExtraData, + ), strings.toBuffer(), guidHeap(options.mvid), blobs.toBuffer(), @@ -965,6 +1007,131 @@ export const buildManagedPeFixture = ( buildManagedFixtureMetadata(options, methodRva), ); +/** Two attribute classes whose pointer-table slots can reorder physical members. */ +export const buildManagedMemberPointerFixture = ( + options: { + readonly fieldPointers?: readonly number[] | null; + readonly methodPointers?: readonly number[] | null; + readonly streamName?: "#~" | "#-"; + } = {}, +): Buffer => { + const fieldPointers = + options.fieldPointers === undefined ? [2, 1] : options.fieldPointers; + const methodPointers = + options.methodPointers === undefined ? [2, 1] : options.methodPointers; + // Both instance constructors call System.Attribute::.ctor(), then return. + const body = Buffer.from([0x1e, 0x02, 0x28, 0x01, 0, 0, 0x0a, 0x2a]); + const image = buildManagedFixtureImage({ ilBody: body }, (methodRva) => { + const strings = new StringHeap(); + const blobs = new BlobHeap(); + const moduleName = strings.add("MemberPointers.dll"); + const assemblyName = strings.add("Fixture.MemberPointers"); + const namespace = strings.add("Fixture"); + const typeNames = ["", "A", "B"].map((name) => strings.add(name)); + const fieldNames = ["firstField", "secondField"].map((name) => + strings.add(name), + ); + const constructorName = strings.add(".ctor"); + const attributeName = strings.add("Attribute"); + const systemNamespace = strings.add("System"); + const runtimeName = strings.add("System.Runtime"); + const fieldSignature = blobs.add(Buffer.from([0x06, 0x08])); + const constructorSignature = blobs.add( + Buffer.from([0x20, 0x01, 0x01, 0x0e]), + ); + const baseConstructorSignature = blobs.add(Buffer.from([0x20, 0, 0x01])); + const attributeValue = blobs.add(fixedStringAttributeBlob("fixture")); + const rows = new Map([ + [0, [moduleRow(moduleName, 2, 2)]], + // ResolutionScope selects AssemblyRef 1, which contains System.Attribute. + [ + 1, + [ + Buffer.concat([ + u16((1 << 2) | 2), + u16(attributeName), + u16(systemNamespace), + ]), + ], + ], + [ + 2, + typeNames.map((name, index) => + typeDefRow({ + name, + namespace: index === 0 ? 0 : namespace, + fieldList: index === 2 ? 2 : 1, + methodList: index === 2 ? 2 : 1, + stringSize: 2, + typeDefOrRefSize: 2, + fieldListSize: 2, + methodListSize: 2, + extendsRaw: index === 0 ? 0 : (1 << 2) | 1, + flags: index === 0 ? 0 : 0x0010_0001, + }), + ), + ], + [4, fieldNames.map((name) => fieldRow(name, fieldSignature, 2, 2))], + [ + 6, + [1, 2].map(() => + methodDefRow({ + name: constructorName, + signature: constructorSignature, + stringSize: 2, + blobSize: 2, + parameterSize: 2, + parameterList: 1, + rva: methodRva, + flags: 0x1886, + }), + ), + ], + [ + 10, + [ + memberRefRow({ + name: constructorName, + signature: baseConstructorSignature, + stringSize: 2, + blobSize: 2, + parentSize: 2, + }), + ], + ], + // The assembly attribute constructor is physical MethodDef row 1. + [ + 12, + [ + customAttributeRow({ + value: attributeValue, + options: { customAttributeTypeRaw: (1 << 3) | 2 }, + parentIndexSize: 2, + typeIndexSize: 2, + blobSize: 2, + row: 1, + }), + ], + ], + [32, [assemblyRow(assemblyName, 2, 2)]], + [35, [assemblyRefRow(runtimeName, 0, false, 2, 2)]], + ]); + if (fieldPointers !== null) rows.set(3, fieldPointers.map(u16)); + if (methodPointers !== null) rows.set(5, methodPointers.map(u16)); + return metadataRoot( + tablesStream(rows, 0n, 0), + strings.toBuffer(), + guidHeap(), + blobs.toBuffer(), + options.streamName ?? "#-", + ); + }); + // The shared PE shell defaults to an executable; this fixture is a DLL. + image.writeUInt16LE(image.readUInt16LE(0x84 + 18) | 0x2000, 0x84 + 18); + image.writeUInt32LE(0, 0x0200 + 20); + return image; +}; + /** Build a syntactically valid PE32 fixture with no CLI directory. */ export const buildNativePeFixture = (): Buffer => { const image = buildManagedPeFixture(); diff --git a/src/firmware/FirmwareFiles.ts b/src/firmware/FirmwareFiles.ts index 4a5aa3d842..10201905ea 100644 --- a/src/firmware/FirmwareFiles.ts +++ b/src/firmware/FirmwareFiles.ts @@ -19,6 +19,7 @@ import { import type { FirmwareRequest } from "../domain/firmware/firmwareAnalysis.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; import { FIRMWARE_LIMITS } from "./FirmwareRelease.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** One regular file or unpublished filesystem entry in a private extraction. */ export interface FirmwareEntry { @@ -261,7 +262,9 @@ export const inventoryFirmwareOutput = async ( }); } } - return entries.sort((a, b) => a.relativePath.localeCompare(b.relativePath)); + return entries.sort((a, b) => + compareUnicodeCodePoints(a.relativePath, b.relativePath), + ); }; /** Bind producer paths to the selected input or private extraction tree. */ diff --git a/src/firmware/FirmwareReports.test.ts b/src/firmware/FirmwareReports.test.ts index 2546c80058..5ffd5a7225 100644 --- a/src/firmware/FirmwareReports.test.ts +++ b/src/firmware/FirmwareReports.test.ts @@ -77,6 +77,47 @@ describe("firmware producer boundaries", () => { }); describe("Unblob multi-file extraction diagnostics", () => { + it("names a missing extractor dependency and stage while retaining the raw report", () => { + const dependencyFailure = { + __typename__: "ExtractorDependencyNotFoundReport", + severity: "ERROR", + dependencies: ["sasquatch"], + }; + const record = root(); + const normalized = normalizeUnblobReport( + [ + { + ...record, + reports: [ + ...record.reports.filter( + (item) => item.__typename__ !== "UnknownChunkReport", + ), + { + __typename__: "ChunkReport", + id: "gzip-id", + start_offset: 0, + end_offset: 20, + size: 20, + handler_name: "gzip", + is_encrypted: false, + extraction_reports: [dependencyFailure], + }, + ], + }, + ], + context, + ); + expect(normalized.coverage).toBe("partial"); + expect(normalized.diagnostics).toEqual([ + { + input_path: "$input", + stage: "gzip", + missing_dependencies: ["sasquatch"], + report: dependencyFailure, + }, + ]); + }); + it("retains nested MultiFile extraction failures in partial coverage", () => { const outputPath = `${context.outputRoot}/multi`; const childTask = { @@ -130,7 +171,12 @@ describe("Unblob multi-file extraction diagnostics", () => { expect(normalized.coverage).toBe("partial"); expect(normalized.diagnostics).toEqual([ - { input_path: "$output/multi", report: multiFileReport }, + { + input_path: "$output/multi", + stage: "multipart-firmware", + missing_dependencies: ["sasquatch"], + report: multiFileReport, + }, ]); }); }); diff --git a/src/firmware/FirmwareReports.ts b/src/firmware/FirmwareReports.ts index 3a7e93d48b..782c52a89f 100644 --- a/src/firmware/FirmwareReports.ts +++ b/src/firmware/FirmwareReports.ts @@ -68,6 +68,53 @@ const known = new Set([ "MultiFileReport", ]); +const dependencyFailure = z.object({ + __typename__: z.literal("ExtractorDependencyNotFoundReport"), + dependencies: z.array(z.string().min(1)).min(1), +}); + +/** Collect the named extractors from any dependency report nested in one value. */ +const missingDependencies = (value: unknown): string[] => { + const found: string[] = []; + const visit = (node: unknown): void => { + if (Array.isArray(node)) { + for (const item of node) visit(item); + return; + } + if (typeof node !== "object" || node === null) return; + const failure = dependencyFailure.safeParse(node); + if (failure.success) { + for (const dependency of failure.data.dependencies) + if (!found.includes(dependency)) found.push(dependency); + return; + } + for (const child of Object.values(node)) visit(child); + }; + visit(value); + return found; +}; + +/** + * Name the missing extractors and the producing stage for one extraction + * problem so a caller can act without decoding the raw report. The original + * report stays retained alongside the normalized fields. + */ +const extractionDiagnostic = ( + inputPath: string, + stage: string | null, + report: JsonValue, +): JsonValue => { + const dependencies = missingDependencies(report); + return { + input_path: inputPath, + ...(stage === null ? {} : { stage }), + ...(dependencies.length === 0 + ? {} + : { missing_dependencies: dependencies }), + report, + }; +}; + /** Interpret the pinned Binwalk report without inventing validated region lengths. */ export const normalizeBinwalkReport = ( raw: unknown, @@ -234,7 +281,13 @@ export const normalizeUnblobReport = ( : null; if (detail !== null) for (const problem of detail.extraction_reports) - diagnostics.push({ input_path: path, report: problem }); + diagnostics.push( + extractionDiagnostic( + path, + detail.handler_name, + jsonValueSchema.parse(problem), + ), + ); chunks.push({ input_path: path, provider_id: parsed.id, @@ -253,7 +306,13 @@ export const normalizeUnblobReport = ( if (item.__typename__ === "MultiFileReport") { const parsed = multiFile.parse(item); if (parsed.extraction_reports.length > 0) - diagnostics.push({ input_path: path, report: item }); + diagnostics.push( + extractionDiagnostic( + path, + parsed.handler_name, + jsonValueSchema.parse(item), + ), + ); } } } diff --git a/src/ghidra/GhidraTransport.ts b/src/ghidra/GhidraTransport.ts index 85e3a19af9..1315a61c6c 100644 --- a/src/ghidra/GhidraTransport.ts +++ b/src/ghidra/GhidraTransport.ts @@ -6,6 +6,7 @@ import { readWindowsPrivateRuntimeFile } from "../windows/WindowsPrivateRuntime. import { err, ok, type Result } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** Local bridge transports used by the current Ghidra session. */ export type GhidraTransportKind = "unix-socket" | "authenticated-loopback-tcp"; @@ -147,7 +148,7 @@ const hasExactKeys = ( expected: readonly string[], ): value is Record => { const keys = Object.keys(value).sort((left, right) => - left.localeCompare(right), + compareUnicodeCodePoints(left, right), ); return ( keys.length === expected.length && diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index a7e8b33afe..8201ee2dfb 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -362,10 +362,9 @@ export class HopperApplicationLauncher implements BridgeLauncher { ? { ok: true, strategy: selection.strategy } : { ok: false, - error: new HopperProcessError( - selection.exitCode, - selection.diagnostic, - ), + error: new HopperProcessError(selection.exitCode, { + diagnostic: selection.diagnostic, + }), }; } } diff --git a/src/hopper/HopperCleanup.ts b/src/hopper/HopperCleanup.ts index 346e0b8949..eb155a9fb5 100644 --- a/src/hopper/HopperCleanup.ts +++ b/src/hopper/HopperCleanup.ts @@ -202,9 +202,7 @@ const shutdownRequestFailure = ( operation: "shutdown" | "shutdown_document", cause: unknown, ): HopperProcessError => { - const failure = new HopperProcessError(null, undefined, operation); - failure.cause = cause; - return failure; + return new HopperProcessError(null, { operation, cause }); }; const shutdownFailureDetails = ( diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 99881422e0..2e903c8d2b 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -129,13 +129,9 @@ export class HopperClient { this.#responses.push(chunk); }; readonly #onSocketError = (): void => { - const error = new HopperProcessError( - null, - undefined, - undefined, - undefined, - "unreachable", - ); + const error = new HopperProcessError(null, { + providerState: "unreachable", + }); this.#failAll(error); this.#rememberProcessFailure(error, []); }; @@ -151,25 +147,18 @@ export class HopperClient { setImmediate(() => { if (this.#closing || this.#launcherExitCode !== undefined) return; const exitCode = launch.process.exitCode ?? null; - const error = new HopperProcessError( - exitCode, - this.#launcherFailureDiagnostic, - undefined, - undefined, - exitCode !== null ? "exited" : "unreachable", - ); + const error = new HopperProcessError(exitCode, { + diagnostic: this.#launcherFailureDiagnostic, + providerState: exitCode !== null ? "exited" : "unreachable", + }); this.#failAll(error); this.#rememberProcessFailure(error, []); }); return; } - const error = new HopperProcessError( - null, - undefined, - undefined, - undefined, - "unreachable", - ); + const error = new HopperProcessError(null, { + providerState: "unreachable", + }); this.#failAll(error); this.#rememberProcessFailure(error, []); }; @@ -729,15 +718,12 @@ export class HopperClient { )) ?? false; const launcherFailure = this.#launcherOutcome(outputClosed); if (ownsProviderLifetime) - return new HopperProcessError( - exitCode, - this.#launcherFailureDiagnostic, - undefined, - undefined, - "exited", - "launch", + return new HopperProcessError(exitCode, { + diagnostic: this.#launcherFailureDiagnostic, + providerState: "exited", + stage: "launch", launcherFailure, - ); + }); if ( this.#resources.launch?.authorizationDenialBeforeHandoff === true && nativeHopperLaunchWasDenied(launcherFailure) @@ -766,18 +752,34 @@ export class HopperClient { }, ): Promise> { const id = this.#nextId++; + if ( + this.#resources.launch?.providerLifetime === "launcher-process" && + this.#launcherExitCode !== undefined + ) { + return this.#unavailableRequest( + new HopperProcessError(this.#launcherExitCode, { + diagnostic: this.#launcherFailureDiagnostic, + operation: method, + requestId: id, + providerState: "exited", + stage: hopperOperationStage(method), + launcherFailure: this.#launcherOutcome( + (await this.#resources.processSupervisor?.waitForOutputClose(0)) ?? + false, + ), + }), + ); + } const socket = this.#socket; const token = this.#token; if (socket === undefined || socket.destroyed || token === undefined) { return this.#unavailableRequest( - new HopperProcessError( - null, - undefined, - method, - id, - "unreachable", - hopperOperationStage(method), - ), + new HopperProcessError(null, { + operation: method, + requestId: id, + providerState: "unreachable", + stage: hopperOperationStage(method), + }), ); } if (options.signal?.aborted === true) @@ -888,13 +890,11 @@ export class HopperClient { if (launch.providerLifetime === "launcher-process" && !this.#closing) { this.#rememberExit(event.code); this.#failAll( - new HopperProcessError( - event.code, - this.#launcherFailureDiagnostic, - undefined, - undefined, - "exited", - ), + new HopperProcessError(event.code, { + diagnostic: this.#launcherFailureDiagnostic, + providerState: "exited", + launcherFailure: this.#launcherOutcome(false), + }), ); } } diff --git a/src/hopper/HopperRequestQueue.ts b/src/hopper/HopperRequestQueue.ts index 6ae162e06f..43f4f2821d 100644 --- a/src/hopper/HopperRequestQueue.ts +++ b/src/hopper/HopperRequestQueue.ts @@ -273,14 +273,12 @@ export class HopperRequestQueue { this.accept( entry.id, err( - new HopperProcessError( - null, - undefined, - entry.method, - entry.id, - "unreachable", - hopperOperationStage(entry.method), - ), + new HopperProcessError(null, { + operation: entry.method, + requestId: entry.id, + providerState: "unreachable", + stage: hopperOperationStage(entry.method), + }), ), ), ); @@ -386,19 +384,20 @@ const withRequestContext = ( entry: QueuedRequest, ): HopperError => { if (error instanceof HopperProcessError) - return new HopperProcessError( - error.exitCode, - error.diagnostic, - error.operation ?? entry.method, - error.requestId ?? entry.id, - error.providerState, - error.operation === undefined && error.stage === "connection" - ? hopperOperationStage(entry.method) - : error.stage, - ); + return new HopperProcessError(error.exitCode, { + ...error, + cause: error.cause, + operation: error.operation ?? entry.method, + requestId: error.requestId ?? entry.id, + stage: + error.operation === undefined && error.stage === "connection" + ? hopperOperationStage(entry.method) + : error.stage, + }); if (error instanceof HopperRemoteError) return new HopperRemoteError(error.code, error.safeMessage, { - diagnosticType: error.diagnosticType, + ...error, + cause: error.cause, operation: error.operation ?? entry.method, requestId: error.requestId ?? entry.id, }); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 4113ec8db4..c02ddaa3e2 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -59,6 +59,7 @@ import { const OPERATION = "analyze_javascript_application"; const PROVIDER = "javascript-static-analysis"; const MIB = 1024 * 1024; +const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6; const pendingCleanup = new Set(); /** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */ const WORKER_HEAP_FLAGS_ENFORCED = @@ -80,7 +81,6 @@ export class JavaScriptAnalysisWorker { #estimatedRetainedBytes = 0; #applicationProjectionFailure: AnalysisResourceConstraintError | undefined; #parentCapacityExhausted = false; - #retainedStaticBytes = 0; #sourcesRemaining: number | undefined; readonly #resultBudgetBytes: number; /** Null when the runtime enforces the V8 heap flags instead. */ @@ -326,7 +326,13 @@ export class JavaScriptAnalysisWorker { { ...this.#limits(), parent_result_budget_bytes: this.#resultBudgetBytes, + parent_available_result_budget_bytes: remaining, module_transfer_bytes: descriptor.module_bytes, + module_retention_estimate_bytes: moduleEstimate, + }, + { + remediationAction: + "Use the retained static facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.", }, ), }); @@ -358,9 +364,16 @@ export class JavaScriptAnalysisWorker { { ...this.#limits(), parent_result_budget_bytes: this.#resultBudgetBytes, + parent_available_result_budget_bytes: remaining, + module_retention_estimate_bytes: moduleEstimate, + graph_retention_estimate_bytes: graphEstimate, graph_transfer_bytes: descriptor.graph_bytes, graph_records: descriptor.graph_records, }, + { + remediationAction: + "Use the retained static and module facts or analyze fewer source files. This limit belongs to the parent result budget; increasing the worker heap alone does not increase it.", + }, ), }); break; @@ -582,16 +595,16 @@ export class JavaScriptAnalysisWorker { const value: unknown = JSON.parse(text); const facts = javaScriptWorkerStaticAnalysisSchema.parse(value); const remaining = this.#remainingTotalResultBudget(); - const retainedBytes = bytes * 6; + const retainedBytes = bytes * STATIC_CHECKPOINT_RETENTION_EXPANSION; this.#parentCapacityExhausted = retainedBytes > remaining; this.#estimatedRetainedBytes += retainedBytes; - this.#retainedStaticBytes += retainedBytes; const expandedBytes = estimateJavaScriptStaticApplicationProjection( facts, bytes, sourcePath, ); - if (expandedBytes <= this.#remainingResultBudget()) + const availableResultBudget = this.#remainingResultBudget(); + if (expandedBytes <= availableResultBudget) this.#estimatedRetainedBytes += expandedBytes; else this.#applicationProjectionFailure = new AnalysisResourceConstraintError( @@ -600,8 +613,13 @@ export class JavaScriptAnalysisWorker { `Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`, { parent_result_budget_bytes: this.#resultBudgetBytes, + parent_available_result_budget_bytes: availableResultBudget, application_projection_required_bytes: expandedBytes, }, + { + remediationAction: + "Read the complete static facts on the file node or analyze fewer source files. Expanded application relationships exceed the parent result budget; increasing the worker heap alone does not increase it.", + }, ); return facts; } @@ -638,16 +656,17 @@ export class JavaScriptAnalysisWorker { ); } #remainingResultBudget(): number { - // Future sources need their compact observations before current modules - // consume capacity with expanded relationships. A final or sole source - // can use the unused reserve as well. + // One sequential worker produces one next checkpoint at a time. Reserve + // its maximum protocol size at the same retained expansion used on receipt; + // earlier checkpoints are already charged to estimatedRetainedBytes. + // Reserving every unexamined source's maximum would reject useful results + // merely because many small files remain. Actual retained growth and live + // heap capacity are checked again for every checkpoint and transfer. + const checkpointReserve = + DEFAULTS.checkpointBytes * STATIC_CHECKPOINT_RETENTION_EXPANSION; const staticReserve = this.#sourcesRemaining !== undefined && this.#sourcesRemaining > 0 - ? Math.max( - 0, - Math.floor((this.#resultBudgetBytes * 4) / 5) - - this.#retainedStaticBytes, - ) + ? checkpointReserve : 0; return Math.max(0, this.#remainingTotalResultBudget() - staticReserve); } diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index bebde09738..838c1082df 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -355,7 +355,10 @@ export const inspectSourceMapValue = ( } const leaf = inspectLeaf(node, budget, options); leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); - if (options.profile !== "browser-collection") + if (options.profile !== "browser-collection") { + const ignoredSources = new Set( + leaf.ignoreList ?? leaf.x_google_ignoreList ?? [], + ); for (const [sourceIndex, source] of leaf.sources.entries()) declarations.push({ sectionPath: node.path, @@ -363,10 +366,9 @@ export const inspectSourceMapValue = ( source, root: leaf.sourceRoot ?? null, content: leaf.sourcesContent?.[sourceIndex] ?? null, - ignored: (leaf.ignoreList ?? leaf.x_google_ignoreList ?? []).includes( - sourceIndex, - ), + ignored: ignoredSources.has(sourceIndex), }); + } } return { format: root.sections === undefined ? "regular" : "indexed", diff --git a/src/javascript/sourceMaps/TraceSourceMap.test.ts b/src/javascript/sourceMaps/TraceSourceMap.test.ts index 0f3c6abdf0..ddb70b9fa4 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.test.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.test.ts @@ -29,6 +29,47 @@ it("traces BOM-prefixed maps with a source-map transport prefix ending in CR", ( }); }); +it("keeps ignore-list precedence and section-local indexes for duplicate sources", () => { + const map = { + version: 3, + sections: [ + { + offset: { line: 0, column: 0 }, + map: { + ...leaf, + sources: ["same.ts", "same.ts"], + sourcesContent: ["first", "second"], + mappings: "AAAA,ACAA", + ignoreList: [1, 1], + x_google_ignoreList: [0], + }, + }, + { + offset: { line: 1, column: 0 }, + map: { ...leaf, x_google_ignoreList: [0] }, + }, + { + offset: { line: 2, column: 0 }, + map: { ...leaf, ignoreList: [], x_google_ignoreList: [0] }, + }, + ], + }; + expect(trace(map).matches).toMatchObject([ + { source_index: 0, ignored: false, content: { text: "first" } }, + { source_index: 1, ignored: true, content: { text: "second" } }, + ]); + expect(trace(map, { line: 2, column: 0 }).matches[0]).toMatchObject({ + section_path: [1], + source_index: 0, + ignored: true, + }); + expect(trace(map, { line: 3, column: 0 }).matches[0]).toMatchObject({ + section_path: [2], + source_index: 0, + ignored: false, + }); +}); + describe("upstream source-map point evidence", () => { it("preserves every duplicate-column/source-URL candidate and its own embedded bytes", () => { const report = trace({ diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index d8fee75721..b1ea22624d 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -13,6 +13,7 @@ import { parseDyldSymbols } from "./parsers/dyldInfo.js"; import { parseLipoArchitectures } from "./parsers/lipo.js"; import { withoutEchoedPathHeader } from "./parsers/echoedPath.js"; import { parseOtoolLoadCommands } from "./parsers/otool.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; interface NativeMachoObservation { readonly result: JsonValue; @@ -270,7 +271,7 @@ const uniqueSymbols = ( for (const item of items) if (!unique.has(item.name)) unique.set(item.name, item); return [...unique.values()].sort((left, right) => - left.name.localeCompare(right.name), + compareUnicodeCodePoints(left.name, right.name), ); }; diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 33d06c5eab..a6b500b1b7 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -7,7 +7,10 @@ import { ProviderProcessSupervisor, spawnOwnedProviderProcess, } from "./ProviderProcess.js"; -import { cleanupOwnedProcessGroup } from "./ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type ProcessCleanupFailure, +} from "./ProcessOwnership.js"; import { ProviderStartupDeadline, waitForAbortableDelay, @@ -15,16 +18,19 @@ import { /** A short-lived command failed its lifecycle or complete-output contract. */ export class OwnedCommandFailure extends Error { + readonly cleanupResult: ProcessCleanupFailure | undefined; + constructor( readonly reason: "cancelled" | "timeout" | "output-limit" | "process", message: string, readonly snapshot: ProviderProcessSnapshot | null = null, readonly cleanupFailure: string | null = null, - options?: ErrorOptions, + options?: ErrorOptions & { readonly cleanupResult?: ProcessCleanupFailure }, readonly resources: readonly string[] = [], readonly cleanupOwner?: ProviderProcessSupervisor, ) { super(message, options); + this.cleanupResult = options?.cleanupResult; } } @@ -190,7 +196,12 @@ const collectOwnedCommand = async ( failure?.message ?? "Command cleanup failed.", snapshot, stopped.reason, - failure === undefined ? undefined : { cause: failure }, + { + ...(failure === undefined ? {} : { cause: failure }), + ...(stopped.cleanupResult === undefined + ? {} + : { cleanupResult: stopped.cleanupResult }), + }, [ spawn.runId, ...(spawn.cwd === undefined ? [] : [spawn.cwd]), @@ -205,7 +216,12 @@ const collectOwnedCommand = async ( failure.message, snapshot, failure.cleanupFailure, - { cause: failure }, + { + cause: failure, + ...(failure.cleanupResult === undefined + ? {} + : { cleanupResult: failure.cleanupResult }), + }, failure.resources, ); if (callerSignal?.aborted) diff --git a/src/process/ProcessOwnership.part2.test.ts b/src/process/ProcessOwnership.part2.test.ts index af34fde651..219b8095da 100644 --- a/src/process/ProcessOwnership.part2.test.ts +++ b/src/process/ProcessOwnership.part2.test.ts @@ -111,7 +111,7 @@ describe("owned process-group cleanup discovery", () => { await expect(cleanupOwnedProcessGroup(ownership, adapter)).resolves.toEqual( { cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 104", failures: [{ pid: 104, reason: "run-token-mismatch" }], }, ); diff --git a/src/process/ProcessOwnership.test.ts b/src/process/ProcessOwnership.test.ts index 1589cce788..5f1906a3ab 100644 --- a/src/process/ProcessOwnership.test.ts +++ b/src/process/ProcessOwnership.test.ts @@ -331,7 +331,7 @@ describe("sampled detached process groups during cleanup", () => { expect(result).toMatchObject({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 200", failures: [{ pid: 200, reason: "run-token-mismatch" }], }); expect(signalGroup).toHaveBeenCalledTimes(1); @@ -369,7 +369,7 @@ describe("sampled detached process groups during cleanup", () => { expect(result).toMatchObject({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 200", failures: [{ pid: 200, reason: "run-token-mismatch" }], }); expect(signalGroup).not.toHaveBeenCalled(); diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index 2582dcf5de..af85d23b47 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -8,6 +8,7 @@ import { import { launcherIdentityFailure } from "./ProcessOwnershipIdentity.js"; import { descendantsOf, liveProcesses } from "./ProcessOwnershipProcessTree.js"; import { readProcessRunTokens } from "./ProcessRunTokenObservations.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const execFileAsync = promisify(execFile); @@ -157,19 +158,15 @@ export type ProcessCleanupResult = readonly signaled: boolean; readonly unverified?: readonly OwnershipSweepUnverifiedProcess[]; } - | { - readonly cleaned: false; - readonly reason: string; - readonly failures?: readonly { - readonly pid: number; - readonly reason: - | "environment-unreadable" - | "run-token-mismatch" - | "process-identity-unavailable"; - readonly diagnostic?: string; - }[]; - readonly unverified?: readonly OwnershipSweepUnverifiedProcess[]; - }; + | ProcessCleanupFailure; + +/** Retained verification failures for one cleanup attempt, without additional signaling authority. */ +export interface ProcessCleanupFailure { + readonly cleaned: false; + readonly reason: string; + readonly failures?: readonly ProcessOwnershipValidationFailure[]; + readonly unverified?: readonly OwnershipSweepUnverifiedProcess[]; +} /** Token-verified liveness result used by post-root-exit settlement. */ export type ProcessGroupObservation = @@ -591,7 +588,15 @@ const cleanupValidationFailure = ( if (failures.some(({ reason }) => reason === "run-token-mismatch")) return { cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: `process ownership token mismatch for PID(s): ${[ + ...new Set( + failures + .filter(({ reason }) => reason === "run-token-mismatch") + .map(({ pid }) => pid), + ), + ] + .sort((left, right) => left - right) + .join(", ")}`, failures, ...unverifiedEntry, }; @@ -609,7 +614,7 @@ const cleanupValidationFailure = ( counts.set(category, (counts.get(category) ?? 0) + 1); } const breakdown = [...counts] - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([category, count]) => `${category}=${String(count)}`) .join(", "); const candidates = unreadable @@ -617,7 +622,7 @@ const cleanupValidationFailure = ( ({ pid, diagnostic }) => `${String(pid)}=${sanitizedTokenReadFailure(diagnostic)}`, ) - .sort((left, right) => left.localeCompare(right)); + .sort((left, right) => compareUnicodeCodePoints(left, right)); return { cleaned: false, reason: `process ownership token could not be read for ${String(unreadable.length)} live process(es): ${breakdown}; live candidates ${candidates.join(", ")}`, diff --git a/src/process/ProcessOwnership.validation.test.ts b/src/process/ProcessOwnership.validation.test.ts index ddbcb7fe42..c718f4b936 100644 --- a/src/process/ProcessOwnership.validation.test.ts +++ b/src/process/ProcessOwnership.validation.test.ts @@ -74,7 +74,7 @@ describe("owned process-group cleanup validation: ownership and lineage", () => }); expect(await cleanupOwnedProcessGroup(ownership, adapter)).toEqual({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 101", failures: [{ pid: 101, reason: "run-token-mismatch" }], }); expect(signalGroup).not.toHaveBeenCalled(); @@ -106,7 +106,7 @@ describe("owned process-group cleanup validation: ownership and lineage", () => }; expect(await cleanupOwnedProcessGroup(ownership, adapter)).toEqual({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 101", failures: [ expect.objectContaining({ pid: 100, @@ -189,7 +189,7 @@ describe("owned process-group cleanup liveness rechecks", () => { cleanupOwnedProcessGroup(ownership, adapter), ).resolves.toEqual({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 101", failures: recheckFails ? [ { @@ -345,7 +345,7 @@ describe("owned process-group cleanup validation: launcher exit races", () => { if (remaining === "foreign") expect(result).toMatchObject({ cleaned: false, - reason: "process tree contains an unowned or PID-reused process", + reason: "process ownership token mismatch for PID(s): 101", }); else expect(result).toEqual({ diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index b6765b93f7..1768c6c5bc 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -6,6 +6,7 @@ import { WindowsOwnedProcess } from "../windows/WindowsOwnedProcess.js"; import type { OwnedProcessGroup, + ProcessCleanupFailure, ProcessCleanupResult, } from "./ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "./ProcessOwnershipObservation.js"; @@ -143,7 +144,11 @@ export type ProviderProcessStopResult = | "terminated" | "killed"; } - | { readonly status: "incomplete"; readonly reason: string }; + | { + readonly status: "incomplete"; + readonly reason: string; + readonly cleanupResult?: ProcessCleanupFailure; + }; /** * Spawn a provider in an owned Windows job or dedicated POSIX process group. @@ -373,13 +378,21 @@ export class ProviderProcessSupervisor { if (this.launch.cleanup !== undefined) { const cleanup = await this.#ownedCleanup(options.cleanupResult); if (!cleanup.cleaned) - return { status: "incomplete", reason: cleanup.reason }; + return { + status: "incomplete", + reason: cleanup.reason, + cleanupResult: cleanup, + }; const graceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS; if (await this.waitForExit(graceMs)) return { status: "verified-cleanup" }; const retried = await this.launch.cleanup(); if (!retried.cleaned) - return { status: "incomplete", reason: retried.reason }; + return { + status: "incomplete", + reason: retried.reason, + cleanupResult: retried, + }; if (await this.waitForExit(graceMs)) return { status: "verified-cleanup" }; return { diff --git a/src/server/registerAndroidTools.ts b/src/server/registerAndroidTools.ts index 887e263ddd..722bd6695f 100644 --- a/src/server/registerAndroidTools.ts +++ b/src/server/registerAndroidTools.ts @@ -1,9 +1,13 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { AndroidAnalysisService } from "../application/android/AndroidAnalysisService.js"; +import { summarizeRetainedAndroidSearch } from "../application/android/AndroidSearchEvidenceSummary.js"; import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "pino"; -import { createContractToolHandler } from "./contractToolHandler.js"; +import { + createContractToolHandler, + type ContractToolService, +} from "./contractToolHandler.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; @@ -22,6 +26,23 @@ export const registerAndroidTools = ( recordEvidence, withAdmittedAnalysis, ); + // Focused class searches keep complete raw inventory in the retained + // record and deliver a bounded summary inline; the generic handler then + // records and delivers the summary as the search contract's Evidence. + const searchService: ContractToolService<"search_android_classes"> = { + execute: async (operation, input, options) => { + const executed = await service.execute(operation, input, options); + if (!executed.ok) return executed; + return summarizeRetainedAndroidSearch(executed.value, recordEvidence); + }, + }; + const searchHandler = createContractToolHandler( + server, + searchService, + logger, + recordEvidence, + withAdmittedAnalysis, + ); const packageContract = toolContract("inspect_android_package"); server.registerTool( packageContract.name, @@ -32,7 +53,7 @@ export const registerAndroidTools = ( server.registerTool( searchContract.name, toolRegistrationOptions(searchContract), - handler(searchContract), + searchHandler(searchContract), ); const classContract = toolContract("inspect_android_class"); server.registerTool( diff --git a/src/server/registerInvestigationTools/helpers.ts b/src/server/registerInvestigationTools/helpers.ts index 1d342c0b56..557f96225f 100644 --- a/src/server/registerInvestigationTools/helpers.ts +++ b/src/server/registerInvestigationTools/helpers.ts @@ -5,6 +5,7 @@ import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import { recordDerivedEvidence } from "../recordDerivedEvidence.js"; import type { WorkflowUnknownInput } from "./types.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; export const isIncomplete = (status: string): boolean => status === "unknown" || status === "truncated"; @@ -21,7 +22,9 @@ export const functionEvidenceIds = (records: readonly Evidence[]): string[] => uniqueIds(records.map(({ evidence_id }) => evidence_id)); const uniqueIds = (ids: readonly string[]): string[] => - [...new Set(ids)].sort((left, right) => left.localeCompare(right)); + [...new Set(ids)].sort((left, right) => + compareUnicodeCodePoints(left, right), + ); export const evidenceClosure = ( session: BinarySessionPort, diff --git a/tests/acceptance/applications/electronModuleInteropCli.test.ts b/tests/acceptance/applications/electronModuleInteropCli.test.ts new file mode 100644 index 0000000000..d4191a2131 --- /dev/null +++ b/tests/acceptance/applications/electronModuleInteropCli.test.ts @@ -0,0 +1,196 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; +import { z } from "zod"; + +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { ROLLDOWN_MODULE_INTEROP } from "../../../src/domain/javascript/javascriptModuleInterop.fixture.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const INTRINSIC_MUTATIONS = [ + ["poisoned.cjs", "({Object} = replacements);"], + [ + "projected.cjs", + 'const {defineProperty:change} = Object; change(Object,"create",{value:customCreate});', + ], + [ + "decorated.cjs", + '@(() => Object) class metadata {}; Object.defineProperty(metadata,"create",{value:customCreate});', + ], + [ + "uncurried.cjs", + 'const change = Function.prototype.call.bind(Object.defineProperty,null); change(Object,"create",{value:customCreate});', + ], + [ + "generic-bind.cjs", + 'const change = Function.prototype.bind.call(Object.defineProperty,null); change(Object,"create",{value:customCreate});', + ], + [ + "replaced-apply.cjs", + 'Reflect.apply = Object.defineProperty; Reflect.apply(Object,"create",{value:customCreate});', + ], + [ + "bound.cjs", + 'const change = Object.defineProperty.bind(null,Object,"create"); change({value:customCreate});', + ], + ["receiver.cjs", 'Reflect.set({},"create",customCreate,Object);'], + [ + "deep-call.cjs", + 'Function.prototype.call.call.call(Object.defineProperty,null,Object,"create",{value:customCreate});', + ], + [ + "deep-apply.cjs", + 'Function.prototype.apply.apply(Object.defineProperty,[null,[Object,"create",{value:customCreate}]]);', + ], +] as const; + +const writePoisonedModules = async (root: string): Promise => { + for (const [name, mutation] of INTRINSIC_MUTATIONS) + await writeFile( + join(root, name), + `${mutation} + ${ROLLDOWN_MODULE_INTEROP} + let poisoned = require("electron"); + poisoned = __toESM(poisoned, 1); + new poisoned.BrowserWindow({}); + throw new Error("must not execute poisoned source");`, + ); +}; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains converted Electron windows and preload evidence through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-electron-interop-"); + await writeFile( + join(root, "package.json"), + JSON.stringify({ main: "main.cjs" }), + ); + await writeFile( + join(root, "preload.js"), + 'throw new Error("must stay inert");', + ); + await writePoisonedModules(root); + const source = `const metadata = {}; + Object.defineProperty(metadata,"x",{value:1}); + const decorateMetadata = Object.defineProperty.bind(null,metadata,"bound"); + decorateMetadata({value:1}); + const decorateViaCall = Function.prototype.call.bind(Object.defineProperty); + decorateViaCall(null,metadata,"uncurried",{value:1}); + const localFunction = () => {}; + Object.defineProperty(localFunction,"x",{value:1}); + const localClass = class {}; + Object.defineProperty(localClass,"x",{value:1}); + const localRegExp = /local/; + Object.defineProperty(localRegExp,"x",{value:1}); + Object.defineProperty(hoistedFunction,"x",{value:1}); + function hoistedFunction() {} + const captured = Object.defineProperty; + Object.defineProperty = custom; + ${ROLLDOWN_MODULE_INTEROP.replace("__defProp = Object.defineProperty", "__defProp = captured")} + let electron = require("electron"); + electron = __toESM(electron, 1); + new electron.BrowserWindow({webPreferences:{preload:"./preload.js"}}); + new electron.BrowserWindow({webPreferences:{preload:runtimePath}}); + var direct; + var direct = require("electron"); + var direct; + new direct.BrowserWindow({}); + const fresh = __toESM(require("electron"), 1); + new fresh.BrowserWindow({}); + const {BrowserWindow: ProjectedWindow} = __toESM(require("electron")); + new ProjectedWindow({}); + const nonNode = __toESM(require("electron"), 0); + new nonNode.BrowserWindow({}); + function identity(mod) {return mod;} + const passthrough = identity(require("electron")); + new passthrough.BrowserWindow({}); + const __importStar = () => ({BrowserWindow:FakeWindow}); + const fake = __importStar(require("electron")); + new fake.BrowserWindow({}); + function decoy(electron) {new electron.BrowserWindow({});} + require("./poisoned.cjs"); + require("./projected.cjs"); + require("./decorated.cjs"); + require("./bound.cjs"); + require("./uncurried.cjs"); + require("./generic-bind.cjs"); + require("./replaced-apply.cjs"); + require("./receiver.cjs"); + require("./deep-call.cjs"); + require("./deep-apply.cjs"); + throw new Error("must not execute main"); + `; + await writeFile(join(root, "main.cjs"), source); + let document: unknown; + if (surface === "CLI") { + const response = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(response.exitCode, response.stderr).toBe(0); + document = response.json; + } else { + const { call } = await connectLocalToolsMcp(); + const response = await call("analyze_javascript_application", { + input_path: root, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + document = response.structuredContent; + } + const result = z + .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) + .parse(document).normalized_result; + expect(result.summary).toMatchObject({ + browser_windows: 7, + preload_entrypoints: 1, + }); + expect(result.statistics.parse_failures).toBe(0); + expect(result.statistics.parsed_javascript_files).toBe(12); + const windows = result.graph.nodes.filter( + ({ kind }) => kind === "browser-window", + ); + expect(windows).toHaveLength(7); + const expectedLines = source + .split("\n") + .flatMap((line, index) => + /^\s*new (?:(?:electron|direct|fresh|passthrough|nonNode)\.BrowserWindow|ProjectedWindow)\(/u.test( + line, + ) + ? [index + 1] + : [], + ); + const observedLines = windows.flatMap((window) => { + const location = window.observations[0]?.evidence.location; + return location?.available && location.value.kind === "source-range" + ? [location.value.start.line] + : []; + }); + expect(observedLines.toSorted()).toEqual(expectedLines.toSorted()); + for (const window of windows) { + expect(window.observations[0]?.evidence.location).toMatchObject({ + available: true, + value: { kind: "source-range", source: "main.cjs" }, + }); + } + expect( + windows.flatMap((window) => + window.observations.map(({ properties }) => properties.web_preferences), + ), + ).toEqual( + expect.arrayContaining([ + expect.arrayContaining([ + expect.objectContaining({ + name: "preload", + value: { + status: "dynamic", + value: null, + expression: "runtimePath", + }, + }), + ]), + ]), + ); + }, +); diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts new file mode 100644 index 0000000000..32d113781e --- /dev/null +++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts @@ -0,0 +1,132 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest.skipIf(process.versions.bun !== undefined).for([ + { laterSources: 1, heapMb: 640, methodsCount: 400, retainGraph: true }, + { laterSources: 10, heapMb: 640, methodsCount: 400, retainGraph: true }, + // Even an otherwise empty parent cannot admit this graph after reserving + // transient decoding and the next checkpoint, independent of GC timing. + { laterSources: 1, heapMb: 448, methodsCount: 600, retainGraph: false }, +])( + "preserves evidence with $laterSources later checkpoints and a $heapMb MiB parent heap", + { timeout: 90_000 }, + async ({ laterSources, heapMb, methodsCount, retainGraph }, { cli }) => { + const root = await createTestTempDirectory("rea-js-checkpoint-reserve-"); + const source = Array.from( + { length: methodsCount }, + (_, index) => + `export function method_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`, + ).join("\n"); + const files: readonly (readonly [string, string])[] = [ + ["a-large.mjs", source], + ...Array.from( + { length: laterSources }, + (_, index) => + [ + `z-tiny-${String(index)}.mjs`, + `export const tiny${String(index)} = 1;\n`, + ] as const, + ), + ]; + for (const [name, text] of files) await writeFile(join(root, name), text); + const response = await cli.run({ + arguments: ["analyze", root, "--format", "json"], + environment: { + NODE_OPTIONS: `--max-old-space-size=${String(heapMb)} --max-semi-space-size=8`, + }, + timeoutMs: 60_000, + }); + expect(response.exitCode).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + expect(result.statistics.parsed_javascript_files).toBe(laterSources + 1); + expect(result.statistics.parse_failures).toBe(0); + const failures = result.graph.nodes.flatMap(({ observations }) => + observations.flatMap(({ properties }) => + properties.analysis_failure === undefined + ? [] + : [properties.analysis_failure], + ), + ); + const methods = result.semantic_graph.nodes.filter( + ({ kind, identity }) => + kind === "function" && identity.module_path === "a-large.mjs", + ); + if (retainGraph) + expect(methods.map(({ label }) => label).sort()).toEqual( + Array.from( + { length: methodsCount }, + (_, index) => `method_${String(index)}`, + ).sort(), + ); + else { + expect(methods).toHaveLength(0); + expect([ + result.graph.coverage.status, + result.semantic_graph.coverage.status, + ]).toContain("partial"); + } + expect( + result.semantic_graph.nodes.some( + ({ identity, label }) => + identity.module_path === `z-tiny-${String(laterSources - 1)}.mjs` && + label === `tiny${String(laterSources - 1)}`, + ), + ).toBe(true); + for (const [name, text] of files) { + const digest = createHash("sha256").update(text).digest("hex"); + const node = result.graph.nodes.find( + ({ kind, identity }) => + kind === "javascript-asset" && + identity.strategy === "content-digest" && + identity.sha256 === digest, + ); + expect(node?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + path: name, + parse_status: "complete", + }), + }), + ]), + ); + if (retainGraph) expect(failures).toEqual([]); + expect(await readFile(join(root, name), "utf8")).toBe(text); + if (!retainGraph && name === "a-large.mjs") { + const failure = z + .object({ + code: z.literal("resource_constraint"), + remediation: z.object({ action: z.string() }), + details: z.object({ + reported_limits: z.object({ + parent_available_result_budget_bytes: z.number(), + module_retention_estimate_bytes: z.number(), + graph_retention_estimate_bytes: z.number(), + }), + }), + }) + .parse(failures[0]); + const limits = failure.details.reported_limits; + expect( + limits.module_retention_estimate_bytes + + limits.graph_retention_estimate_bytes, + ).toBeGreaterThan(limits.parent_available_result_budget_bytes); + expect(failure.remediation.action).toContain("parent result budget"); + expect(failure.remediation.action).toContain( + "retained static and module facts", + ); + } + } + expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); + }, +); diff --git a/tests/acceptance/applications/packageImportAliasesCli.test.ts b/tests/acceptance/applications/packageImportAliasesCli.test.ts new file mode 100644 index 0000000000..7d3187a7f5 --- /dev/null +++ b/tests/acceptance/applications/packageImportAliasesCli.test.ts @@ -0,0 +1,282 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join, relative } from "node:path"; + +import { expect } from "vitest"; +import { z } from "zod"; + +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const literalAliases = ["#/x", "#x%2Fy", "#x%2Ey", "#x\\y", "#x\0y"]; +const fallbackAliases = { + "#tab-escape": ["./.\t./outside.cjs", "./config.cjs"], + "#newline-escape": ["./.%\n2e/outside.cjs", "./config.cjs"], + "#normalized-url": "./con\tfig.cjs", + "#trimmed-url": "./config.cjs \t", + "#url-suffix": "./config.cjs?selected=one#fragment", + "#inactive-array": { + node: [{ browser: "./browser.cjs" }], + default: "./config.cjs", + }, + "#nested-inactive-array": { + node: [[{ browser: "./browser.cjs" }]], + default: "./config.cjs", + }, + "#parent": ["../outside.cjs", "./config.cjs"], + "#absolute": { node: [["/outside.cjs"], "./config.cjs"] }, + "#url": { + import: ["https://example.invalid/outside.cjs", "./config.cjs"], + require: ["file:///outside.cjs", "./config.cjs"], + }, +}; +const resolvedAliases = [...literalAliases, ...Object.keys(fallbackAliases)]; +const refusedAliases = { + "#empty": { + target: [], + status: "external", + limitation: "an empty imports target array", + }, + "#empty-active": { + target: { node: [] }, + status: "external", + limitation: "an empty imports target array", + }, + "#null-last": { + target: [[], null], + status: "external", + limitation: "an explicit null imports target", + }, + "#empty-last": { + target: [null, []], + status: "external", + limitation: "an empty imports target array", + }, + "#inactive": { + target: [{ browser: "./browser.cjs" }], + status: "external", + limitation: "it declares browser.", + }, + "#nested-inactive": { + status: "external", + target: [ + [{ browser: "./browser.cjs" }, { "rea-inactive": "./inactive.cjs" }], + ], + limitation: "it declares browser, rea-inactive.", + }, + "#escape": { + target: "./.\t./outside.cjs", + status: "rejected", + limitation: "escapes the package root", + }, + "#encoded-separator": { + target: ["./con%2ffig.cjs", "./config.cjs"], + status: "rejected", + limitation: "separator bytes", + }, + "#missing-url": { + target: ["./miss\ting.cjs", "./config.cjs"], + status: "not-found", + limitation: JSON.stringify("./miss\ting.cjs"), + }, +}; +const allAliases = [...resolvedAliases, ...Object.keys(refusedAliases)]; + +const createPackageImportAliasFixture = async () => { + const root = await createTestTempDirectory("rea-package-import-aliases-"); + const mainSource = [ + 'const config = require("#config");', + 'const nested = require("./node_modules/fixture/index.cjs");', + 'const missing = require("#missing");', + ...allAliases.map((specifier) => `require(${JSON.stringify(specifier)});`), + ].join("\n"); + const moduleSource = allAliases + .map((specifier) => `import ${JSON.stringify(specifier)};`) + .join("\n"); + const nestedSource = [ + 'const config = require("#config");', + "module.exports = config;", + ].join("\n"); + const files = { + "package.json": JSON.stringify({ + name: "app", + imports: { + "#config": "./config.cjs", + ...Object.fromEntries( + literalAliases.map((specifier) => [specifier, "./config.cjs"]), + ), + ...fallbackAliases, + ...Object.fromEntries( + Object.entries(refusedAliases).map(([specifier, { target }]) => [ + specifier, + target, + ]), + ), + }, + }), + "main.cjs": mainSource, + "main.mjs": moduleSource, + "config.cjs": "module.exports = 'app-config';", + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + imports: { "#config": "./local.cjs" }, + }), + "node_modules/fixture/index.cjs": nestedSource, + "node_modules/fixture/local.cjs": "module.exports = 'fixture-config';", + }; + await Promise.all( + Object.entries(files).map(async ([path, content]) => { + const absolute = join(root, path); + await mkdir(dirname(absolute), { recursive: true }); + await writeFile(absolute, content); + }), + ); + return { root, mainSource, moduleSource, nestedSource }; +}; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "resolves exact package import aliases with package scope through %s", + async (surface, { cli }) => { + const { root, mainSource, moduleSource, nestedSource } = + await createPackageImportAliasFixture(); + const nativeRequire = createRequire(join(root, "main.cjs")); + expect( + relative(root, nativeRequire.resolve("#config")).replaceAll("\\", "/"), + ).toBe("config.cjs"); + expect( + relative( + root, + createRequire(join(root, "node_modules/fixture/index.cjs")).resolve( + "#config", + ), + ).replaceAll("\\", "/"), + ).toBe("node_modules/fixture/local.cjs"); + expect(() => nativeRequire.resolve("#missing")).toThrow( + expect.objectContaining({ code: "ERR_PACKAGE_IMPORT_NOT_DEFINED" }), + ); + + let document: unknown; + if (surface === "CLI") { + const response = await cli.run({ + arguments: ["analyze-javascript-application", root, "--format", "json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + document = response.json; + } else { + const { call } = await connectLocalToolsMcp(); + const response = await call("analyze_javascript_application", { + input_path: root, + format: "directory", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + document = response.structuredContent; + } + const { graph } = z + .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) + .parse(document).normalized_result; + for (const specifier of resolvedAliases) + for (const [moduleKind, sourcePath, source] of [ + ["require", "main.cjs", mainSource], + ["import", "main.mjs", moduleSource], + ] as const) + expect(graph.edges).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + module_link_kind: moduleKind, + specifier, + resolved_path: "config.cjs", + resolution_status: "resolved", + }), + evidence: sourceEvidence(sourcePath, source), + }), + ); + const imports = graph.edges.filter( + ({ properties }) => properties.module_link_kind === "require", + ); + for (const [specifier, { status, limitation }] of Object.entries( + refusedAliases, + )) + for (const [moduleKind, sourcePath, source] of [ + ["require", "main.cjs", mainSource], + ["import", "main.mjs", moduleSource], + ] as const) + expect(graph.edges).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + module_link_kind: moduleKind, + specifier, + resolved_path: null, + resolution_status: status, + }), + evidence: sourceEvidence(sourcePath, source, limitation), + }), + ); + expect(imports).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + specifier: "#config", + resolved_path: "config.cjs", + resolution_status: "resolved", + }), + evidence: sourceEvidence("main.cjs", mainSource), + }), + ); + expect(imports).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + specifier: "#config", + resolved_path: "node_modules/fixture/local.cjs", + resolution_status: "resolved", + }), + evidence: sourceEvidence( + "node_modules/fixture/index.cjs", + nestedSource, + ), + }), + ); + expect(imports).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + specifier: "#missing", + resolved_path: null, + resolution_status: "external", + }), + evidence: expect.objectContaining({ + limitations: expect.arrayContaining([ + expect.stringContaining("does not define package import alias"), + ]), + }), + }), + ); + }, +); + +const sourceEvidence = ( + sourcePath: string, + source: string, + limitation?: string, +): unknown => + expect.objectContaining({ + artifact: expect.objectContaining({ + available: true, + sha256: createHash("sha256").update(source).digest("hex"), + }), + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + kind: "source-range", + source: sourcePath, + }), + }), + ...(limitation === undefined + ? {} + : { + limitations: expect.arrayContaining([ + expect.stringContaining(limitation), + ]), + }), + }); diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index a336a2a063..d8a4bb998f 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -7,6 +7,72 @@ import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "managed CLI retains metadata and CIL after repeated local constructor owner lookups", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-managed-row-lookup-"); + const path = join(root, "local-attributes.dll"); + await writeFile( + path, + buildManagedPeFixture({ + additionalEmptyTypes: 512, + customAttributeRowCount: 512, + customAttributeTypeRaw: (1 << 3) | 2, + typeName: "LookupAttribute", + extendsRaw: 0, + methodName: ".ctor", + methodFlags: 0x1886, + methodSignature: Buffer.from([0x20, 0x01, 0x01, 0x0e]), + ilBody: Buffer.from([0x06, 0x2a]), + }), + ); + const artifact = await cli.run({ + arguments: ["inspect-managed-artifact", path, "--json"], + }); + expect(artifact.exitCode).toBe(0); + expect(artifact.json).toMatchObject({ + normalized_result: { + coverage: { state: "complete", issues: [] }, + attributes: expect.arrayContaining([ + expect.objectContaining({ + type_name: "Fixture.LookupAttribute", + constructor_token: "0x06000001", + }), + ]), + }, + }); + expect(artifact.json).toHaveProperty( + "normalized_result.attributes.length", + 512, + ); + const members = await cli.run({ + arguments: ["inspect-managed-members", path, "--json"], + }); + expect(members.exitCode).toBe(0); + expect(members.json).toMatchObject({ + normalized_result: { + coverage: { state: "complete", issues: [] }, + types: expect.arrayContaining([ + expect.objectContaining({ full_name: "Fixture.Empty0" }), + expect.objectContaining({ full_name: "Fixture.Empty511" }), + expect.objectContaining({ full_name: "Fixture.LookupAttribute" }), + ]), + methods: [ + expect.objectContaining({ + name: ".ctor", + declaring_type: "Fixture.LookupAttribute", + body: expect.objectContaining({ + status: "present", + instruction_count: 1, + }), + }), + ], + }, + }); + expect(members.json).toHaveProperty("normalized_result.types.length", 513); + }, +); + cliTest( "managed CLI separates .NET Standard API targets from runtime evidence", async ({ cli }) => { diff --git a/tests/acceptance/managedMemberPointers.test.ts b/tests/acceptance/managedMemberPointers.test.ts new file mode 100644 index 0000000000..53fad72573 --- /dev/null +++ b/tests/acceptance/managedMemberPointers.test.ts @@ -0,0 +1,94 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { expect } from "vitest"; + +import { parseEvidence } from "../../src/domain/evidence.js"; +import { buildManagedMemberPointerFixture } from "../../src/dotnet/ManagedPe.fixture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "CLI and stdio MCP retain physical member identities through reordered pointers", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-managed-member-pointers-"); + const path = join(root, "reordered.dll"); + const bytes = buildManagedMemberPointerFixture(); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await writeFile(path, bytes); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + const client = new Client({ name: "managed-pointer-parity", version: "1" }); + try { + await client.connect(transport); + for (const name of [ + "inspect_managed_artifact", + "inspect_managed_members", + ] as const) { + const cliResult = await cli.run({ + arguments: [name.replaceAll("_", "-"), path, "--json"], + }); + expect(cliResult.exitCode, cliResult.stderr).toBe(0); + const cliEvidence = parseEvidence(cliResult.json); + const response = await client.callTool({ name, arguments: { path } }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcpEvidence = parseEvidence(response.structuredContent); + expect(mcpEvidence.normalized_result).toEqual( + cliEvidence.normalized_result, + ); + expect(mcpEvidence.subject?.digest.sha256).toBe(sha256); + expect(cliEvidence.subject?.digest.sha256).toBe(sha256); + expect(mcpEvidence.normalized_result).toMatchObject({ + coverage: { state: "complete", issues: [] }, + }); + if (name === "inspect_managed_artifact") { + expect(mcpEvidence.normalized_result).toMatchObject({ + attributes: [ + expect.objectContaining({ + constructor_token: "0x06000001", + type_name: "Fixture.B", + }), + ], + }); + } else { + expect(mcpEvidence.normalized_result).toMatchObject({ + fields: [ + expect.objectContaining({ + token: "0x04000001", + declaring_type: "Fixture.B", + }), + expect.objectContaining({ + token: "0x04000002", + declaring_type: "Fixture.A", + }), + ], + methods: [ + expect.objectContaining({ + token: "0x06000001", + declaring_type: "Fixture.B", + }), + expect.objectContaining({ + token: "0x06000002", + declaring_type: "Fixture.A", + }), + ], + }); + } + } + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + }, +); diff --git a/tests/acceptance/managedMetadataExtraData.test.ts b/tests/acceptance/managedMetadataExtraData.test.ts new file mode 100644 index 0000000000..6eb93c0554 --- /dev/null +++ b/tests/acceptance/managedMetadataExtraData.test.ts @@ -0,0 +1,84 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { expect } from "vitest"; + +import { parseEvidence } from "../../src/domain/evidence.js"; +import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "CLI and stdio MCP inspect CLR ExtraData metadata without shifting table identities", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-managed-extra-data-"); + const path = join(root, "extra-data.dll"); + const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 }); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await writeFile(path, bytes); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + const client = new Client({ + name: "managed-extra-data-parity", + version: "1", + }); + try { + await client.connect(transport); + for (const name of [ + "inspect_managed_artifact", + "inspect_managed_members", + ] as const) { + const cliResult = await cli.run({ + arguments: [name.replaceAll("_", "-"), path, "--json"], + }); + expect(cliResult.exitCode, cliResult.stderr).toBe(0); + const cliEvidence = parseEvidence(cliResult.json); + const response = await client.callTool({ name, arguments: { path } }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcpEvidence = parseEvidence(response.structuredContent); + expect(mcpEvidence.normalized_result).toEqual( + cliEvidence.normalized_result, + ); + expect(mcpEvidence.subject?.digest.sha256).toBe(sha256); + expect(cliEvidence.subject?.digest.sha256).toBe(sha256); + expect(mcpEvidence.normalized_result).toMatchObject({ + module: { + name: "Fixture.dll", + mvid: "00112233-4455-6677-8899-aabbccddeeff", + }, + coverage: { state: "complete", issues: [] }, + }); + if (name === "inspect_managed_artifact") { + expect(mcpEvidence.normalized_result).toMatchObject({ + assembly: { name: "Fixture.Managed" }, + }); + } else { + expect(mcpEvidence.normalized_result).toMatchObject({ + types: [expect.objectContaining({ full_name: "Fixture.Program" })], + methods: [ + expect.objectContaining({ + token: "0x06000001", + name: "Main", + declaring_type: "Fixture.Program", + }), + ], + }); + } + } + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + }, +); diff --git a/tests/acceptance/setup/selectedSkillSetup.test.ts b/tests/acceptance/setup/selectedSkillSetup.test.ts new file mode 100644 index 0000000000..bc73c85b3d --- /dev/null +++ b/tests/acceptance/setup/selectedSkillSetup.test.ts @@ -0,0 +1,156 @@ +import { access, readFile, readdir, writeFile } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "skill-only setup verifies its selected client copy on install, rerun and repair", + async ({ cli }) => { + const directory = await createTestTempDirectory("rea-selected-skill-"); + const selected = join( + directory, + ".claude/skills/reverse-engineer-anything", + ); + const shared = join(directory, ".agents/skills/reverse-engineer-anything"); + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd: directory, + environment: { USERPROFILE: directory }, + }); + const preview = await run([ + "setup", + "--skill-client", + "claude_code", + "--dry-run", + ]); + expect(preview.exitCode, preview.stdout).toBe(0); + expect(preview.json).toMatchObject({ + status: "planned", + doctor: { + scope: { + clients: [], + skill: true, + skill_clients: ["claude_code"], + }, + }, + }); + const registered = await run([ + "setup", + "--client", + "claude_code", + "--skill=false", + "--yes", + ]); + expect(registered.exitCode, registered.stderr).toBe(0); + const configuration = join(directory, ".claude.json"); + const before = await readFile(configuration, "utf8"); + const args = ["setup", "--skill-client", "claude_code", "--yes"]; + const installed = await run(args); + expect(installed.exitCode, installed.stdout).toBe(0); + expect(installed.json).toMatchObject({ + status: "ready", + appliedActions: ["installed_skill"], + doctor: { + healthy: true, + scope: { clients: [], skill: true, skill_clients: ["claude_code"] }, + identity: { skill: { state: "aligned" } }, + }, + }); + const bundle = resolve("skills/reverse-engineer-anything"); + const assertAligned = async () => { + for (const entry of await readdir(bundle, { + recursive: true, + withFileTypes: true, + })) { + if (!entry.isFile()) continue; + const path = relative(bundle, join(entry.parentPath, entry.name)); + expect(await readFile(join(selected, path))).toEqual( + await readFile(join(bundle, path)), + ); + } + }; + await assertAligned(); + const repeated = await run(args); + expect(repeated.exitCode, repeated.stdout).toBe(0); + expect(repeated.json).toMatchObject({ + status: "ready", + appliedActions: [], + }); + + await writeFile( + join(selected, "references/native-and-artifacts.md"), + "damaged selected copy\n", + ); + const damaged = await run(["doctor", "--client", "claude_code", "--skill"]); + expect(damaged.json).toMatchObject({ + healthy: false, + identity: { skill: { state: "stale" } }, + }); + const repaired = await run(args); + expect(repaired.exitCode, repaired.stdout).toBe(0); + expect(repaired.json).toMatchObject({ + status: "ready", + appliedActions: ["installed_skill"], + }); + await assertAligned(); + await expect(access(shared)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(configuration, "utf8")).toBe(before); + + const sharedControl = await run([ + "setup", + "--skill-client", + "shared", + "--yes", + ]); + expect(sharedControl.exitCode, sharedControl.stdout).toBe(0); + expect(sharedControl.json).toMatchObject({ status: "ready" }); + await access(join(shared, "SKILL.md")); + }, +); + +cliTest( + "setup keeps registration and skill readiness selections independent", + async ({ cli }) => { + const directory = await createTestTempDirectory("rea-independent-skill-"); + const result = await cli.run({ + arguments: [ + "setup", + "--client", + "codex", + "--skill-client", + "claude_code", + "--yes", + "--json", + ], + cwd: directory, + environment: { USERPROFILE: directory }, + }); + expect(result.exitCode, result.stdout).toBe(0); + expect(result.json).toMatchObject({ + status: "ready", + doctor: { + healthy: true, + scope: { + clients: ["codex"], + skill_clients: ["claude_code"], + skill: true, + }, + identity: { skill: { state: "aligned" } }, + }, + }); + await access(join(directory, ".codex/config.toml")); + await access( + join(directory, ".claude/skills/reverse-engineer-anything/SKILL.md"), + ); + await expect(access(join(directory, ".claude.json"))).rejects.toMatchObject( + { code: "ENOENT" }, + ); + await expect( + access(join(directory, ".agents/skills/reverse-engineer-anything")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, +); diff --git a/tests/boundary/browser/cdpSvgXlinkHref.test.ts b/tests/boundary/browser/cdpSvgXlinkHref.test.ts index be4fe268ef..56c3fc3cfa 100644 --- a/tests/boundary/browser/cdpSvgXlinkHref.test.ts +++ b/tests/boundary/browser/cdpSvgXlinkHref.test.ts @@ -29,6 +29,11 @@ const chromeExecutable = [ existsSync(candidate), ); +// Match the real-browser verifier: a cold Chrome launch on a loaded coverage +// runner can exceed 20 seconds before publishing DevToolsActivePort. +const CHROME_STARTUP_TIMEOUT_MS = 60_000; +const CHROME_TEST_TIMEOUT_MS = CHROME_STARTUP_TIMEOUT_MS + 30_000; + // Issue 1586's page, plus the HTML, image, use, and foreign-origin cases the // real snapshot must distinguish. The XHTML page is the prefixed-svg gap // reported on PR 1602. The tests are skipped when no Chrome-family executable @@ -90,7 +95,7 @@ it.skipIf(chromeExecutable === undefined)( executable, activePortPath: join(profile, "DevToolsActivePort"), stderr: () => stderr, - timeoutMs: 20_000, + timeoutMs: CHROME_STARTUP_TIMEOUT_MS, }); const endpoint = `http://127.0.0.1:${String(port)}`; const provider = new CdpBrowserProvider(); @@ -158,6 +163,7 @@ it.skipIf(chromeExecutable === undefined)( }); } }, + CHROME_TEST_TIMEOUT_MS, ); const xhtmlPage = ` @@ -261,7 +267,7 @@ for (const { format, page, contentType, nodeNames, link, destinations } of [ executable, activePortPath: join(profile, "DevToolsActivePort"), stderr: () => stderr, - timeoutMs: 20_000, + timeoutMs: CHROME_STARTUP_TIMEOUT_MS, }); const endpoint = `http://127.0.0.1:${String(port)}`; const provider = new CdpBrowserProvider(); @@ -312,6 +318,7 @@ for (const { format, page, contentType, nodeNames, link, destinations } of [ }); } }, + CHROME_TEST_TIMEOUT_MS, ); } diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.ts index 7a3698d778..5c2ce87854 100644 --- a/tests/boundary/browser/webCaptureDiff.test.ts +++ b/tests/boundary/browser/webCaptureDiff.test.ts @@ -446,6 +446,92 @@ describe("web capture WebSocket payload byte coverage", () => { }); describe("web capture diff", () => { + it("treats unavailable HTTP response facts as unknown while retaining observed changes", async () => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const captured = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }), + ); + if (!captured.ok) throw captured.error; + const before = structuredClone(captured.value); + const request = before.network.requests[0]; + if (request === undefined) throw new Error("Missing network request"); + request.status = null; + request.mime_type = null; + request.encoded_data_length = null; + markSectionsComplete(before, [ + "network_requests", + "websocket_connections", + "websocket_frames", + "json_body_shapes", + ]); + for (const item of before.network.requests) + item.body_shapes.status = "not_approved"; + + const after = structuredClone(before); + const responseObserved = after.network.requests[0]; + if (responseObserved === undefined) + throw new Error("Missing network request"); + responseObserved.status = 200; + responseObserved.mime_type = "application/json"; + responseObserved.encoded_data_length = 100; + expect(compareNetwork(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + expect(compareNetwork(after, before)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + expect(compareNetwork(before, before).status).toBe("unknown"); + + for (const field of [ + "status", + "mime_type", + "encoded_data_length", + ] as const) { + const partial = structuredClone(after); + partial.network.requests[0]![field] = null; + expect(compareNetwork(partial, after).status).toBe("unknown"); + expect(compareNetwork(after, partial).status).toBe("unknown"); + } + + const inventoryBefore = structuredClone(before); + inventoryBefore.network.requests = []; + expect(compareNetwork(inventoryBefore, before)).toMatchObject({ + status: "changed", + total_changes: 1, + changes: [expect.objectContaining({ change: "added" })], + }); + + const statusChanged = structuredClone(after); + statusChanged.network.requests[0]!.status = 500; + expect(compareNetwork(after, statusChanged)).toMatchObject({ + status: "changed", + changes: [expect.objectContaining({ change: "modified" })], + }); + after.network.requests[0]!.encoded_data_length = null; + expect(compareNetwork(after, statusChanged).status).toBe("changed"); + + const repeated = structuredClone(statusChanged); + const duplicate = structuredClone(repeated.network.requests[0]!); + duplicate.status = 200; + duplicate.mime_type = "text/plain"; + repeated.network.requests.push(duplicate); + const reassociated = structuredClone(repeated); + reassociated.network.requests[0]!.mime_type = "text/plain"; + reassociated.network.requests.at(-1)!.mime_type = "application/json"; + expect(compareNetwork(repeated, reassociated).status).toBe("changed"); + repeated.network.requests[0]!.status = null; + expect(compareNetwork(repeated, reassociated).status).toBe("unknown"); + }); + it("ignores transient request IDs and capture-approval state", async () => { const browser = await startFakeCdpBrowser(); browsers.push(browser); diff --git a/tests/boundary/cli/javascriptSelfImportRelationships.test.ts b/tests/boundary/cli/javascriptSelfImportRelationships.test.ts index c3e1238bef..44b741d4cd 100644 --- a/tests/boundary/cli/javascriptSelfImportRelationships.test.ts +++ b/tests/boundary/cli/javascriptSelfImportRelationships.test.ts @@ -11,6 +11,78 @@ import { writeFixtureFiles, } from "../../support/javascriptApplicationFixture.js"; +cliTest( + "resolves emitted module imports without substituting Electron runtime paths", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-typescript-runtime-paths-"); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: "main.ts" }), + "main.ts": `import { BrowserWindow, utilityProcess } from "electron"; + import "./worker.js"; + import "./config.mjs"; + import "./legacy.cjs"; + import "./view.js"; + import "./runtime.js"; + const win = new BrowserWindow({webPreferences:{preload:"./worker.js"}}); + win.loadURL("./renderer.js"); + utilityProcess.fork("./worker.js"); + utilityProcess.fork("./runtime.js"); + throw new Error("source must stay inert");`, + "worker.ts": "export const worker = 1;", + "config.mts": "export const config = 1;", + "legacy.cts": "export const legacy = 1;", + "view.tsx": "export const view =
;", + "renderer.ts": "export const renderer = 1;", + "runtime.ts": "export const source = 1;", + "runtime.js": "throw new Error('runtime must stay inert');", + }); + const output = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + const { graph } = result; + const source = findSourceModule(graph, "main.ts"); + for (const [specifier, target] of [ + ["./worker.js", "worker.ts"], + ["./config.mjs", "config.mts"], + ["./legacy.cjs", "legacy.cts"], + ["./view.js", "view.tsx"], + ["./runtime.js", "runtime.js"], + ] as const) + expect( + findGraphEdge(graph, source, findSourceModule(graph, target), "imports") + ?.properties, + ).toMatchObject({ specifier, resolved_path: target }); + expect(result.summary).toMatchObject({ + utility_processes: 2, + resolved_utility_entrypoints: 1, + }); + expect( + graph.edges.find( + (edge) => + edge.evidence.extractor.operation === + "associate-browser-window-preload", + )?.properties, + ).toMatchObject({ + declared_path: "./worker.js", + resolved_path: null, + resolution_status: "not-found", + }); + expect( + graph.edges + .filter( + (edge) => + edge.properties.declared_path === "./worker.js" || + edge.properties.declared_path === "./renderer.js", + ) + .some((edge) => edge.properties.resolved_path != null), + ).toBe(false); + }, +); + cliTest( "returns literal self-import evidence while preserving sibling imports and self-re-exports", async ({ cli }) => { @@ -75,7 +147,7 @@ cliTest( ); cliTest( - "discloses CommonJS and TypeScript self-imports while retaining unresolved extension rewrites", + "discloses CommonJS and TypeScript self-imports including emitted-name rewrites", async ({ cli }) => { const root = await createTestTempDirectory("rea-self-require-"); await writeFixtureFiles(root, { @@ -94,8 +166,10 @@ cliTest( expect( graph.edges.filter((edge) => edge.source_node_id === edge.target_node_id), ).toEqual([]); + // ./util.ts is a literal self-import and ./util.js resolves under NodeNext + // emitted-name semantics back to util.ts itself, as ./c.cjs does for CJS. expect(graph.limitations).toContainEqual( - expect.stringMatching(/^2 import specifiers .*were omitted;/u), + expect.stringMatching(/^3 import specifiers .*were omitted;/u), ); expect(result.limitations).toEqual( expect.arrayContaining(graph.limitations), @@ -128,12 +202,21 @@ cliTest( ).toBe(false); } expect( - graph.edges.find( + graph.edges.some( (edge) => - edge.source_node_id === findSourceModule(graph, "util.ts")?.node_id && edge.properties.specifier === "./util.js" && - edge.properties.module_link_kind === "import", - )?.properties, - ).toMatchObject({ resolved_path: null, resolution_status: "not-found" }); + (edge.properties.module_link_kind === "import" || + edge.properties.resolution_status === "not-found"), + ), + ).toBe(false); + expect( + graph.nodes.some((node) => + node.observations.some( + ({ properties }) => + properties.semantic_role === "module-reference" && + properties.declared_specifier === "./util.js", + ), + ), + ).toBe(false); }, ); diff --git a/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts b/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts deleted file mode 100644 index 4508c05fa6..0000000000 --- a/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { describe, expect } from "vitest"; -import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; -import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; -import { parseEvidence } from "../../../src/domain/evidence.js"; -import { - keyedArchiveIntegerFormats, - keyedArchiveIntegerCases, - keyedArchiveIntegerFixtures, - expectKeyedArchiveIntegerCase, - type KeyedArchiveIntegerFixtures, -} from "../../fixtures/keyedArchiveIntegers.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; -import { - createTestWorkspace, - removeTestWorkspace, -} from "../../support/workspace/workspaceFixture.js"; - -const archiveTest = cliTest.extend<{ - $worker: { keyedArchiveIntegerArchives: KeyedArchiveIntegerFixtures }; -}>({ - keyedArchiveIntegerArchives: [ - // oxlint-disable-next-line no-empty-pattern -- Vitest parses fixture dependencies from object destructuring. - async ({}, use) => { - const workspace = await createTestWorkspace("rea-keyed-integers-"); - try { - await use(await keyedArchiveIntegerFixtures(workspace.root)); - } finally { - await removeTestWorkspace(workspace.root); - } - }, - { scope: "worker" }, - ], -}); - -const cases = keyedArchiveIntegerFormats.flatMap((format) => - keyedArchiveIntegerCases.map((selectedCase) => ({ format, selectedCase })), -); -// Selected by the existing required native keyed-archive verifier, not Linux skips. -describe.skipIf(process.platform !== "darwin")( - "Foundation archive integer precision", - () => { - archiveTest.for(cases)( - "filesystem preserves $format $selectedCase numeric meaning", - async ({ format, selectedCase }, { keyedArchiveIntegerArchives }) => { - const fixture = keyedArchiveIntegerArchives[format]; - const result = await inspectBundleKeyedArchive({ - bundlePath: fixture.root, - targetSha256: fixture.digest, - parameters: { path: fixture.selected.filename, limit: 100 }, - }); - expectKeyedArchiveIntegerCase( - result, - selectedCase, - fixture.selected, - fixture.digest, - ); - }, - ); - archiveTest.for(cases)( - "built CLI preserves $format $selectedCase numeric meaning", - async ( - { format, selectedCase }, - { cli, keyedArchiveIntegerArchives }, - ) => { - const fixture = keyedArchiveIntegerArchives[format]; - const output = await cli.run({ - arguments: ["inspect-keyed-archive", fixture.archivePath, "--json"], - environment: { - REA_LOG_LEVEL: "silent", - REA_ANALYSIS_PROVIDER: "auto", - }, - }); - expect(output.exitCode).toBe(0); - const result = keyedArchiveResultSchema.parse( - parseEvidence(output.json).normalized_result, - ); - expectKeyedArchiveIntegerCase( - result, - selectedCase, - fixture.selected, - fixture.digest, - ); - }, - ); - }, -); diff --git a/tests/boundary/firmware/firmwareIntegration.test.ts b/tests/boundary/firmware/firmwareIntegration.test.ts index 8fee7e83c9..4244212d3c 100644 --- a/tests/boundary/firmware/firmwareIntegration.test.ts +++ b/tests/boundary/firmware/firmwareIntegration.test.ts @@ -308,6 +308,8 @@ it.each(["depth", "dependency", "link"])( expect(normalized.diagnostics).toEqual([ { input_path: "$input", + stage: "gzip", + missing_dependencies: ["sasquatch"], report: { __typename__: "ExtractorDependencyNotFoundReport", severity: "ERROR", diff --git a/tests/boundary/firmware/firmwareReportFiles.test.ts b/tests/boundary/firmware/firmwareReportFiles.test.ts index f3a1816242..c63b81893a 100644 --- a/tests/boundary/firmware/firmwareReportFiles.test.ts +++ b/tests/boundary/firmware/firmwareReportFiles.test.ts @@ -1,8 +1,11 @@ -import { writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; -import { readFirmwareReport } from "../../../src/firmware/FirmwareFiles.js"; +import { + inventoryFirmwareOutput, + readFirmwareReport, +} from "../../../src/firmware/FirmwareFiles.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it.each([ @@ -41,3 +44,24 @@ it("preserves valid Unicode and a literal replacement character in producer JSON report, ); }); + +it("orders extracted firmware paths by code point independently of host locale", async () => { + const root = await createTestTempDirectory("rea-firmware-inventory-"); + await mkdir(join(root, "Lib")); + for (const path of ["a.bin", "z.bin", "\u00e4.bin", "B.bin", "Lib/x.bin"]) + await writeFile(join(root, path), ""); + const entries = await inventoryFirmwareOutput( + root, + 1024, + 16, + "inspect_firmware_regions", + ); + // Locale collation would interleave case and place "ä" by language rules. + expect(entries.map(({ relativePath }) => relativePath)).toEqual([ + "B.bin", + "Lib/x.bin", + "a.bin", + "z.bin", + "\u00e4.bin", + ]); +}); diff --git a/tests/boundary/javascript/packageConditionalExports.test.ts b/tests/boundary/javascript/packageConditionalExports.test.ts index 1eb9927605..64b762072b 100644 --- a/tests/boundary/javascript/packageConditionalExports.test.ts +++ b/tests/boundary/javascript/packageConditionalExports.test.ts @@ -741,7 +741,76 @@ describe("nested and array package export targets", () => { }, ); }); +describe("package export target URL normalization compared with Node", () => { + it.each([ + [["./.\t./outside.cjs", "./array.cjs"]], + [["./.%\n2e/outside.cjs", "./array.cjs"]], + [["./.\r./outside.cjs", "./array.cjs"]], + ["./arr\tay.cjs"], + ["./array.cjs \t"], + ["./array.cjs?selected=one#fragment"], + ["./.\t./example/array.cjs"], + ] as const)( + "resolves normalized target %j through both loaders", + async (target) => { + for (const moduleKind of ["import", "require"] as const) { + const { node, outcome } = await compareWithNode(target, moduleKind); + expect(node).toBe("array.cjs"); + expect(outcome).toMatchObject({ + resolution_status: "resolved", + resolved_path: "node_modules/example/array.cjs", + }); + } + }, + ); + + it.each([ + [["./arr%2fay.cjs", "./array.cjs"], "rejected"], + [["./miss\ting.cjs", "./array.cjs"], "not-found"], + ] as const)( + "retains the first selected target failure in %j", + async (target, status) => { + for (const moduleKind of ["import", "require"] as const) { + const { node, node_error, outcome } = await compareWithNode( + target, + moduleKind, + ); + expect(node).toBeNull(); + expect(node_error).not.toBeNull(); + expect(outcome).toMatchObject({ + resolution_status: status, + resolved_path: null, + }); + } + }, + ); +}); + describe("unmatched package export conditions", () => { + it.each([ + [[{ browser: "./default.cjs" }], "browser"], + [ + [[{ browser: "./default.cjs" }, { "rea-inactive": "./node.cjs" }]], + "browser, rea-inactive", + ], + [{ node: [{ browser: "./default.cjs" }] }, "node, browser"], + ] as const)( + "preserves condition names in unmatched target %j", + async (target, declared) => { + for (const moduleKind of ["import", "require"] as const) { + const { node_error, outcome } = await compareWithNode( + target, + moduleKind, + ); + expect(node_error).toBe("ERR_PACKAGE_PATH_NOT_EXPORTED"); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations[0]).toContain(`It declares ${declared}.`); + } + }, + ); it.each(["import", "require"] as const)( "names the declared conditions for %s", async (moduleKind) => { @@ -769,6 +838,31 @@ describe("unmatched package export conditions", () => { }); }); +describe("blocked package export provenance", () => { + it.each([ + [[], "an empty exports target array"], + [{ node: [] }, "an empty exports target array"], + [[null, []], "an empty exports target array"], + [[[], null], "an explicit null exports target"], + ] as const)( + "preserves the blocking source value in %j", + async (target, limitation) => { + for (const moduleKind of ["import", "require"] as const) { + const { node_error, outcome } = await compareWithNode( + target, + moduleKind, + ); + expect(node_error).toBe("ERR_PACKAGE_PATH_NOT_EXPORTED"); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations[0]).toContain(limitation); + } + }, + ); +}); + describe("package exports fallback refusal reasons", () => { it.each([ [{ import: [42, 7] }, "ERR_INVALID_PACKAGE_TARGET", "rejected"], diff --git a/tests/boundary/javascript/packageImportAliases.test.ts b/tests/boundary/javascript/packageImportAliases.test.ts new file mode 100644 index 0000000000..64da6c235b --- /dev/null +++ b/tests/boundary/javascript/packageImportAliases.test.ts @@ -0,0 +1,786 @@ +import { execFile } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join, relative } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { + resolveArtifactPathByContext, + type ArtifactPathResolution, +} from "../../../src/application/javascript/JavaScriptArtifactPathResolution.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const nodeOutcomeSchema = z.strictObject({ + target: z.string().nullable(), + error_code: z.string().nullable(), +}); + +const posixRelative = (root: string, absolute: string): string => + relative(root, absolute).replaceAll("\\", "/"); + +const inventory = ( + contents: ReadonlyMap, +): Map => + new Map( + [...contents].map(([path, value]) => [ + path, + { + path, + container_sha256: "a".repeat(64), + sha256: "b".repeat(64), + bytes: Buffer.byteLength(value), + inventory_artifact_id: path, + kind: path.endsWith("package.json") ? "package-json" : "javascript", + unpacked: false, + text: { included: true, value }, + }, + ]), + ); + +const writeTree = async ( + root: string, + files: Readonly>, +): Promise> => { + const contents = new Map(); + await Promise.all( + Object.entries(files).map(async ([path, value]) => { + const absolute = join(root, path); + await mkdir(dirname(absolute), { recursive: true }); + await writeFile(absolute, value); + contents.set(path, value); + }), + ); + return contents; +}; + +const nodeRequireTarget = async ( + importer: string, + specifier: string, +): Promise> => { + try { + return { + target: createRequire(importer).resolve(specifier), + error_code: null, + }; + } catch (error) { + const code = + error instanceof Error && "code" in error + ? z.string().parse(Reflect.get(error, "code")) + : null; + return { target: null, error_code: code }; + } +}; + +const nodeImportTarget = async ( + cwd: string, + specifier: string, +): Promise> => { + const { stdout } = await execute( + process.execPath, + [ + "--input-type=module", + "-e", + ` + try { + const url = await import.meta.resolve(${JSON.stringify(specifier)}); + console.log(JSON.stringify({ target: url, error_code: null })); + } catch (error) { + console.log(JSON.stringify({ + target: null, + error_code: error.code ?? null, + })); + } + `, + ], + { cwd }, + ); + return nodeOutcomeSchema.parse(JSON.parse(stdout)); +}; + +const resolveAlias = ( + files: Map, + declaredPath: string, + sourcePath: string, + moduleKind: "import" | "require", +): ArtifactPathResolution => + resolveArtifactPathByContext({ + declaredPath, + sourcePath, + context: "module-specifier", + moduleKind, + files, + }); + +describe("literal package import keys compared with Node", () => { + it("matches literal imports keys containing path-like bytes in both Node loaders", async () => { + const root = await createTestTempDirectory("rea-import-alias-literal-"); + const specifiers = ["#/x", "#x%2Fy", "#x%2Ey", "#x\\y", "#x\0y"]; + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: Object.fromEntries( + specifiers.map((specifier) => [specifier, "./target.cjs"]), + ), + }), + "main.cjs": "module.exports = 1;", + "target.cjs": "module.exports = 1;", + }); + const inventoried = inventory(contents); + for (const specifier of specifiers) { + const required = await nodeRequireTarget( + join(root, "main.cjs"), + specifier, + ); + expect(required.error_code).toBeNull(); + expect(posixRelative(root, required.target ?? "")).toBe("target.cjs"); + const imported = await nodeImportTarget(root, specifier); + expect(imported.error_code).toBeNull(); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + "target.cjs", + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, specifier, "main.cjs", moduleKind), + JSON.stringify({ specifier, moduleKind }), + ).toMatchObject({ + declared_path: specifier, + resolution_status: "resolved", + resolved_path: "target.cjs", + }); + } + }); + + it("still rejects unsafe selected targets for literal imports keys", async () => { + const root = await createTestTempDirectory("rea-import-alias-target-"); + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: { "#x%2Fy": "./nested%2Ftarget.cjs" }, + }), + "main.cjs": "module.exports = 1;", + "nested/target.cjs": "module.exports = 1;", + }); + const inventoried = inventory(contents); + const required = await nodeRequireTarget(join(root, "main.cjs"), "#x%2Fy"); + expect(required.error_code).toBe("ERR_INVALID_MODULE_SPECIFIER"); + const imported = await nodeImportTarget(root, "#x%2Fy"); + expect(imported.error_code).toBe("ERR_INVALID_MODULE_SPECIFIER"); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, "#x%2Fy", "main.cjs", moduleKind), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }); +}); + +describe("inactive package target arrays compared with Node", () => { + it("continues past inactive arrays in enclosing imports and exports conditions", async () => { + const root = await createTestTempDirectory("rea-inactive-package-array-"); + const conditional = { + node: [{ browser: "./browser.cjs" }], + default: "./fallback.cjs", + }; + const nested = { + node: [ + [{ browser: "./browser.cjs" }], + { "rea-inactive": "./inactive.cjs" }, + ], + default: "./fallback.cjs", + }; + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: { "#conditional": conditional, "#nested": nested }, + }), + "main.cjs": "module.exports = 1;", + "fallback.cjs": "module.exports = 1;", + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + exports: { ".": conditional, "./nested": nested }, + }), + "node_modules/fixture/fallback.cjs": "module.exports = 2;", + }); + const inventoried = inventory(contents); + for (const [specifier, expected] of [ + ["#conditional", "fallback.cjs"], + ["#nested", "fallback.cjs"], + ["fixture", "node_modules/fixture/fallback.cjs"], + ["fixture/nested", "node_modules/fixture/fallback.cjs"], + ] as const) { + const required = await nodeRequireTarget( + join(root, "main.cjs"), + specifier, + ); + expect(required.error_code, specifier).toBeNull(); + expect(posixRelative(root, required.target ?? "")).toBe(expected); + const imported = await nodeImportTarget(root, specifier); + expect(imported.error_code).toBeNull(); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + expected, + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, specifier, "main.cjs", moduleKind), + ).toMatchObject({ + declared_path: specifier, + resolution_status: "resolved", + resolved_path: expected, + }); + } + }); +}); + +describe("package target refusal evidence compared with Node", () => { + it("keeps empty, null, invalid, and unmatched array outcomes distinct", async () => { + const root = await createTestTempDirectory("rea-inactive-package-refusal-"); + const inactive = { browser: "./browser.cjs" }; + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: { + "#empty-direct": [], + "#empty": { node: [], default: "./fallback.cjs" }, + "#null": { node: [null, inactive], default: "./fallback.cjs" }, + "#null-last": [[], null], + "#empty-last": [null, []], + "#invalid": { + node: ["../outside.cjs", inactive], + default: "./fallback.cjs", + }, + "#unmatched": [inactive], + "#unmatched-nested": [ + [inactive, { "rea-inactive": "./inactive.cjs" }], + ], + "#unmatched-active": { node: [inactive] }, + }, + }), + "main.cjs": "module.exports = 1;", + "fallback.cjs": "module.exports = 1;", + }); + const inventoried = inventory(contents); + for (const [specifier, errorCode, status, limitation] of [ + [ + "#empty-direct", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "an empty imports target array", + ], + [ + "#empty", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "an empty imports target array", + ], + [ + "#null", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "an explicit null imports target", + ], + [ + "#null-last", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "an explicit null imports target", + ], + [ + "#empty-last", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "an empty imports target array", + ], + ["#invalid", "ERR_INVALID_PACKAGE_TARGET", "rejected", "../outside.cjs"], + [ + "#unmatched", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "it declares browser.", + ], + [ + "#unmatched-nested", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "it declares browser, rea-inactive.", + ], + [ + "#unmatched-active", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "it declares node, browser.", + ], + ] as const) { + expect( + (await nodeRequireTarget(join(root, "main.cjs"), specifier)).error_code, + ).toBe(errorCode); + expect((await nodeImportTarget(root, specifier)).error_code).toBe( + errorCode, + ); + for (const moduleKind of ["require", "import"] as const) { + const result = resolveAlias( + inventoried, + specifier, + "main.cjs", + moduleKind, + ); + expect(result).toMatchObject({ + resolution_status: status, + resolved_path: null, + }); + expect(result.limitations.join(" "), specifier).toContain(limitation); + } + } + }); +}); + +describe("package import target arrays compared with Node", () => { + it("skips invalid parent, absolute, and URL targets before an exact fallback", async () => { + const root = await createTestTempDirectory("rea-import-alias-fallback-"); + const imports = { + "#parent": ["../outside.cjs", "./fallback.cjs"], + "#absolute": { node: [["/outside.cjs"], "./fallback.cjs"] }, + "#url": { + import: ["https://example.invalid/outside.cjs", "./fallback.cjs"], + require: ["file:///outside.cjs", "./fallback.cjs"], + }, + }; + const contents = await writeTree(root, { + "package.json": JSON.stringify({ imports }), + "main.cjs": "module.exports = 1;", + "fallback.cjs": "module.exports = 1;", + }); + const inventoried = inventory(contents); + for (const specifier of Object.keys(imports)) { + const required = await nodeRequireTarget( + join(root, "main.cjs"), + specifier, + ); + expect(required.error_code).toBeNull(); + expect(posixRelative(root, required.target ?? "")).toBe("fallback.cjs"); + const imported = await nodeImportTarget(root, specifier); + expect(imported.error_code).toBeNull(); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + "fallback.cjs", + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, specifier, "main.cjs", moduleKind), + ).toMatchObject({ + declared_path: specifier, + resolution_status: "resolved", + resolved_path: "fallback.cjs", + }); + } + }); + + it("preserves invalid, null, and unsupported-package outcomes in arrays", async () => { + const root = await createTestTempDirectory( + "rea-import-alias-fallback-refuse-", + ); + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: { + "#invalid": [null, "../outside.cjs"], + "#blocked": ["../outside.cjs", null], + "#package": ["fixture", "./fallback.cjs"], + }, + }), + "main.cjs": "module.exports = 1;", + "fallback.cjs": "module.exports = 1;", + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + main: "index.cjs", + }), + "node_modules/fixture/index.cjs": "module.exports = 2;", + }); + const inventoried = inventory(contents); + for (const [specifier, errorCode, resolutionStatus, reason] of [ + ["#invalid", "ERR_INVALID_PACKAGE_TARGET", "rejected", "../outside.cjs"], + [ + "#blocked", + "ERR_PACKAGE_IMPORT_NOT_DEFINED", + "external", + "explicit null", + ], + ] as const) { + const required = await nodeRequireTarget( + join(root, "main.cjs"), + specifier, + ); + expect(required.error_code).toBe(errorCode); + expect((await nodeImportTarget(root, specifier)).error_code).toBe( + errorCode, + ); + for (const moduleKind of ["require", "import"] as const) { + const result = resolveAlias( + inventoried, + specifier, + "main.cjs", + moduleKind, + ); + expect(result).toMatchObject({ + resolution_status: resolutionStatus, + resolved_path: null, + }); + expect(result.limitations.join(" ")).toContain(reason); + } + } + const required = await nodeRequireTarget( + join(root, "main.cjs"), + "#package", + ); + expect(posixRelative(root, required.target ?? "")).toBe( + "node_modules/fixture/index.cjs", + ); + const imported = await nodeImportTarget(root, "#package"); + expect(imported.error_code).toBeNull(); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + "node_modules/fixture/index.cjs", + ); + for (const moduleKind of ["require", "import"] as const) { + const result = resolveAlias( + inventoried, + "#package", + "main.cjs", + moduleKind, + ); + expect(result).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(result.limitations.join(" ")).toContain( + "External-package import targets remain unresolved", + ); + } + }); +}); + +describe("package import target URL normalization compared with Node", () => { + it("normalizes targets before package-root validation and array selection", async () => { + const root = await createTestTempDirectory("rea-package-target-url-"); + const directory = "app space#?%λ"; + const imports = { + "#tab-escape": ["./.\t./outside.cjs", "./fallback.cjs"], + "#newline-escape": ["./.%\n2e/outside.cjs", "./fallback.cjs"], + "#cr-escape": ["./.\r./outside.cjs", "./fallback.cjs"], + "#normalized": "./fall\tback.cjs", + "#trimmed": "./fallback.cjs \t", + "#suffix": "./fallback.cjs?selected=one#fragment", + "#return": `./.\t./${encodeURIComponent(directory)}/fallback.cjs`, + }; + const contents = await writeTree(root, { + [`${directory}/package.json`]: JSON.stringify({ imports }), + [`${directory}/main.cjs`]: "module.exports = 1;", + [`${directory}/fallback.cjs`]: "module.exports = 1;", + }); + const inventoried = inventory(contents); + for (const specifier of Object.keys(imports)) { + const required = await nodeRequireTarget( + join(root, directory, "main.cjs"), + specifier, + ); + expect(required.error_code, specifier).toBeNull(); + expect(posixRelative(root, required.target ?? "")).toBe( + `${directory}/fallback.cjs`, + ); + const imported = await nodeImportTarget(join(root, directory), specifier); + expect(imported.error_code, specifier).toBeNull(); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + `${directory}/fallback.cjs`, + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias( + inventoried, + specifier, + `${directory}/main.cjs`, + moduleKind, + ), + specifier, + ).toMatchObject({ + declared_path: specifier, + resolution_status: "resolved", + resolved_path: `${directory}/fallback.cjs`, + }); + } + }); + + it("keeps URL decoding and missing-file failures after array selection", async () => { + const root = await createTestTempDirectory( + "rea-package-target-url-refusal-", + ); + const contents = await writeTree(root, { + "package.json": JSON.stringify({ + imports: { + "#escape": "./.\t./outside.cjs", + "#separator": ["./fall%2fback.cjs", "./fallback.cjs"], + "#missing": ["./miss\ting.cjs", "./fallback.cjs"], + "#query-separator": ["./first.cjs?selected=%2f", "./fallback.cjs"], + }, + }), + "main.cjs": "module.exports = 1;", + "fallback.cjs": "module.exports = 1;", + "first.cjs": "module.exports = 2;", + }); + const inventoried = inventory(contents); + for (const [specifier, code] of [ + ["#escape", "ERR_INVALID_PACKAGE_TARGET"], + ["#separator", "ERR_INVALID_MODULE_SPECIFIER"], + ] as const) { + expect( + (await nodeRequireTarget(join(root, "main.cjs"), specifier)).error_code, + ).toBe(code); + expect((await nodeImportTarget(root, specifier)).error_code).toBe(code); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, specifier, "main.cjs", moduleKind), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + } + expect( + (await nodeRequireTarget(join(root, "main.cjs"), "#missing")).error_code, + ).toBe("MODULE_NOT_FOUND"); + const imported = await nodeImportTarget(root, "#missing"); + expect(posixRelative(root, fileURLToPath(imported.target ?? ""))).toBe( + "missing.cjs", + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, "#missing", "main.cjs", moduleKind), + ).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + limitations: [expect.stringContaining("missing.cjs")], + }); + expect( + resolveAlias( + inventoried, + "#escape", + "main.cjs", + "require", + ).limitations.join(" "), + ).toContain(JSON.stringify("./.\t./outside.cjs")); + expect( + (await nodeRequireTarget(join(root, "main.cjs"), "#query-separator")) + .error_code, + ).toBe("ERR_INVALID_MODULE_SPECIFIER"); + expect( + resolveAlias(inventoried, "#query-separator", "main.cjs", "require"), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + const withQuery = await nodeImportTarget(root, "#query-separator"); + expect(posixRelative(root, fileURLToPath(withQuery.target ?? ""))).toBe( + "first.cjs", + ); + expect( + resolveAlias(inventoried, "#query-separator", "main.cjs", "import"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "first.cjs", + }); + expect( + resolveAlias( + inventoried, + "#missing", + "main.cjs", + "require", + ).limitations.join(" "), + ).toContain(JSON.stringify("./miss\ting.cjs")); + }); +}); + +describe("package import aliases compared with Node", () => { + it("matches Node require and import for an exact root alias", async () => { + const root = await createTestTempDirectory("rea-import-alias-root-"); + const files = { + "package.json": JSON.stringify({ + name: "app", + imports: { "#config": "./config.cjs" }, + }), + "main.cjs": 'module.exports = require("#config");', + "config.cjs": "module.exports = 'config.cjs';", + }; + const contents = await writeTree(root, files); + const inventoried = inventory(contents); + const required = await nodeRequireTarget(join(root, "main.cjs"), "#config"); + expect(posixRelative(root, required.target ?? "")).toBe("config.cjs"); + expect( + resolveAlias(inventoried, "#config", "main.cjs", "require"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + }); + const imported = await nodeImportTarget(root, "#config"); + expect(imported.error_code).toBeNull(); + expect( + resolveAlias(inventoried, "#config", "main.cjs", "import"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + }); + }); + + it("matches Node when a nested package has its own imports map", async () => { + const root = await createTestTempDirectory("rea-import-alias-nested-"); + const files = { + "package.json": JSON.stringify({ + name: "app", + imports: { "#config": "./root.cjs" }, + }), + "root.cjs": "module.exports = 'root';", + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + imports: { "#config": "./local.cjs" }, + }), + "node_modules/fixture/index.cjs": 'module.exports = require("#config");', + "node_modules/fixture/local.cjs": "module.exports = 'local';", + }; + const contents = await writeTree(root, files); + const inventoried = inventory(contents); + const nested = await nodeRequireTarget( + join(root, "node_modules/fixture/index.cjs"), + "#config", + ); + expect(posixRelative(root, nested.target ?? "")).toBe( + "node_modules/fixture/local.cjs", + ); + expect( + resolveAlias( + inventoried, + "#config", + "node_modules/fixture/index.cjs", + "require", + ), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "node_modules/fixture/local.cjs", + }); + const fromRoot = await nodeRequireTarget(join(root, "root.cjs"), "#config"); + expect(posixRelative(root, fromRoot.target ?? "")).toBe("root.cjs"); + expect( + resolveAlias(inventoried, "#config", "root.cjs", "require"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "root.cjs", + }); + }); + + it("matches Node when a nested package does not define the alias", async () => { + const root = await createTestTempDirectory("rea-import-alias-scope-"); + const files = { + "package.json": JSON.stringify({ + name: "app", + imports: { "#config": "./root.cjs" }, + }), + "root.cjs": "module.exports = 'root';", + "node_modules/fixture/package.json": JSON.stringify({ + name: "fixture", + }), + "node_modules/fixture/index.cjs": 'module.exports = require("#config");', + }; + await writeTree(root, files); + const nested = await nodeRequireTarget( + join(root, "node_modules/fixture/index.cjs"), + "#config", + ); + expect(nested.target).toBeNull(); + // CJS reports MODULE_NOT_FOUND when the nested package.json has no + // `imports` field at all; it still does not inherit the ancestor map. + expect(nested.error_code).toBe("MODULE_NOT_FOUND"); + const outcome = resolveAlias( + inventory(new Map(Object.entries(files))), + "#config", + "node_modules/fixture/index.cjs", + "require", + ); + expect(outcome).toMatchObject({ + resolution_status: "external", + resolved_path: null, + }); + expect(outcome.limitations.join(" ")).toContain( + "node_modules/fixture/package.json", + ); + }); +}); + +describe("package import alias refusals compared with Node", () => { + it("matches Node refusal for a missing key, explicit null, and reserved specifier", async () => { + const root = await createTestTempDirectory("rea-import-alias-refuse-"); + const files = { + "package.json": JSON.stringify({ + name: "app", + imports: { + "#config": "./config.cjs", + "#blocked": null, + "#": "./config.cjs", + "#/": "./config.cjs", + "#config/": "./config.cjs", + }, + }), + "main.cjs": "module.exports = 1;", + "config.cjs": "module.exports = 1;", + }; + await writeTree(root, files); + const inventoried = inventory(new Map(Object.entries(files))); + const missing = await nodeRequireTarget(join(root, "main.cjs"), "#missing"); + expect(missing.error_code).toBe("ERR_PACKAGE_IMPORT_NOT_DEFINED"); + expect( + resolveAlias(inventoried, "#missing", "main.cjs", "require"), + ).toMatchObject({ resolution_status: "external", resolved_path: null }); + const blocked = await nodeRequireTarget(join(root, "main.cjs"), "#blocked"); + expect(blocked.error_code).toBe("ERR_PACKAGE_IMPORT_NOT_DEFINED"); + expect( + resolveAlias(inventoried, "#blocked", "main.cjs", "require"), + ).toMatchObject({ resolution_status: "external", resolved_path: null }); + for (const specifier of ["#", "#/", "#config/"]) { + const reserved = await nodeRequireTarget( + join(root, "main.cjs"), + specifier, + ); + expect(reserved.error_code).toBe("ERR_INVALID_MODULE_SPECIFIER"); + expect((await nodeImportTarget(root, specifier)).error_code).toBe( + "ERR_INVALID_MODULE_SPECIFIER", + ); + for (const moduleKind of ["require", "import"] as const) + expect( + resolveAlias(inventoried, specifier, "main.cjs", moduleKind), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + } + }); + + it("matches Node when an import condition selects a different exact file", async () => { + const root = await createTestTempDirectory("rea-import-alias-cond-"); + const files = { + "package.json": JSON.stringify({ + name: "app", + type: "module", + imports: { + "#config": { + import: "./config.mjs", + require: "./config.cjs", + }, + }, + }), + "main.cjs": 'module.exports = require("#config");', + "config.cjs": "export default 'cjs'; module.exports = 'cjs';", + "config.mjs": "export default 'mjs';", + }; + await writeTree(root, files); + const inventoried = inventory(new Map(Object.entries(files))); + const required = await nodeRequireTarget(join(root, "main.cjs"), "#config"); + expect(posixRelative(root, required.target ?? "")).toBe("config.cjs"); + expect( + resolveAlias(inventoried, "#config", "main.cjs", "require"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.cjs", + }); + const imported = await nodeImportTarget(root, "#config"); + expect(imported.error_code).toBeNull(); + expect(imported.target ?? "").toContain("config.mjs"); + expect( + resolveAlias(inventoried, "#config", "main.cjs", "import"), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "config.mjs", + }); + }); +}); diff --git a/tests/boundary/mcp/toolResult.test.ts b/tests/boundary/mcp/toolResult.test.ts index a71ac46f63..c82edd0797 100644 --- a/tests/boundary/mcp/toolResult.test.ts +++ b/tests/boundary/mcp/toolResult.test.ts @@ -50,6 +50,8 @@ const contract: ToolContract = { accessesNetwork: false, changesUiState: false, mayDiscardData: false, + executesForeignCode: false, + mutatesDevice: false, idempotent: true, }, annotations: { @@ -336,20 +338,22 @@ describe("tool result projection", () => { const result = delivery.toCallToolResult( err( new HopperProcessError(80, { - component: "hopper_private_display", - operation: "launch", - status: "error", - failure_code: "x11_socket_directory_unusable", - reason: "socket_directory_read_only", - socket_directory: "/tmp/.X11-unix", - socket_directory_mode: "0777", - mount_read_only: true, - effective_socket_directory_mode: "1777", - effective_mount_read_only: false, - wsl: true, - strategy: "user-mount-namespace", - fallback_reason: null, - xvfb_stderr_bytes: 512, + diagnostic: { + component: "hopper_private_display", + operation: "launch", + status: "error", + failure_code: "x11_socket_directory_unusable", + reason: "socket_directory_read_only", + socket_directory: "/tmp/.X11-unix", + socket_directory_mode: "0777", + mount_read_only: true, + effective_socket_directory_mode: "1777", + effective_mount_read_only: false, + wsl: true, + strategy: "user-mount-namespace", + fallback_reason: null, + xvfb_stderr_bytes: 512, + }, }), ), contract, diff --git a/tests/boundary/mcp/toolSchemaValidity.test.ts b/tests/boundary/mcp/toolSchemaValidity.test.ts index 6855f81885..adb5f83e99 100644 --- a/tests/boundary/mcp/toolSchemaValidity.test.ts +++ b/tests/boundary/mcp/toolSchemaValidity.test.ts @@ -73,7 +73,7 @@ function expectKnownAuthorityHints(tools: readonly ToolSchemas[]): void { }, annotate_native_function: { readOnlyHint: false, - destructiveHint: false, + destructiveHint: true, idempotentHint: true, openWorldHint: true, }, @@ -102,8 +102,22 @@ function expectKnownAuthorityHints(tools: readonly ToolSchemas[]): void { openWorldHint: false, }, capture_process_scenario: { + readOnlyHint: false, + destructiveHint: true, openWorldHint: true, }, + observe_native_calls: { + readOnlyHint: false, + destructiveHint: true, + }, + uninstall_adb_package: { + readOnlyHint: false, + destructiveHint: true, + }, + start_adb_app: { + readOnlyHint: false, + destructiveHint: true, + }, export_evidence_bundle: { readOnlyHint: false, destructiveHint: true, diff --git a/tests/boundary/process/androidSearchEvidenceBudget.test.ts b/tests/boundary/process/androidSearchEvidenceBudget.test.ts new file mode 100644 index 0000000000..143c72189d --- /dev/null +++ b/tests/boundary/process/androidSearchEvidenceBudget.test.ts @@ -0,0 +1,131 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it as test, onTestFinished } from "vitest"; +import { androidResultSchemas } from "../../../src/domain/android/androidAnalysis.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + createJadxProtocolFixture, + verifyJadxFixtureCleanup, +} from "../../fixtures/android/jadx.js"; + +const INVENTORY_CLASS_COUNT = 7096; + +/** Count inventoried names across raw call records: MCP pages ride in text content. */ +const rawInventoryNames = (raw: unknown): number => { + const calls = + ( + raw as { + calls?: { + operation: string; + response: { content?: { text?: string }[] }; + }[]; + } + ).calls ?? []; + return calls + .flatMap((call) => call.response.content ?? []) + .map((entry) => + entry.text === undefined ? {} : (JSON.parse(entry.text) as unknown), + ) + .filter((page) => Array.isArray((page as { items?: unknown[] }).items)) + .flatMap((page) => (page as { items: unknown[] }).items).length; +}; + +test("delivers bounded search_android_classes Evidence while retaining the complete inventory", async () => { + const { provider, apk, launches } = await createJadxProtocolFixture( + "normal", + undefined, + { classCount: INVENTORY_CLASS_COUNT - 1 }, + ); + const session = createTestBinarySession(() => { + throw new Error("APK analysis must not acquire a deep binary provider"); + }); + const server = createServer( + { kind: "session", session }, + { androidAnalysis: provider }, + ); + const client = new Client({ + name: "android-search-inventory-budget", + version: "1", + }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + + const search = async (query: string) => + parseEvidence( + ( + await client.callTool({ + name: "search_android_classes", + arguments: { path: apk, query }, + }) + ).structuredContent, + ); + + const single = await search("Target"); + expect( + androidResultSchemas.search_android_classes.parse(single.normalized_result), + ).toMatchObject({ + query: "Target", + total_classes: INVENTORY_CLASS_COUNT, + matches: ["fixture.Target"], + coverage: "complete", + }); + // The inline raw record keeps the pagination audit, not the class names. + const singleRaw = JSON.stringify(single.raw_result); + expect(singleRaw).not.toContain("fixture.Class"); + expect(singleRaw).toContain("items_omitted"); + expect(Buffer.byteLength(JSON.stringify(single))).toBeLessThan(32 * 1024); + // The complete inventory stays locally retrievable through the retained record. + const completeId = single.evidence_links[0]; + expect(completeId).toBeDefined(); + expect(completeId).toBe( + (single.raw_result as { complete_evidence_id: string }) + .complete_evidence_id, + ); + const complete = session.evidenceById(completeId as string); + expect(complete).toBeDefined(); + expect(complete?.operation).toBe("search_android_classes"); + expect(rawInventoryNames(complete?.raw_result)).toBe(INVENTORY_CLASS_COUNT); + expect(JSON.stringify(complete?.raw_result)).toContain("fixture.Class7094"); + + const empty = await search("fixture.Missing"); + expect( + androidResultSchemas.search_android_classes.parse(empty.normalized_result), + ).toMatchObject({ + query: "fixture.Missing", + total_classes: INVENTORY_CLASS_COUNT, + matches: [], + coverage: "complete", + }); + expect(JSON.stringify(empty.raw_result)).not.toContain("fixture.Class"); + + await client.close(); + await server.close(); + await verifyJadxFixtureCleanup(launches); +}); + +test("keeps the complete search inventory inline without a retaining session", async () => { + const { service, apk, launches, provider } = await createJadxProtocolFixture( + "normal", + undefined, + { classCount: INVENTORY_CLASS_COUNT - 1 }, + ); + const result = await service.execute("search_android_classes", { + path: apk, + query: "Target", + }); + if (!result.ok) throw result.error; + // Complete CLI JSON output keeps the full raw provider inventory. + expect(rawInventoryNames(result.value.raw_result)).toBe( + INVENTORY_CLASS_COUNT, + ); + await provider.close(); + await verifyJadxFixtureCleanup(launches); +}); diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index 05ccdb81cd..34205af3d0 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -8,7 +8,10 @@ import { runOwnedCommand, } from "../../../src/process/OwnedCommand.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; -import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type ProcessCleanupResult, +} from "../../../src/process/ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "../../../src/process/ProcessOwnershipObservation.js"; import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; @@ -186,6 +189,99 @@ it("returns a timeout for a running process after independent cleanup", async () ).rejects.toMatchObject({ reason: "timeout", cleanupFailure: null }); }); +it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)( + "retains the real per-member cleanup ledger after %s and settles after cooperative exit", + async (reason) => { + const childScript = + 'process.stdin.once("data", () => process.exit(0)); process.stdout.write("child-ready\\n")'; + const request = { + ...command(` + const { spawn } = require("node:child_process"); + const child = spawn(process.execPath, ["-e", ${JSON.stringify(childScript)}], { + env: { ...process.env, REA_PROCESS_RUN_ID: "different-child-owner" }, + stdio: ["pipe", "inherit", "inherit"], + }); + const release = () => child.stdin.end("release"); + process.stdin.once("data", release); + const failsafe = setTimeout(release, 10000); + child.once("exit", () => { clearTimeout(failsafe); process.stdin.destroy(); }); + `), + stdin: "pipe" as const, + }; + const controller = new AbortController(); + let launched: + | Awaited> + | undefined; + let cleanup: ProcessCleanupResult | undefined; + let failure: OwnedCommandFailure | undefined; + try { + await runOwnedCommand( + request, + { timeoutMs: 2000, diagnosticBytes: 1024 }, + { + signal: controller.signal, + launcher: async (input) => { + const acquired = await spawnOwnedProviderProcess(input); + launched = acquired; + acquired.process.stdout?.on("data", (chunk: Buffer) => { + if ( + reason === "cancelled" && + chunk.toString().includes("child-ready") + ) + controller.abort(); + }); + return { + ...acquired, + cleanup: async () => { + const observed = await cleanupOwnedProcessGroup( + acquired.ownership, + ); + cleanup ??= observed; + return observed; + }, + }; + }, + }, + ).catch((cause: unknown) => { + if (!(cause instanceof OwnedCommandFailure)) throw cause; + failure = cause; + }); + if (failure === undefined || launched === undefined) + throw new Error( + "The acquired command must retain its failed cleanup owner", + ); + expect(failure.reason).toBe(reason); + expect(failure.snapshot?.stdout.text).toContain("child-ready"); + expect(cleanup).toMatchObject({ + cleaned: false, + failures: [expect.objectContaining({ reason: "run-token-mismatch" })], + }); + expect(failure.cleanupResult).toEqual(cleanup); + expect(failure.resources).toEqual( + expect.arrayContaining([request.runId]), + ); + // An unresolved member does not grant authority to signal the shared group. + const acquiredPid = launched.ownership.leaderPid; + expect(() => process.kill(acquiredPid, 0)).not.toThrow(); + if (cleanup?.cleaned === false) { + for (const member of cleanup.failures ?? []) { + expect(() => process.kill(member.pid, 0)).not.toThrow(); + expect(failure.cleanupFailure).toContain(String(member.pid)); + } + } + } finally { + // Release through the fixture's acquired input channel, never by an unverified PID. + launched?.process.stdin?.end("release"); + if (failure?.cleanupOwner !== undefined) { + expect(await failure.cleanupOwner.waitForOutputClose(5000)).toBe(true); + expect(await failure.cleanupOwner.stop()).toEqual({ + status: "verified-cleanup", + }); + } + } + }, +); + it("honors cancellation during cleanup after a successful command", async () => { const controller = new AbortController(); await expect( diff --git a/tests/boundary/process/providerProcess.test.ts b/tests/boundary/process/providerProcess.test.ts index 3874be079f..b2813ce468 100644 --- a/tests/boundary/process/providerProcess.test.ts +++ b/tests/boundary/process/providerProcess.test.ts @@ -358,6 +358,57 @@ describe("provider process output and cleanup primitives", () => { }); }); +it("retains member failures and sweep uncertainty from a failed cleanup recheck", async () => { + const child = spawnProviderProcessFixture("stubborn"); + await waitForProviderProcessReady(child); + const pid = child.pid; + if (pid === undefined) { + await stopProviderProcessFixture(child); + throw new Error("The acquired ready fixture must have a process ID"); + } + const failed = { + cleaned: false as const, + reason: "fixture identity unavailable during recheck", + failures: [ + { + pid, + reason: "process-identity-unavailable" as const, + diagnostic: "fixture identity reader unavailable", + }, + ], + unverified: [ + { pid: 777, diagnostic: "fixture foreign process token unavailable" }, + ], + }; + let attempts = 0; + let release = false; + const supervisor = new ProviderProcessSupervisor({ + process: child, + ownsProcessLifetime: true, + cleanup: async () => { + if (release) child.kill("SIGKILL"); + attempts += 1; + return release || attempts === 1 + ? { cleaned: true as const, signaled: release } + : failed; + }, + }); + try { + await expect(supervisor.stop({ killGraceMs: 20 })).resolves.toEqual({ + status: "incomplete", + reason: failed.reason, + cleanupResult: failed, + }); + expect(child.exitCode).toBeNull(); + release = true; + await expect(supervisor.stop()).resolves.toEqual({ + status: "verified-cleanup", + }); + } finally { + await stopProviderProcessFixture(child); + } +}); + describe("provider process host injection", () => { it("uses injected host platform and environment when spawning", async () => { const spawned = await spawnOwnedProviderProcess({ diff --git a/tests/boundary/providers/hopper/hopperClient.cleanup.test.ts b/tests/boundary/providers/hopper/hopperClient.cleanup.test.ts index abfce5834a..cc4a8d1ff6 100644 --- a/tests/boundary/providers/hopper/hopperClient.cleanup.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.cleanup.test.ts @@ -3,7 +3,6 @@ import { access } from "node:fs/promises"; import { afterEach, describe, expect, it } from "vitest"; -import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { ok } from "../../../../src/domain/result.js"; import type { BridgeLauncher, @@ -53,31 +52,6 @@ class OwnedFixtureLauncher implements BridgeLauncher { } } -class UnconfirmedFixtureLauncher implements BridgeLauncher { - constructor(readonly ownsProcessLifetime = false) {} - - launch(session: BridgeSession) { - return Promise.resolve( - ok({ - process: spawn( - process.execPath, - [ - fixturePath, - session.socketPath, - session.token, - session.runId, - "unconfirmed", - ], - { stdio: ["ignore", "ignore", "pipe"] }, - ), - ownsProcessLifetime: this.ownsProcessLifetime, - providerLifetime: "external-application" as const, - shutdownMode: "bridge-request" as const, - }), - ); - } -} - const clients: HopperClient[] = []; afterEach(async () => { @@ -137,42 +111,6 @@ describe("HopperClient cleanup", () => { }); }); - it.each([ - ["unowned", false], - ["owned helper", true], - ] as const)( - "reports unconfirmed external document cleanup for an %s process", - async (_description, ownsProcessLifetime) => { - const client = new HopperClient({ - launcher: new UnconfirmedFixtureLauncher(ownsProcessLifetime), - startupTimeoutMs: 1_000, - }); - clients.push(client); - expect((await client.start()).ok).toBe(true); - - const closed = await client.close(); - - expect(closed).toMatchObject({ - ok: false, - error: { - _tag: "ProviderAdapterError", - cleanupIncomplete: true, - cleanupResources: expect.arrayContaining(["hopper-document"]), - }, - }); - if (!closed.ok) - expect(projectAnalysisError(closed.error)).toMatchObject({ - code: "cleanup_incomplete", - details: { - provider_id: "hopper", - operation: "close_binary", - cleanup: "incomplete", - resources: expect.arrayContaining(["hopper-document"]), - }, - }); - }, - ); - it("kills only its owned Hopper group and emits sanitized shutdown coordinates", async () => { const unrelated = spawn( process.execPath, diff --git a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts index eac881e588..e6f8215a03 100644 --- a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts @@ -160,50 +160,71 @@ it("retains failed startup cleanup until an owned process and its backing image }); }); -it("keeps an unconfirmed external document visible across sequential closes", async () => { - const launcher = new RetryCleanupLauncher(true); - const runtimeRoot = await PrivateRuntimeRoot.create(); - onTestFinished(() => runtimeRoot.close()); - const launched = await launcher.launch({ - directory: runtimeRoot.path, - socketPath: join(runtimeRoot.path, "bridge.sock"), - token: "fixture-token", - runId: "fixture-run", - }); - if (!launched.ok) throw launched.error; - const child = launched.value.process; - const supervisor = new ProviderProcessSupervisor(launched.value); - onTestFinished(() => supervisor.dispose()); - const resources: HopperOwnedResources = { - launch: launched.value, - processSupervisor: supervisor, - runtimeRoot, - shutdownConfirmed: false, - }; - for (let attempt = 0; attempt < 2; attempt += 1) { - await expect( - cleanupHopperSession({ - socket: undefined, - resources, - activeRequest: null, - retainDocument: false, - progress: undefined, - logger: silentLogger, - onDiagnostic: undefined, - request: () => Promise.resolve(ok(null)), - releaseTransport: (socket) => socket?.destroy(), - }), - ).resolves.toMatchObject({ - ok: false, - error: { cleanupResources: ["hopper-document", runtimeRoot.path] }, +it.each([false, true])( + "keeps an unconfirmed external document across closes when helper ownership is %s", + async (ownsHelper) => { + const launcher = new RetryCleanupLauncher(true); + const runtimeRoot = await PrivateRuntimeRoot.create(); + onTestFinished(() => runtimeRoot.close()); + const launched = await launcher.launch({ + directory: runtimeRoot.path, + socketPath: join(runtimeRoot.path, "bridge.sock"), + token: "fixture-token", + runId: "fixture-run", }); - } - expect(child.exitCode).toBeNull(); - expect(child.signalCode).toBeNull(); - await expect( - access(join(runtimeRoot.path, "image.macho")), - ).resolves.toBeUndefined(); -}); + if (!launched.ok) throw launched.error; + const child = launched.value.process; + const launch: BridgeLaunch = ownsHelper + ? { ...launched.value, ownsProcessLifetime: true } + : launched.value; + const supervisor = new ProviderProcessSupervisor(launch); + onTestFinished(() => supervisor.dispose()); + const resources: HopperOwnedResources = { + launch, + processSupervisor: supervisor, + runtimeRoot, + shutdownConfirmed: false, + }; + const socket = new Socket(); + onTestFinished(() => { + socket.destroy(); + }); + for (let attempt = 0; attempt < 2; attempt += 1) { + await expect( + cleanupHopperSession({ + socket: attempt === 0 ? socket : undefined, + resources, + activeRequest: null, + retainDocument: false, + progress: undefined, + logger: silentLogger, + onDiagnostic: undefined, + request: () => + Promise.resolve( + ok({ + shutdown: true, + analysis_stopped: true, + document_closed: false, + }), + ), + releaseTransport: (socket) => socket?.destroy(), + }), + ).resolves.toMatchObject({ + ok: false, + error: { cleanupResources: ["hopper-document", runtimeRoot.path] }, + }); + } + if (ownsHelper) + expect(child.exitCode !== null || child.signalCode !== null).toBe(true); + else { + expect(child.exitCode).toBeNull(); + expect(child.signalCode).toBeNull(); + } + await expect( + access(join(runtimeRoot.path, "image.macho")), + ).resolves.toBeUndefined(); + }, +); it("blocks a fresh launch while cleanup remains unverified", async () => { const launcher = new RetryCleanupLauncher(); diff --git a/tests/boundary/providers/hopper/hopperClient.health.test.ts b/tests/boundary/providers/hopper/hopperClient.health.test.ts index 76b9be52f5..66748051ef 100644 --- a/tests/boundary/providers/hopper/hopperClient.health.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.health.test.ts @@ -1,4 +1,5 @@ import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; import { describe, expect, it, onTestFinished, vi } from "vitest"; @@ -12,15 +13,18 @@ import { HopperClient } from "../../../../src/hopper/HopperClient.js"; import { HopperFixtureLauncher } from "./hopperClient.fixture.js"; class ObservedProcess extends EventEmitter { - readonly stdout = null; - readonly stderr = null; + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); exitCode: number | null = null; - readonly signalCode = null; + signalCode: NodeJS.Signals | null = null; - exit(code: number): void { + exit(code: number | null, signal: NodeJS.Signals | null = null): void { this.exitCode = code; - this.emit("exit", code, null); - this.emit("close", code, null); + this.signalCode = signal; + this.emit("exit", code, signal); + this.stdout.end(); + this.stderr.end(); + this.emit("close", code, signal); } kill(): boolean { @@ -66,7 +70,7 @@ describe("Hopper provider lifecycle health", () => { launcher.observed.exit(0); const pending = client.callTool("hang"); await launcher.fixture.waitForRequest("hang"); - launcher.fixture.processes.at(-1)?.kill("SIGKILL"); + launcher.fixture.processes.at(-1)?.send({ disconnect: true }); await vi.waitFor(() => expect(client.operationHealth().state).toBe("unreachable"), ); @@ -92,6 +96,7 @@ describe("Hopper provider lifecycle health", () => { operation: "procedure_pseudo_code", }, }); + expect((await client.close()).ok).toBe(true); }); it("projects the exit code when the owned process exit is observed before disconnection", async () => { @@ -120,6 +125,7 @@ describe("Hopper provider lifecycle health", () => { stage: "analysis", operation: "hang", request_id: request.id, + launcher: { exit_code: 7, signal: null, output_closed: false }, }, }); }); @@ -166,3 +172,52 @@ describe("Hopper provider lifecycle health", () => { expect(client.operationHealth().state).toBe("idle"); }); }); + +it("preserves an owned signal exit and diagnostics for active, queued and later callers", async () => { + const { client, launcher } = await start(true); + const active = client.callTool("hang"); + const request = await launcher.fixture.waitForRequest("hang"); + const queued = client.callTool("echo"); + await vi.waitFor(() => + expect(client.requestActivity()?.queuedRequests).toBe(1), + ); + const token = launcher.fixture.tokens[0]; + launcher.observed.stderr.write(`API_KEY=local-evidence-kept ${token}\n`); + launcher.observed.exit(null, "SIGTERM"); + const results = await Promise.all([active, queued]); + for (const [index, result] of results.entries()) { + if (result.ok) throw new Error("Expected an observed provider exit"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "provider_unavailable", + message: expect.stringContaining("signal SIGTERM"), + details: { + provider_state: "exited", + operation: index === 0 ? "hang" : "echo", + request_id: request.id + index, + launcher: { + exit_code: null, + signal: "SIGTERM", + output_closed: false, + stderr: { + text: "API_KEY=local-evidence-kept [redacted transport credential]\n", + }, + }, + }, + }); + expect(JSON.stringify(projected)).not.toContain(token); + } + const later = await client.callTool("procedure_pseudo_code", { + procedure: "main", + }); + if (later.ok) throw new Error("An exited provider must reject later queries"); + expect(projectAnalysisError(later.error)).toMatchObject({ + code: "provider_unavailable", + details: { + provider_state: "exited", + stage: "decompilation", + operation: "procedure_pseudo_code", + launcher: { signal: "SIGTERM", output_closed: true }, + }, + }); +}); diff --git a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts index 928ec76a22..13601e172b 100644 --- a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts @@ -1,13 +1,68 @@ import { spawn } from "node:child_process"; +import { access } from "node:fs/promises"; import { expect, it, onTestFinished } from "vitest"; import { AnalysisError } from "../../../../src/domain/analysisErrorBase.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { ok } from "../../../../src/domain/result.js"; -import type { BridgeLauncher } from "../../../../src/hopper/BridgeLauncher.js"; +import type { + BridgeLauncher, + BridgeSession, +} from "../../../../src/hopper/BridgeLauncher.js"; import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; import { HopperClient } from "../../../../src/hopper/HopperClient.js"; +import { HopperFixtureLauncher } from "./hopperClient.fixture.js"; + +// Failure assertions keep the external document unknown. Teardown then publishes +// this run's authenticated bridge so normal cleanup can release its runtime/lease. +const trackFailedHelperClient = ( + launcher: BridgeLauncher, + startupTimeoutMs = 10_000, +): HopperClient => { + let session: BridgeSession | undefined; + let external = false; + const client = new HopperClient({ + launcher: { + async launch(input, options) { + session = input; + const launched = await launcher.launch(input, options); + external = + launched.ok && + launched.value.providerLifetime === "external-application"; + return launched; + }, + }, + startupTimeoutMs, + }); + onTestFinished(async () => { + const fixture = new HopperFixtureLauncher(); + const published = + external && session !== undefined + ? await fixture.launch(session) + : undefined; + try { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + if (published !== undefined) { + expect(fixture.requests.map((request) => request.method)).toEqual([ + "health", + "shutdown_document", + ]); + } + if (session !== undefined) + await expect(access(session.directory)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + if (published?.ok) { + const cleaned = await published.value.cleanup(); + expect(cleaned.cleaned).toBe(true); + } + } + }); + return client; +}; const projectPrimaryStartupFailure = (error: AnalysisError) => { if (!error.cleanupIncomplete) return projectAnalysisError(error); @@ -32,10 +87,7 @@ it("does not equate an owned native helper exit with an external GUI exit", asyn bridgeScriptPath: "/rea/hopper_bridge.py", launchMode: "native", }); - const client = new HopperClient({ launcher, startupTimeoutMs: 10_000 }); - onTestFinished(async () => { - await client.close(); - }, 45_000); + const client = trackFailedHelperClient(launcher); const result = await client.start(); expect(result).toMatchObject({ ok: false, @@ -84,10 +136,7 @@ it.each(["exit", "signal", "exit75"] as const)( ); }, }; - const client = new HopperClient({ launcher, startupTimeoutMs: 10_000 }); - onTestFinished(async () => { - await client.close(); - }, 45_000); + const client = trackFailedHelperClient(launcher); const result = await client.start(); expect(result).toMatchObject({ ok: false, @@ -136,10 +185,7 @@ it.each([0, 1])( ); }, }; - const client = new HopperClient({ launcher, startupTimeoutMs: 10_000 }); - onTestFinished(async () => { - await client.close(); - }, 45_000); + const client = trackFailedHelperClient(launcher); const result = await client.start(); expect(result).toMatchObject({ ok: false, @@ -174,10 +220,7 @@ it("preserves a successful helper's diagnostics when bridge readiness is never o ); }, }; - const client = new HopperClient({ launcher, startupTimeoutMs: 1500 }); - onTestFinished(async () => { - await client.close(); - }, 45_000); + const client = trackFailedHelperClient(launcher, 1500); const result = await client.start(); if (result.ok) throw new Error("Expected missing bridge readiness to time out"); diff --git a/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts b/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts index 88db157995..0388346b92 100644 --- a/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts @@ -53,10 +53,7 @@ class CancelThenFixtureLauncher implements BridgeLauncher { } class ExitingLauncher implements BridgeLauncher { - constructor( - readonly code: number, - readonly ownsProcessLifetime = true, - ) {} + constructor(readonly code: number) {} launch() { return Promise.resolve( @@ -64,10 +61,8 @@ class ExitingLauncher implements BridgeLauncher { process: spawn(process.execPath, ["-e", `process.exit(${this.code})`], { stdio: ["ignore", "ignore", "pipe"], }), - ownsProcessLifetime: this.ownsProcessLifetime, - providerLifetime: this.ownsProcessLifetime - ? ("launcher-process" as const) - : ("external-application" as const), + ownsProcessLifetime: true as const, + providerLifetime: "launcher-process" as const, shutdownMode: "bridge-request" as const, }), ); @@ -286,21 +281,4 @@ describe("HopperClient startup failures", () => { }, }); }); - - it("allows a short-lived launcher to hand off bridge startup", async () => { - const client = new HopperClient({ - launcher: new ExitingLauncher(0, false), - startupTimeoutMs: 100, - }); - clients.push(client); - const result = await client.start(); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error).toMatchObject({ - cleanupIncomplete: true, - cause: { _tag: "HopperTimeoutError" }, - diagnostics: { primary_error: { code: "provider_timeout" } }, - }); - } - }); }); diff --git a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts index 8de8212bfb..812cc2df72 100644 --- a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts +++ b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts @@ -218,6 +218,8 @@ const lipoRunner = (stdout: string): NativeCommandRunner => ({ // the same slice `arm64e`, and its physical identity must still match. describe("artifact Mach-O arm64e variant slices", () => { const ARM64E_V1 = { type: 0x0100000c, subtype: 0x81000002 } as const; + // CPU_SUBTYPE_ARM64E_X1 (12) with the pointer-auth ABI bit, as in macOS 27 /bin/ls. + const ARM64E_X1 = { type: 0x0100000c, subtype: 0x8000000c } as const; const variantSlices = async ( cpu: { readonly type: number; readonly subtype: number }, lipo: (size: number) => string, @@ -249,24 +251,29 @@ describe("artifact Mach-O arm64e variant slices", () => { it.each([ [ "arm64e.v1", + ARM64E_V1, " cputype CPU_TYPE_ARM64\n cpusubtype CPU_SUBTYPE_ARM64E\n capabilities PTR_AUTH_VERSION USERSPACE 1", ], [ "arm64e.x1", + ARM64E_X1, " cputype CPU_TYPE_ARM64\n cpusubtype unknown arm64 cpusubtype", ], - ])("accepts lipo's %s name for the FAT arm64e slice", async (name, cpu) => { - expect( - await variantSlices( - ARM64E_V1, - (size) => - `architecture ${name}\n${cpu}\n offset 8192\n size ${String(size)}\n align 2^12 (4096)\n`, - ), - ).toEqual([ - ["slices/x86_64", 4096], - [`slices/${name}`, 8192], - ]); - }); + ])( + "accepts lipo's %s name for the FAT arm64e slice", + async (name, fatCpu, cpu) => { + expect( + await variantSlices( + fatCpu, + (size) => + `architecture ${name}\n${cpu}\n offset 8192\n size ${String(size)}\n align 2^12 (4096)\n`, + ), + ).toEqual([ + ["slices/x86_64", 4096], + [`slices/${name}`, 8192], + ]); + }, + ); it.each([ ["a plain arm64 slice", CPU.arm64, "8192"], diff --git a/tests/evaluation/moduleBoundaries.test.ts b/tests/evaluation/moduleBoundaries.test.ts index e421017d42..09f906963e 100644 --- a/tests/evaluation/moduleBoundaries.test.ts +++ b/tests/evaluation/moduleBoundaries.test.ts @@ -161,21 +161,17 @@ describe("incremental module import boundaries", () => { ["src/application/probe.ts", "../browser/CdpCaptureValues.js", true], ["src/ghidra/probe.ts", "../generatedPackageMetadata.js", true], ["src/hopper/probe.ts", "../contracts/officialToolContracts.js", true], - ])("checks %s importing %s", async (file, dependency, allowed) => { - const temporary = await createTestTempDirectory("rea-module-boundary-"); - const fixture = join(temporary, file); - await mkdir(dirname(fixture), { recursive: true }); - await writeFile( - fixture, + ])("checks %s importing %s", (file, dependency, allowed) => { + const violations = inspectModuleBoundaries( + file, `import { value } from ${JSON.stringify(dependency)}; export { value };\n`, + process.cwd(), ); - const result = spawnSync(process.execPath, [verifier, file], { - cwd: temporary, - encoding: "utf8", - }); - if (result.error !== undefined) throw result.error; - expect(result.status).toBe(allowed ? 0 : 1); - if (!allowed) expect(result.stderr).toContain("resolves to src/"); + if (allowed) expect(violations).toEqual([]); + else + expect(violations).toMatchObject([ + { file, boundary: expect.any(String) }, + ]); }); }); @@ -282,21 +278,25 @@ describe("module import syntax and source discovery", () => { }); describe("test lane import boundaries", () => { - it.each([ + const cases: Array<[string, string, boolean]> = [ ["src/application/probe.test.ts", "../server/createServer.js", false], - ["tests/composition/probe.test.ts", "node:child_process", false], - ["tests/composition/probe.test.ts", "node:net", false], - ["tests/composition/probe.test.ts", "playwright", false], - ["tests/composition/probe.test.ts", "../../src/domain/result.js", true], - ])("checks %s importing %s", async (file, dependency, allowed) => { + ["tests/composition/process.test.ts", "node:child_process", false], + ["tests/composition/net.test.ts", "node:net", false], + ["tests/composition/browser.test.ts", "playwright", false], + ["tests/composition/domain.test.ts", "../../src/domain/result.js", true], + ]; + + it("checks the test-lane import matrix in one lint invocation", async () => { const temporary = await createTestTempDirectory("rea-test-lane-"); const config = join(temporary, ".oxlintrc.json"); await copyFile(resolve(".oxlintrc.json"), config); - await mkdir(dirname(join(temporary, file)), { recursive: true }); - await writeFile( - join(temporary, file), - `import { value } from ${JSON.stringify(dependency)}; export { value };\n`, - ); + for (const [file, dependency] of cases) { + await mkdir(dirname(join(temporary, file)), { recursive: true }); + await writeFile( + join(temporary, file), + `import { value } from ${JSON.stringify(dependency)}; export { value };\n`, + ); + } const result = spawnSync( process.execPath, [ @@ -305,13 +305,22 @@ describe("test lane import boundaries", () => { config, "--format", "json", - file, + ...cases.map(([file]) => file), ], { cwd: temporary, encoding: "utf8" }, ); if (result.error !== undefined) throw result.error; - expect(result.status).toBe(allowed ? 0 : 1); - if (!allowed) expect(result.stdout).toContain("no-restricted-imports"); + const report = JSON.parse(result.stdout) as { + diagnostics: Array<{ filename: string; code: string }>; + }; + const violations = new Set( + report.diagnostics + .filter(({ code }) => code === "eslint(no-restricted-imports)") + .map(({ filename }) => filename.replaceAll("\\", "/")), + ); + for (const [file, , allowed] of cases) + expect(violations.has(file)).toBe(!allowed); + expect(result.status).toBe(cases.some(([, , allowed]) => !allowed) ? 1 : 0); }); }); diff --git a/tests/fixtures/android/jadx-mcp.mjs b/tests/fixtures/android/jadx-mcp.mjs index e95029d531..4c008a7f44 100644 --- a/tests/fixtures/android/jadx-mcp.mjs +++ b/tests/fixtures/android/jadx-mcp.mjs @@ -10,10 +10,20 @@ const server = new McpServer({ version: mode === "empty-server-identity" ? "" : mode === "version" ? "99.0.0" : "1", }); +const requestedCount = Number.parseInt( + process.env.REA_FAKE_JADX_CLASS_COUNT ?? "", + 10, +); +const generatedCount = + Number.isInteger(requestedCount) && requestedCount >= 0 + ? requestedCount + : mode === "large-inventory" + ? 9000 + : 400; const classes = [ "fixture.Target", ...Array.from( - { length: mode === "large-inventory" ? 9000 : 400 }, + { length: generatedCount }, (_, index) => `fixture.Class${index}`, ), ]; diff --git a/tests/fixtures/android/jadx.ts b/tests/fixtures/android/jadx.ts index 49e17c2d50..373f3a6248 100644 --- a/tests/fixtures/android/jadx.ts +++ b/tests/fixtures/android/jadx.ts @@ -27,6 +27,7 @@ const fixture = fileURLToPath(new URL("./jadx-mcp.mjs", import.meta.url)); export const createJadxProtocolFixture = async ( mode = "normal", cleanupFailure?: () => boolean, + inventory?: { readonly classCount?: number }, ) => { const root = await createTestTempDirectory("rea-android-boundary-"); const apk = join(root, "fixture.apk"); @@ -64,7 +65,15 @@ export const createJadxProtocolFixture = async ( command: process.execPath, arguments: [fixture], expectedCommand: process.execPath, - env: { ...options.env, REA_FAKE_JADX_MODE: mode }, + env: { + ...options.env, + REA_FAKE_JADX_MODE: mode, + ...(inventory?.classCount === undefined + ? {} + : { + REA_FAKE_JADX_CLASS_COUNT: String(inventory.classCount), + }), + }, }); launches.push({ runId: options.runId, diff --git a/tests/fixtures/fakeHopper.mjs b/tests/fixtures/fakeHopper.mjs index 89886f76d1..192d69851a 100644 --- a/tests/fixtures/fakeHopper.mjs +++ b/tests/fixtures/fakeHopper.mjs @@ -10,8 +10,13 @@ const [ ] = process.argv.slice(2); let diagnosticOutputEmitted = false; const heldReplies = new Map(); +const sockets = new Set(); if (process.send !== undefined) process.on("message", (message) => { + if (message?.disconnect === true) { + for (const socket of sockets) socket.destroy(); + return; + } const reply = heldReplies.get(message?.release); if (reply === undefined) return; heldReplies.delete(message.release); @@ -173,6 +178,8 @@ const sendEchoWithProgress = (send, request) => { }; const server = createServer((socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); socket.on("error", () => undefined); socket.setEncoding("utf8"); let buffer = ""; diff --git a/tests/fixtures/keyedArchiveIntegers.ts b/tests/fixtures/keyedArchiveIntegers.ts deleted file mode 100644 index 11f5437512..0000000000 --- a/tests/fixtures/keyedArchiveIntegers.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { promisify } from "node:util"; - -import { expect } from "vitest"; -import { z } from "zod"; -import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js"; - -const rowSchema = z.object({ - container: z.enum(["roots", "objects"]), - root: z.string(), - objectId: z.number().int().nonnegative().optional(), - path: z.array(z.union([z.string(), z.number().int().nonnegative()])), - kind: z.enum(["integer", "real"]), - literal: z.string(), -}); -const archiveSchema = z.object({ - filename: z.string(), - sha256: z.string().regex(/^[a-f0-9]{64}$/u), - rows: z.array(rowSchema), -}); -const oracleSchema = z.object({ xml: archiveSchema, binary: archiveSchema }); -export const keyedArchiveIntegerFormats = ["xml", "binary"] as const; -export const keyedArchiveIntegerCases = ["controls", "unsafe"] as const; -type Case = (typeof keyedArchiveIntegerCases)[number]; -type Row = z.infer; -type SelectedFixture = { - readonly root: string; - readonly selected: z.infer; - readonly archivePath: string; - readonly digest: string; -}; -export interface KeyedArchiveIntegerFixtures { - readonly xml: SelectedFixture; - readonly binary: SelectedFixture; -} - -export const keyedArchiveIntegerFixtures = async ( - root: string, -): Promise => { - await promisify(execFile)( - "/usr/bin/xcrun", - [ - "swift", - "-module-cache-path", - join(root, "modules"), - "tests/conformance/native/keyed-archive-integers.swift", - root, - ], - { maxBuffer: 1024 * 1024 }, - ); - const oracle = oracleSchema.parse( - JSON.parse(await readFile(join(root, "oracle.json"), "utf8")), - ); - for (const entry of [oracle.xml, oracle.binary]) { - expect(entry.rows).toContainEqual( - expect.objectContaining({ - root: "unsafeSigned", - kind: "integer", - literal: "9007199254740993", - }), - ); - expect(entry.rows).toContainEqual( - expect.objectContaining({ - root: "unsafeNegative", - kind: "integer", - literal: "-9223372036854775808", - }), - ); - expect( - entry.rows.some( - (row) => - row.root === "boxedUnsigned" && - row.kind === "integer" && - row.literal === "9007199254740993", - ), - ).toBe(true); - expect( - entry.rows.some( - (row) => - row.root === "boxedReal" && - row.kind === "real" && - Number(row.literal) === 9007199254740992, - ), - ).toBe(true); - } - const selectedFixture = async ( - selected: z.infer, - ): Promise => { - const archivePath = join(root, selected.filename); - const bytes = await readFile(archivePath); - const digest = createHash("sha256").update(bytes).digest("hex"); - expect(digest).toBe(selected.sha256); - return { root, selected, archivePath, digest }; - }; - return { - xml: await selectedFixture(oracle.xml), - binary: await selectedFixture(oracle.binary), - }; -}; -const at = (value: unknown, path: Row["path"]): unknown => { - for (const key of path) { - if (value === null || typeof value !== "object") return undefined; - value = (value as Record)[key]; - } - return value; -}; -export const expectKeyedArchiveIntegerCase = ( - result: z.infer, - selectedCase: Case, - selected: z.infer, - digest: string, -) => { - expect(result.archive_sha256).toBe(digest); - const unsafe = (row: Row) => - row.kind === "integer" && - (BigInt(row.literal) > BigInt(Number.MAX_SAFE_INTEGER) || - BigInt(row.literal) < BigInt(Number.MIN_SAFE_INTEGER)); - const rows = selected.rows.filter((row) => - selectedCase === "unsafe" ? unsafe(row) : !unsafe(row), - ); - expect(rows.length).toBeGreaterThan(0); - for (const row of rows) { - const value = at( - row.container === "roots" - ? result.roots - : result.objects.find((object) => object.id === row.objectId)?.value, - row.path, - ); - expect( - value, - `${row.root} numeric slot remains present`, - ).not.toBeUndefined(); - if (unsafe(row)) { - const exact = - typeof value === "object" && - value !== null && - (value as Record).$plist_type === "integer" && - (value as Record).decimal === - BigInt(row.literal).toString(); - // The accepted alternate contract must explicitly disclose numeric precision loss. - const disclosed = result.limitations.some( - (reason) => - /integer|number/iu.test(reason) && - /precision|exact range|rounded/iu.test(reason), - ); - expect( - exact || disclosed, - `${row.root} ${row.literal} must be exact or disclose numeric loss`, - ).toBe(true); - } else { - expect(typeof value, `${row.root} retains its numeric type`).toBe( - "number", - ); - expect(value).toBe(Number(row.literal)); - } - } -};