diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml new file mode 100644 index 0000000000..5138df212f --- /dev/null +++ b/.github/workflows/real-recorded-crash.yml @@ -0,0 +1,99 @@ +name: Real recorded crash evidence + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/real-recorded-crash.yml" + - "bridge/pwntools/**" + - "bridge/pwndbg/**" + - "src/native/pwntools/**" + - "src/native/pwndbg/**" + - "src/domain/native/recordedCrash*" + - "src/application/binaryDiagnostics/**" + - "src/contracts/native/binaryDiagnosticsToolContracts.ts" + - "src/contracts/toolEffects.ts" + - "src/contracts/toolOutputSchemaGroups.ts" + - "src/composition/binaryDiagnostics.ts" + - "src/server/registerRecordedCrashTools.ts" + - "src/server/createServer.ts" + - "src/server/sessionAvailabilityPolicy.ts" + - "src/cli/binaryDiagnosticsCommands.ts" + - "src/cliCommandNames.ts" + - "src/process/**" + - "src/artifacts/readStableArtifact*.ts" + - "src/domain/analysisError*.ts" + - "scripts/verify-recorded-crash.mjs" + - "scripts/verify/recorded-crash/**" + - "scripts/lib/recorded-core-fixtures.py" + - "scripts/fixtures/recorded-crash.c" + - "third_party/pwndbg/**" + - "package.json" + - "package-lock.json" + +permissions: + contents: read + +concurrency: + group: real-recorded-crash-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NODE_OPTIONS: --max-old-space-size=768 + HUSKY: "0" + UV_CONCURRENT_BUILDS: "1" + UV_CONCURRENT_INSTALLS: "1" + UV_PYTHON_DOWNLOADS: never + +jobs: + real-recorded-crash: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - name: Install GDB on the disposable CI runner + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install --yes --no-install-recommends gdb + - name: Acquire exact unchanged upstreams in isolated runner storage + shell: bash + run: | + command -v gcc + command -v gdb + command -v strace + core_decoder_root="${RUNNER_TEMP}/rea-core-decoder" + core_pwndbg_root="${RUNNER_TEMP}/rea-core-pwndbg" + core_pwndbg_venv="${RUNNER_TEMP}/rea-core-pwndbg-venv" + python3 -m venv "${core_decoder_root}" + "${core_decoder_root}/bin/python" -m pip install --disable-pip-version-check \ + 'pwntools==4.15.0' 'pyelftools==0.33' 'unicorn==2.1.2' 'uv==0.12.0' + git init "${core_pwndbg_root}" + git -C "${core_pwndbg_root}" remote add origin https://github.com/pwndbg/pwndbg.git + git -C "${core_pwndbg_root}" fetch --depth=1 origin 55596aa676a2756efb8d67034f31675170eabe48 + git -C "${core_pwndbg_root}" checkout --detach FETCH_HEAD + test "$(git -C "${core_pwndbg_root}" rev-parse HEAD)" = 55596aa676a2756efb8d67034f31675170eabe48 + UV_PROJECT_ENVIRONMENT="${core_pwndbg_venv}" \ + "${core_decoder_root}/bin/uv" sync --project "${core_pwndbg_root}" \ + --frozen --no-editable --no-default-groups --python /usr/bin/python3 --no-managed-python + test -z "$(git -C "${core_pwndbg_root}" status --porcelain --untracked-files=no)" + echo "REA_PWNTOOLS_PYTHON=${core_decoder_root}/bin/python" >> "${GITHUB_ENV}" + echo "REA_PWNDBG_GDB=$(command -v gdb)" >> "${GITHUB_ENV}" + echo "REA_PWNDBG_GDBINIT=${core_pwndbg_root}/gdbinit.py" >> "${GITHUB_ENV}" + echo "REA_PWNDBG_VENV_PATH=${core_pwndbg_venv}" >> "${GITHUB_ENV}" + echo "REA_VERIFY_STRACE_COMMAND=$(command -v strace)" >> "${GITHUB_ENV}" + - name: Build and install the exact package in isolated runner storage + shell: bash + run: | + npm run build:cached + npm pack --ignore-scripts --json --pack-destination "${RUNNER_TEMP}" > "${RUNNER_TEMP}/rea-core-pack.json" + core_package_name=$(node -p 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))[0].filename' "${RUNNER_TEMP}/rea-core-pack.json") + npm install --prefix "${RUNNER_TEMP}/rea-core-installed" --ignore-scripts --no-fund --no-audit "${RUNNER_TEMP}/${core_package_name}" + - name: Verify installed CLI and MCP against the source-owned recording + shell: bash + run: node scripts/verify-recorded-crash.mjs "${RUNNER_TEMP}/rea-core-installed/node_modules/rea-agents/scripts/rea.mjs" diff --git a/README.md b/README.md index 348e188296..225f13db8e 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ Additional tools and host support depend on the target: | Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](https://morluto.github.io/rea/guides/native/) | | Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | | EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier; [offline EVM guide](docs/evm-bytecode.md) | +| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | | JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; [application analysis](https://morluto.github.io/rea/guides/javascript/) | | Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](https://morluto.github.io/rea/guides/browser/) | | Saved network captures | Requests, responses, exposed payloads and source locations | HAR; mitmdump on Linux for native mitmproxy captures; [capture guide](docs/web-network-captures.md) | diff --git a/README_ar.md b/README_ar.md index 8bc08d594a..a1dcb398db 100644 --- a/README_ar.md +++ b/README_ar.md @@ -217,7 +217,7 @@ rea setup | ---------------------- | ----: | -------------------------------------------------------------------------------------------------------------------------------------------------------- | | الفحص الأصلي | 41 | الدوال والشيفرة شبه المصدرية والتعليمات والسلاسل والرموز والاستدعاءات والمراجع والتعليقات وقراءة البايتات ومواضع الملفات | | سير عمل التحقيق | 14 | نظرات عامة على التطبيقات وملفات الدوال وواجهات API الأصلية والتوزيع وفك الترجمة الدفعي وتتبع الميزات ومسارات الاستدعاء ومخططاته واكتشاف Swift وObjective-C | -| أدوات الملفات الثنائية | 9 | بيانات Mach-O والتوقيعات وplist والمعماريات وأسماء Swift ومراقبة الاستدعاءات عبر LLDB على macOS؛ تخطيط ELF وأدلة الحماية الثابتة على Linux | +| أدوات الملفات الثنائية | 10 | بيانات Mach-O والتوقيعات وplist والمعماريات وأسماء Swift ومراقبة الاستدعاءات عبر LLDB على macOS؛ تخطيط ELF وأدلة الحماية الثابتة وسجلات الأعطال على Linux | | مخطط المخرجات | 7 | جرد الدلائل والحزم وملفات Interface Builder المترجمة وكتالوجات موارد Apple وتحليل تحميل Mach-O dylib والاستخراج واستدلال واجهات EVM دون اتصال | | PE/CLI المُدار | 7 | هوية تجميعات .NET وبياناتها الوصفية وتعليمات CIL والاعتماديات الأصلية واستيراد إعادة البناء ومقارنة الإصدارات | | البرامج الثابتة | 2 | فحص مناطق البرامج الثابتة على Linux واستخراجها الصريح | diff --git a/README_ja.md b/README_ja.md index 85fd9ba551..b10306692d 100644 --- a/README_ja.md +++ b/README_ja.md @@ -240,7 +240,7 @@ TypeScript と SQLite を使って私のプロジェクト向けに実装して | -------------------- | --: | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ネイティブ検査 | 41 | 関数、疑似コード、アセンブリ、文字列、シンボル、呼び出し、参照、注釈、バイト読み取り、ファイルオフセット | | 調査ワークフロー | 14 | アプリ概要、関数ドシエ、ネイティブ API とディスパッチ、一括逆コンパイル、機能トレース、コールパス、コールグラフ、Swift と Objective-C の検出 | -| ネイティブバイナリ | 9 | macOS の Mach-O メタデータ、署名、plist、アーキテクチャ、Swift シンボル、LLDB による呼び出し観測;Linux の ELF レイアウトと静的な保護機構の証拠 | +| ネイティブバイナリ | 10 | macOS の Mach-O メタデータ、署名、plist、アーキテクチャ、Swift シンボル、LLDB による呼び出し観測;Linux の ELF レイアウト、静的な保護機構の証拠と記録済みクラッシュ | | 成果物グラフ | 7 | ディレクトリとパッケージの目録、コンパイル済み Interface Builder ファイル、Apple アセットカタログ、Mach-O dylib のロード解決、抽出、オフライン EVM インターフェース推定 | | マネージド PE/CLI | 7 | .NET の識別情報、メタデータ、CIL 命令、ネイティブ依存関係、再構築結果のインポート、ビルド比較 | | ファームウェア | 2 | Linux ファームウェア領域の検査と明示的な展開 | diff --git a/README_ko.md b/README_ko.md index 3eb11722eb..cea69ecb3d 100644 --- a/README_ko.md +++ b/README_ko.md @@ -207,7 +207,7 @@ REA는 1~5단계의 바이너리 분석을 처리합니다. 6단계는 에이전 | ----------------- | --: | ---------------------------------------------------------------------------------------------------------------------------------------- | | 네이티브 검사 | 41 | 함수, 의사 코드, 어셈블리, 문자열, 심볼, 호출, 참조, 주석, 바이트 읽기, 파일 오프셋 | | 조사 워크플로 | 14 | 앱 개요, 함수 기록서, 네이티브 API와 디스패치, 일괄 디컴파일, 기능 추적, 호출 경로, 호출 그래프, Swift와 Objective-C 탐색 | -| 네이티브 바이너리 | 9 | macOS의 Mach-O 메타데이터, 서명, plist, 아키텍처, Swift 심볼과 LLDB 호출 관찰; Linux의 ELF 레이아웃과 정적 보호 증거 | +| 네이티브 바이너리 | 10 | macOS의 Mach-O 메타데이터, 서명, plist, 아키텍처, Swift 심볼과 LLDB 호출 관찰; Linux의 ELF 레이아웃, 정적 보호 증거와 기록된 크래시 | | 아티팩트 그래프 | 7 | 디렉터리와 패키지 목록, 컴파일된 Interface Builder 파일, Apple 애셋 카탈로그, Mach-O dylib 로드 해석, 추출, 오프라인 EVM 인터페이스 추론 | | 관리 PE/CLI | 7 | .NET 식별, 메타데이터, CIL 명령어, 네이티브 종속성, 재구성 가져오기, 빌드 비교 | | 펌웨어 | 2 | Linux 펌웨어 영역 검사 및 명시적 추출 | diff --git a/README_zh.md b/README_zh.md index 9b187b5ce1..4631c35636 100644 --- a/README_zh.md +++ b/README_zh.md @@ -202,20 +202,20 @@ REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常 ## 调查工具目录 -| 工具类别 | 数量 | 用途 | -| ----------------- | ---: | ----------------------------------------------------------------------------------------------------------------- | -| 原生检查 | 41 | 函数、伪代码、汇编、字符串、符号、调用、引用、注释、字节读取与文件偏移 | -| 调查工作流 | 14 | 应用概览、函数档案、原生 API 与分发、批量反编译、功能追踪、调用路径与调用图、Swift 与 Objective-C 发现 | -| 原生二进制工具 | 9 | macOS 的 Mach-O 元数据、签名、plist、架构、Swift 符号与 LLDB 调用观察;Linux 的 ELF 布局和静态保护证据 | -| 产物图 | 7 | 目录与软件包清单、编译后的 Interface Builder 文件、Apple 资源目录、Mach-O dylib 加载解析、提取与离线 EVM 接口推断 | -| 托管 PE/CLI | 7 | .NET 程序集身份、元数据、CIL 指令、原生依赖声明、重建导入与构建比较 | -| 固件 | 2 | Linux 固件区域检查与显式提取 | -| Android APK | 5 | 包与 manifest 声明、类搜索、成员清单、方法反编译与静态引用 | -| 浏览器观察 | 12 | 页面结构、网络元数据、脚本、来源映射、WebMCP 发现、截图与捕获比较 | -| Electron 分析 | 5 | 渲染进程观察、静态应用映射、静态/运行时结果关联 | -| JavaScript 运行时 | 2 | Node/Electron Inspector 目标发现、脚本位置与执行上下文事件 | -| 应用工作流 | 13 | 捕获的网站脚本导出、跨层功能追踪、构建比较、历史源码映射、静态返回结构比较与重建验证 | -| 工作区与观察 | 21 | 会话、证据包、导航上下文、进程/产物/函数比较与待解决问题跟踪 | +| 工具类别 | 数量 | 用途 | +| ----------------- | ---: | -------------------------------------------------------------------------------------------------------------------- | +| 原生检查 | 41 | 函数、伪代码、汇编、字符串、符号、调用、引用、注释、字节读取与文件偏移 | +| 调查工作流 | 14 | 应用概览、函数档案、原生 API 与分发、批量反编译、功能追踪、调用路径与调用图、Swift 与 Objective-C 发现 | +| 原生二进制工具 | 10 | macOS 的 Mach-O 元数据、签名、plist、架构、Swift 符号与 LLDB 调用观察;Linux 的 ELF 布局、静态保护证据与历史崩溃记录 | +| 产物图 | 7 | 目录与软件包清单、编译后的 Interface Builder 文件、Apple 资源目录、Mach-O dylib 加载解析、提取与离线 EVM 接口推断 | +| 托管 PE/CLI | 7 | .NET 程序集身份、元数据、CIL 指令、原生依赖声明、重建导入与构建比较 | +| 固件 | 2 | Linux 固件区域检查与显式提取 | +| Android APK | 5 | 包与 manifest 声明、类搜索、成员清单、方法反编译与静态引用 | +| 浏览器观察 | 12 | 页面结构、网络元数据、脚本、来源映射、WebMCP 发现、截图与捕获比较 | +| Electron 分析 | 5 | 渲染进程观察、静态应用映射、静态/运行时结果关联 | +| JavaScript 运行时 | 2 | Node/Electron Inspector 目标发现、脚本位置与执行上下文事件 | +| 应用工作流 | 13 | 捕获的网站脚本导出、跨层功能追踪、构建比较、历史源码映射、静态返回结构比较与重建验证 | +| 工作区与观察 | 21 | 会话、证据包、导航上下文、进程/产物/函数比较与待解决问题跟踪 | ## 路线图 diff --git a/bridge/pwndbg/core_context.py b/bridge/pwndbg/core_context.py new file mode 100644 index 0000000000..57f86f6882 --- /dev/null +++ b/bridge/pwndbg/core_context.py @@ -0,0 +1,101 @@ +"""Fixed core-only GDB/pwndbg operation; no caller commands or historical PID lookup.""" +import json +import os +from pathlib import Path +import resource +import gdb + +OUTPUT_BYTES = 64 * 1024 * 1024 + + +class ContextFailure(Exception): + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason + + +def inspect(request): + if gdb.selected_inferior().connection is not None: + raise ContextFailure("decoder", "Owned GDB unexpectedly has an inferior connection before core loading.") + root = Path(os.environ["REA_CORE_CONTEXT_REQUEST"]).parent + empty = root / "no-libraries" + empty.mkdir(mode=0o700) + for command in ["set auto-load off", "set debuginfod enabled off", "set auto-solib-add off", "set history save off", "set pagination off", + "set sysroot ./no-libraries", "set solib-search-path ./no-libraries"]: + gdb.execute(command, to_string=True) + entry = request["gdbinit"] + try: + exec(compile(Path(entry).read_bytes(), entry, "exec"), {"__name__": "__main__", "__file__": entry}) + import pwndbg.aglib.vmmap + import pwndbg.lib.version + except (ImportError, OSError) as error: + raise ContextFailure("unavailable", "Selected pwndbg could not initialize: " + str(error)) from error + version = pwndbg.lib.version.__version__ + if version != "2026.09.15" and not version.startswith("2026.09.15 build: "): + raise ContextFailure("unsupported", "Expected unchanged pwndbg2026.09.15; selected plugin reports " + version + ".") + gdb.execute('set context-sections ""', to_string=True) + # core-file takes a raw filename and retains quote characters. Use a fixed + # relative filename in the owned cwd so no selected path becomes GDB syntax. + if Path(request["snapshot_path"]) != root.parent / "object.snapshot": + raise ContextFailure("decoder", "Core context request did not select the shared owned snapshot.") + gdb.execute("core-file ../object.snapshot", to_string=True) + inferior = gdb.selected_inferior() + if inferior.connection is None or inferior.connection.type != "core" or gdb.current_progspace().filename is not None: + raise ContextFailure("decoder", "Requested core-only boundary was not established; no mapping query performed.") + maps = [] + for page in pwndbg.aglib.vmmap.get(): + flags = page.flags + maps.append({"start": hex(page.start), "end": hex(page.end), "offset": hex(page.offset), "reported_flags": flags, + "permissions": {"read": bool(flags & 4), "write": bool(flags & 2), "execute": bool(flags & 1)} if 1 <= flags <= 7 else None, + "pathname_display": page.objfile, "current_file_identity": "unknown"}) + return {"status": "available", "gdb_version": gdb.VERSION, "pwndbg_version": version, "connection": "core", "executable": None, + "historical_pid": inferior.pid, "reported_limits": {"address_space_bytes": resource.getrlimit(resource.RLIMIT_AS)[0], "cpu_seconds": resource.getrlimit(resource.RLIMIT_CPU)[0], "file_size_bytes": resource.getrlimit(resource.RLIMIT_FSIZE)[0]}, "maps": maps, "confidence": "derived", "completeness": "unknown", + "diagnostics": {"stdout": "", "stderr": "", "truncated": False}} + + +def failure_exit(request_path, status, marker): + try: + descriptor = os.open(Path(request_path).parent / "resource.failure", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write(descriptor, marker) + os.close(descriptor) + except (OSError, MemoryError): + pass + os._exit(status) + + +def main(): + request_path = os.environ["REA_CORE_CONTEXT_REQUEST"] + request = json.loads(Path(request_path).read_text(encoding="utf-8")) + reserve = bytearray(1024 * 1024) + try: + reply = {"ok": True, "value": inspect(request)} + except ContextFailure as error: + reply = {"ok": False, "reason": error.reason, "message": str(error)} + except MemoryError: + reserve = None + failure_exit(request_path, 75, b"M") + except Exception as error: + if isinstance(error, OSError) and error.errno == 12: + reserve = None + failure_exit(request_path, 75, b"M") + reply = {"ok": False, "reason": "decoder", "message": type(error).__name__ + ": " + str(error)} + try: + encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") + if len(encoded) > OUTPUT_BYTES: + encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete debugger context exceeds64 MiB reply budget."}).encode("utf-8") + descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + except MemoryError: + reserve = None + failure_exit(request_path, 75, b"M") + except OSError as error: + if error.errno == 27: + failure_exit(request_path, 76, b"F") + if error.errno == 12: + reserve = None + failure_exit(request_path, 75, b"M") + raise + + +main() diff --git a/bridge/pwndbg/launch.py b/bridge/pwndbg/launch.py new file mode 100644 index 0000000000..7439167e25 --- /dev/null +++ b/bridge/pwndbg/launch.py @@ -0,0 +1,39 @@ +"""Catch typed resource failures while loading or running the owned debugger launcher. + +Only this package's fixed sibling is executed; caller input stays in request.json. +Interpreter startup failures before this boundary remain unclassified. +""" +import os +import sys + + +def resource_exit(status, marker): + try: + if len(sys.argv) > 2: + descriptor = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write(descriptor, marker) + os.close(descriptor) + except (OSError, MemoryError): + # Without a complete marker the parent keeps this failure unclassified. + pass + os._exit(status) + +try: + implementation = os.path.join(os.path.dirname(__file__), "launch_impl.py") + with open(implementation, "rb") as handle: + source = handle.read() + exec(compile(source, implementation, "exec"), { + "__file__": implementation, + "__name__": "__main__", + }) +except MemoryError: + # Reserved status: an observed allocation failure, including imports/compile. + resource_exit(75, b"M") +except OSError as error: + # Linux ENOMEM is an observed allocation failure, independent of error text. + if error.errno == 12: + resource_exit(75, b"M") + # Linux EFBIG is observed directly, including a partial limits/reply write. + if error.errno == 27: + resource_exit(76, b"F") + raise diff --git a/bridge/pwndbg/launch_impl.py b/bridge/pwndbg/launch_impl.py new file mode 100644 index 0000000000..c1cd04b04d --- /dev/null +++ b/bridge/pwndbg/launch_impl.py @@ -0,0 +1,34 @@ +"""Lower inherited limits before replacing this owned process with GDB.""" +import json +import os +from pathlib import Path +import sys + +runtime_path = Path(__file__).parent.parent / "pwntools" / "decoder_runtime.py" +runtime = {"__file__": str(runtime_path), "__name__": "rea_decoder_runtime"} +exec(compile(runtime_path.read_bytes(), str(runtime_path), "exec"), runtime) + + +class LaunchFailure(Exception): + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason + + +def launch(snapshot, cache): + request_path = sys.argv[-1] + request = json.loads(Path(request_path).read_text(encoding="utf-8")) + bridge = str(Path(__file__).with_name("core_context.py")) + environment = dict(os.environ, REA_CORE_CONTEXT_REQUEST=request_path) + # Only fixed package code appears in the GDB Python command. Selected paths stay JSON data. + command = "python exec(compile(open(" + repr(bridge) + ", 'rb').read(), " + repr(bridge) + ", 'exec'), {'__name__':'__main__','__file__':" + repr(bridge) + "})" + try: + os.execve(request["gdb"], [request["gdb"], "-nx", "-nh", "--batch", "--quiet", + "-iex", "set auto-load off", "-iex", "set debuginfod enabled off", + "-iex", "set auto-solib-add off", "-ex", command], environment) + except OSError as error: + raise LaunchFailure("unavailable", "Selected GDB could not start (" + str(error.errno) + "): " + request["gdb"] + "; " + str(error)) from error + + +if __name__ == "__main__": + runtime["main"](sys.argv[-1], launch, LaunchFailure, "pwndbg-core-launcher") diff --git a/bridge/pwntools/decoder_runtime.py b/bridge/pwntools/decoder_runtime.py new file mode 100644 index 0000000000..6681dffee0 --- /dev/null +++ b/bridge/pwntools/decoder_runtime.py @@ -0,0 +1,59 @@ +"""Shared bounded reply/limit protocol, independent of the supplied artifact format.""" +import json +import os +from pathlib import Path +import resource +import sys + +OUTPUT_BYTES = 64 * 1024 * 1024 + +def lower_resource_limits(): + limits = {} + for name, kind, maximum in ( + ("address_space_bytes", resource.RLIMIT_AS, 3 * 1024**3), + ("cpu_seconds", resource.RLIMIT_CPU, 30), + ("file_size_bytes", resource.RLIMIT_FSIZE, OUTPUT_BYTES), + ): + soft, hard = resource.getrlimit(kind) + finite = [maximum] + [value for value in (soft, hard) if value != resource.RLIM_INFINITY] + effective = min(finite) + # Preserve the inherited hard boundary and never raise a caller's soft limit. + resource.setrlimit(kind, (effective, hard)) + limits[name] = effective + return limits + + +def main(request_path, inspect, failure_type, profile): + request = json.loads(Path(request_path).read_text(encoding="utf-8")) + os.environ["PWNLIB_NOTERM"] = "1" + os.environ["PWNLIB_CACHE_DIR"] = str(Path(request_path).parent / "cache") + limits = None + # Release bounded emergency headroom before constructing a MemoryError reply. + # The outer exit-status contract also covers allocation failure during serialization. + memory_reserve = bytearray(1024 * 1024) + try: + limits = lower_resource_limits() + descriptor = os.open(Path(request_path).parent / "limits.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(limits, handle) + value = inspect(Path(request["snapshot_path"]), Path(request_path).parent / "cache") + value["limitations"].append("Effective owned Python resource soft limits: " + json.dumps(limits, sort_keys=True) + ". Inherited tighter limits are retained.") + reply = {"ok": True, "profile": profile, "value": value} + except failure_type as error: + reply = {"ok": False, "reason": error.reason, "message": str(error)} + except MemoryError: + memory_reserve = None + reply = {"ok": False, "reason": "resource-limit", "message": "pwntools memory allocation failed under the effective resource limits; the exact allocation cause is unknown.", "reported_limits": limits} + except Exception as error: + if isinstance(error, OSError) and error.errno == 12: + memory_reserve = None + reply = {"ok": False, "reason": "resource-limit", "message": "pwntools reported OSError(ENOMEM); the exact allocation cause is unknown.", "reported_limits": limits} + else: + reply = {"ok": False, "reason": "decoder", "message": type(error).__name__ + ": " + str(error)} + encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") + if len(encoded) > OUTPUT_BYTES: + encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete artifact result exceeds the 64 MiB reply budget."}).encode("utf-8") + descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + diff --git a/bridge/pwntools/layout_impl.py b/bridge/pwntools/layout_impl.py index 92b3ea9e85..0cae2b46f0 100644 --- a/bridge/pwntools/layout_impl.py +++ b/bridge/pwntools/layout_impl.py @@ -65,7 +65,7 @@ def dynamic_name_reference(tags, offset, image, content): if not starts or not sizes: raise LayoutFailure("format", "ELF dependency requires DT_STRTAB and DT_STRSZ.") if len(starts) != 1 or len(sizes) != 1: - raise LayoutFailure("unsupported", "ELF dependency string-table tags are ambiguous: " + repr({"DT_STRTAB": starts, "DT_STRSZ": sizes})) + raise LayoutFailure("unsupported-target", "ELF dependency string-table tags are ambiguous: " + repr({"DT_STRTAB": starts, "DT_STRSZ": sizes})) if not 0 <= offset < sizes[0]: raise LayoutFailure("format", f"ELF dependency name offset {offset} lies outside declared DT_STRSZ {sizes[0]}.") offsets = set() @@ -75,7 +75,7 @@ def dynamic_name_reference(tags, offset, image, content): if starts[0] >= h.p_vaddr and starts[0] + sizes[0] <= h.p_vaddr + h.p_filesz: offsets.add(h.p_offset + starts[0] - h.p_vaddr) if len(offsets) != 1: - raise LayoutFailure("unsupported", "Dynamic string table lacks a unique file-backed mapping: " + repr({"DT_STRTAB": starts, "DT_STRSZ": sizes, "candidate_file_offsets": sorted(offsets)})) + raise LayoutFailure("unsupported-target", "Dynamic string table lacks a unique file-backed mapping: " + repr({"DT_STRTAB": starts, "DT_STRSZ": sizes, "candidate_file_offsets": sorted(offsets)})) return name_in_table(None, offsets.pop(), sizes[0], offset, content) @@ -117,13 +117,13 @@ def inspect_elf(path, cache): if len(content) < 16: raise LayoutFailure("format", "Selected ELF identification is truncated.") if content[4] != 2 or content[5] != 1: - raise LayoutFailure("unsupported", "Initial layout profile requires ELF64 little-endian objects.") + raise LayoutFailure("unsupported-target", "Initial layout profile requires ELF64 little-endian objects.") if len(content) < 64: raise LayoutFailure("format", "Selected ELF64 header is truncated.") if int.from_bytes(content[18:20], "little") != 62: - raise LayoutFailure("unsupported", "Initial layout profile requires x86-64 ELF objects.") + raise LayoutFailure("unsupported-target", "Initial layout profile requires x86-64 ELF objects.") if int.from_bytes(content[16:18], "little") not in (1, 2, 3): - raise LayoutFailure("unsupported", "Selected ELF type is outside the EXEC/DYN/REL layout profile; recorded core analysis is separate.") + raise LayoutFailure("unsupported-target", "Selected ELF type is outside the EXEC/DYN/REL layout profile; recorded core analysis is separate.") from pwnlib.context import context with context.local(cache_dir=str(cache), log_level="warning"): from elftools.common.exceptions import ELFError @@ -169,9 +169,9 @@ def inspect_elf(path, cache): try: image = LayoutELF(str(path), checksec=False) if image.arch != "amd64" or image.bits != 64 or image.endian != "little": - raise LayoutFailure("unsupported", "Initial layout profile requires x86-64 ELF64 little-endian objects.") + raise LayoutFailure("unsupported-target", "Initial layout profile requires x86-64 ELF64 little-endian objects.") if image.header.e_type not in ("ET_EXEC", "ET_DYN", "ET_REL"): - raise LayoutFailure("unsupported", "Selected ELF type is outside the EXEC/DYN/REL layout profile; recorded core analysis is separate.") + raise LayoutFailure("unsupported-target", "Selected ELF type is outside the EXEC/DYN/REL layout profile; recorded core analysis is separate.") length = len(content) all_sections = list(image.iter_sections()) shstrings = image.get_section(section_name_index) if all_sections and section_name_index != 0 else None @@ -301,56 +301,9 @@ def inspect_elf(path, cache): raise LayoutFailure("format", "Selected ELF failed unchanged upstream structural parsing: " + str(error)) from error -def lower_resource_limits(): - limits = {} - for name, kind, maximum in ( - ("address_space_bytes", resource.RLIMIT_AS, 3 * 1024**3), - ("cpu_seconds", resource.RLIMIT_CPU, 30), - ("file_size_bytes", resource.RLIMIT_FSIZE, OUTPUT_BYTES), - ): - soft, hard = resource.getrlimit(kind) - finite = [maximum] + [value for value in (soft, hard) if value != resource.RLIM_INFINITY] - effective = min(finite) - # Preserve the inherited hard boundary and never raise a caller's soft limit. - resource.setrlimit(kind, (effective, hard)) - limits[name] = effective - return limits - - -def main(request_path): - request = json.loads(Path(request_path).read_text(encoding="utf-8")) - os.environ["PWNLIB_NOTERM"] = "1" - os.environ["PWNLIB_CACHE_DIR"] = str(Path(request_path).parent / "cache") - limits = None - # Release bounded emergency headroom before constructing a MemoryError reply. - # The outer exit-status contract also covers allocation failure during serialization. - memory_reserve = bytearray(1024 * 1024) - try: - limits = lower_resource_limits() - descriptor = os.open(Path(request_path).parent / "limits.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - json.dump(limits, handle) - value = inspect_elf(Path(request["snapshot_path"]), Path(request_path).parent / "cache") - value["limitations"].append("Effective owned Python resource soft limits: " + json.dumps(limits, sort_keys=True) + ". Inherited tighter limits are retained.") - reply = {"ok": True, "profile": PROFILE, "value": value} - except LayoutFailure as error: - reply = {"ok": False, "reason": error.reason, "message": str(error)} - except MemoryError: - memory_reserve = None - reply = {"ok": False, "reason": "resource-limit", "message": "pwntools memory allocation failed under the effective resource limits; the exact allocation cause is unknown.", "reported_limits": limits} - except Exception as error: - if isinstance(error, OSError) and error.errno == 12: - memory_reserve = None - reply = {"ok": False, "reason": "resource-limit", "message": "pwntools reported OSError(ENOMEM); the exact allocation cause is unknown.", "reported_limits": limits} - else: - reply = {"ok": False, "reason": "decoder", "message": type(error).__name__ + ": " + str(error)} - encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") - if len(encoded) > OUTPUT_BYTES: - encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete ELF layout exceeds the 64 MiB reply budget."}).encode("utf-8") - descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "wb") as handle: - handle.write(encoded) - if __name__ == "__main__": - main(sys.argv[-1]) + runtime_path = Path(__file__).with_name("decoder_runtime.py") + runtime = {"__file__": str(runtime_path), "__name__": "rea_decoder_runtime"} + exec(compile(runtime_path.read_bytes(), str(runtime_path), "exec"), runtime) + runtime["main"](sys.argv[-1], inspect_elf, LayoutFailure, PROFILE) diff --git a/bridge/pwntools/recorded_crash.py b/bridge/pwntools/recorded_crash.py new file mode 100644 index 0000000000..f8bc3f3053 --- /dev/null +++ b/bridge/pwntools/recorded_crash.py @@ -0,0 +1,39 @@ +"""Catch typed resource failures while loading or running the owned recorded-core adapter. + +Only this package's fixed sibling is executed; caller input stays in request.json. +Interpreter startup failures before this boundary remain unclassified. +""" +import os +import sys + + +def resource_exit(status, marker): + try: + if len(sys.argv) > 2: + descriptor = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write(descriptor, marker) + os.close(descriptor) + except (OSError, MemoryError): + # Without a complete marker the parent keeps this failure unclassified. + pass + os._exit(status) + +try: + implementation = os.path.join(os.path.dirname(__file__), "recorded_crash_impl.py") + with open(implementation, "rb") as handle: + source = handle.read() + exec(compile(source, implementation, "exec"), { + "__file__": implementation, + "__name__": "__main__", + }) +except MemoryError: + # Reserved status: an observed allocation failure, including imports/compile. + resource_exit(75, b"M") +except OSError as error: + # Linux ENOMEM is an observed allocation failure, independent of error text. + if error.errno == 12: + resource_exit(75, b"M") + # Linux EFBIG is observed directly, including a partial limits/reply write. + if error.errno == 27: + resource_exit(76, b"F") + raise diff --git a/bridge/pwntools/recorded_crash_impl.py b/bridge/pwntools/recorded_crash_impl.py new file mode 100644 index 0000000000..84aa72ac54 --- /dev/null +++ b/bridge/pwntools/recorded_crash_impl.py @@ -0,0 +1,143 @@ +"""Bounded Linux core evidence via unchanged ELFFile/structs and pwntools ctypes. + +No Corefile-selected thread, executable guessing, process lookup or target launch. +""" +import base64 +import ctypes +import importlib.metadata +import io +from pathlib import Path +import sys + +PROFILE = "pwntools@4.15.0;pyelftools@0.33;unicorn@2.1.2" + + +class CoreFailure(Exception): + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason + + +def encoded(value): + return base64.b64encode(value).decode("ascii") + + +def location(offset, size, limit): + if not 0 <= offset <= limit or not 0 <= size <= limit - offset: + raise CoreFailure("format", "Recorded source range exceeds its declared container.") + return {"offset": hex(offset), "bytes": hex(size)} + + +def inspect_core(path, cache): + for package, required in {"pwntools": "4.15.0", "pyelftools": "0.33", "unicorn": "2.1.2"}.items(): + try: + actual = importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError as error: + raise CoreFailure("unavailable", "Selected Python lacks " + package + " " + required + ".") from error + if actual != required: + raise CoreFailure("unsupported", "Expected " + package + " " + required + "; selected Python reports " + actual + ".") + content = path.read_bytes() + if len(content) < 16 or content[:4] != b"\x7fELF": + raise CoreFailure("format", "Selected recording lacks a complete ELF identification.") + if content[4:6] != b"\x02\x01": + raise CoreFailure("unsupported-target", "Initial recorded-core profile requires ELF64 little-endian.") + if len(content) < 64: + raise CoreFailure("format", "Selected ELF64 core header is truncated.") + # Linux commonly emits ELFOSABI_NONE (0), also accepted as GNU/Linux (3). + # Other declared OSABIs must never select Linux note ctypes. + if content[7] not in (0, 3) or content[8] != 0: + raise CoreFailure("unsupported-target", "Initial recorded-core profile requires System V or GNU/Linux OSABI with ABI version zero; selected OSABI=" + str(content[7]) + ", ABI version=" + str(content[8]) + ".") + if int.from_bytes(content[16:18], "little") != 4 or int.from_bytes(content[18:20], "little") != 62: + raise CoreFailure("unsupported-target", "Initial recorded-core profile requires x86-64 ET_CORE.") + from elftools.common.exceptions import ELFError + from elftools.common.utils import struct_parse + from elftools.elf.elffile import ELFFile + from pwnlib.elf.datatypes import elf_prstatus_amd64, elf_siginfo_64, user_regs_struct_amd64 + try: + image = ELFFile(io.BytesIO(content)) + header = image.header + count = image.num_segments() + if count and header.e_phentsize < image.structs.Elf_Phdr.sizeof(): + raise CoreFailure("format", "Program-header entry is shorter than the required ELF64 structure.") + location(header.e_phoff, count * header.e_phentsize, len(content)) + segments, notes, padding, threads, signals = [], [], [], [], [] + for segment_index, segment in enumerate(image.iter_segments()): + h = segment.header + backing = h.p_type != "PT_NULL" and h.p_filesz > 0 + if backing: + location(h.p_offset, h.p_filesz, len(content)) + if h.p_type == "PT_LOAD" and h.p_filesz > h.p_memsz: + raise CoreFailure("format", "Recorded PT_LOAD file bytes exceed memory size.") + segments.append({"index": segment_index, "type": h.p_type, + "header_location": location(header.e_phoff + segment_index * header.e_phentsize, header.e_phentsize, len(content)), + "offset": hex(h.p_offset), "file_size": hex(h.p_filesz), "memory_size": hex(h.p_memsz), + "virtual_address": hex(h.p_vaddr), "physical_address": hex(h.p_paddr), "alignment": hex(h.p_align), "flags": hex(h.p_flags), "file_backing": "file" if backing else "none"}) + if h.p_type != "PT_NOTE" or h.p_filesz == 0: + continue + position, end = h.p_offset, h.p_offset + h.p_filesz + header_size = image.structs.Elf_Nhdr.sizeof() + while position < end: + if end - position < header_size: + raw = content[position:end] + if any(raw): + raise CoreFailure("format", "Nonzero trailing note bytes lack a complete note header.") + padding.append({"segment_index": segment_index, "location": location(position, len(raw), end), "bytes_base64": encoded(raw)}) + break + note = struct_parse(image.structs.Elf_Nhdr, image.stream, stream_pos=position) + owner_start = position + header_size + descriptor_start = owner_start + ((note.n_namesz + 3) // 4) * 4 + note_end = descriptor_start + ((note.n_descsz + 3) // 4) * 4 + location(position, note_end - position, end) + owner = content[owner_start:owner_start + note.n_namesz] + if owner and b"\0" not in owner: + raise CoreFailure("format", "Declared note owner has no NUL terminator.") + display = owner.split(b"\0", 1)[0].decode("utf-8", "replace") if owner else None + descriptor = content[descriptor_start:descriptor_start + note.n_descsz] + note_index = len(notes) + notes.append({"index": note_index, "segment_index": segment_index, "type": note.n_type, + "location": location(position, note_end - position, end), + "owner_location": location(owner_start, len(owner), end), "owner_bytes_base64": encoded(owner), "owner_display": display, + "descriptor_location": location(descriptor_start, len(descriptor), end), "descriptor_bytes_base64": encoded(descriptor)}) + position = note_end + # Only the initial Linux CORE ABI is interpreted. Other owners retain exact bytes. + if owner != b"CORE\0": + continue + if note.n_type == "NT_PRSTATUS": + if len(descriptor) < ctypes.sizeof(elf_prstatus_amd64): + raise CoreFailure("format", "Linux amd64 NT_PRSTATUS descriptor is truncated.") + status = elf_prstatus_amd64.from_buffer_copy(descriptor) + registers = [] + for name, kind in user_regs_struct_amd64._fields_: + relative = elf_prstatus_amd64.pr_reg.offset + getattr(user_regs_struct_amd64, name).offset + registers.append({"name": name, "value": hex(getattr(status.pr_reg, name)), "location": location(descriptor_start + relative, ctypes.sizeof(kind), end)}) + threads.append({"note_index": note_index, "historical_pid": ctypes.c_int32(status.pr_pid).value, "recorded_current_signal": status.pr_cursig, "registers": registers}) + elif note.n_type == "NT_SIGINFO": + if len(descriptor) < ctypes.sizeof(elf_siginfo_64): + raise CoreFailure("format", "Linux amd64 NT_SIGINFO descriptor is truncated.") + signal = elf_siginfo_64.from_buffer_copy(descriptor) + number = ctypes.c_int32(signal.si_signo).value + code = ctypes.c_int32(signal.si_code).value + known = number == 11 and code in (1, 2) + signals.append({"note_index": note_index, "number": number, "code": code, "errno": ctypes.c_int32(signal.si_errno).value, + "fault_address": hex(signal.sigfault_addr) if known else None, + "fault_address_meaning": "recorded-sigsegv-address" if known else "unknown", "thread_association": "unknown"}) + return {"format": "elf-core", "architecture": "x86_64-little-endian", "target_execution": "not-performed", "live_process_identity": "unknown", + "segments": segments, "notes": notes, "note_padding": padding, "threads": threads, "signals": signals, + "note_interpretation_completeness": "unknown", + "limitations": ["The initial Linux note profile accepts System V/unspecified or GNU/Linux OSABI with ABI version zero; an unspecified OSABI alone does not prove operating-system origin.", + "Historical PID and signal values are recorded metadata, not current process identity or attach authority.", + "Only exact Linux CORE NT_PRSTATUS/NT_SIGINFO owners are interpreted; every descriptor and owner retains original bytes. Other note semantics remain unknown.", + "NT_SIGINFO thread association is unknown. Only SIGSEGV SEGV_MAPERR/SEGV_ACCERR initially establish a fault-address union meaning.", + "PT_LOAD file bytes are recorded bytes. Undumped memory and current executable/library identity remain unknown; no current host file is resolved from a recorded pathname.", + "Missing notes/registers/signals do not establish their absence at the original crash. Full unwinding, executable pairing and live control are not provided."]} + except CoreFailure: + raise + except (ELFError, ValueError, AssertionError, IndexError) as error: + raise CoreFailure("format", "Recorded core failed unchanged upstream structural parsing: " + str(error)) from error + + +if __name__ == "__main__": + runtime_path = Path(__file__).with_name("decoder_runtime.py") + runtime = {"__file__": str(runtime_path), "__name__": "rea_decoder_runtime"} + exec(compile(runtime_path.read_bytes(), str(runtime_path), "exec"), runtime) + runtime["main"](sys.argv[-1], inspect_core, CoreFailure, PROFILE) diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index 8bbb9a1474..b02bfafbe9 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -26,7 +26,7 @@ graph TD end subgraph "Provider Layers" - PWNTOOLS["src/native/pwntools/
BYO offline ELF adapter
owned Python process, unchanged upstream tables/heuristics"] + PWNTOOLS["src/native/pwntools/ + src/native/pwndbg/
BYO offline ELF/core adapters
owned snapshots/Python and optional core-only GDB
unchanged upstream parsers, ABI structures and mapping API"] EVM["src/evm/
unchanged EVMole/WASM in bounded owned worker
explicit carrier parsing and inferred interface adapter"] HOPPER["src/hopper/
Hopper launch & Unix-socket protocol
BridgeLauncher, HopperClient, protocol"] GHIDRA["src/ghidra/
Ghidra discovery, profile & function analysis
snapshot, Unix/Windows transport, bounded lifecycle"] diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 7bdf399598..f3eda9698c 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -74,6 +74,18 @@ for the tool-result contract. ## Large results +Application analysis projects semantic relationships one source file at a time. +After projection, it retains only the lexical module facts and exported return +shapes needed to build application relationships. Full source IR does not stay +resident for every file together. Callable ownership and contained-reference +queries use per-file range indexes, while call sites use exact-range lookups. +Existing node budgets and coverage reporting remain in effect. + +Progress identifies the source file being parsed and projected. Cancellation is +checked between files, with event-loop yields that allow notifications and +completed file-local allocations to be released. A single file's synchronous +parser and semantic extraction still need to finish before that boundary. + Graph and Evidence identifiers hash canonical JSON incrementally, without assembling a single string for the whole graph. The canonical bytes and existing identifiers remain unchanged. The opt-in regression check is @@ -93,6 +105,9 @@ assemble whole strings. Field selection remains useful when the caller needs a smaller view, for example `--format json --filter-output evidence_id,normalized_result.statistics`. Streaming output does not bound the memory needed to construct the analysis graph itself. +The separate large-response transport work is tracked in +[#1053](https://github.com/morluto/rea/issues/1053) and +[#1050](https://github.com/morluto/rea/issues/1050). Client framing limits also apply: the pinned Node MCP SDK's stdio transport defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate the complete response, including text and structured Evidence projections. diff --git a/docs/product-catalog.json b/docs/product-catalog.json index 96e9f7cb86..bd5fb4168a 100644 --- a/docs/product-catalog.json +++ b/docs/product-catalog.json @@ -7,10 +7,10 @@ "client": "2.3.1", "core": "2.3.1" }, - "skill_version": "32" + "skill_version": "33" }, "tools": { - "total": 138, + "total": 139, "families": [ { "id": "direct", @@ -84,13 +84,14 @@ { "id": "native", "surface": "native-provider", - "count": 9, + "count": 10, "tools": [ "capture_native_ui_scenario", "demangle_swift", "inspect_binary_layout", "inspect_macho", "inspect_plist", + "inspect_recorded_crash", "inspect_signature", "list_architectures", "observe_native_calls", @@ -385,7 +386,7 @@ "id": "pwntools-elf", "name": "REA pwntools ELF adapter", "version": "pwntools@4.15.0;pyelftools@0.33;unicorn@2.1.2", - "capabilities": ["inspect_binary_layout"] + "capabilities": ["inspect_binary_layout", "inspect_recorded_crash"] }, { "id": "rea-android-application", @@ -601,7 +602,7 @@ } ], "cli": { - "primary_count": 93, + "primary_count": 94, "commands": [ "address-to-file-offset", "analyze", @@ -658,6 +659,7 @@ "inspect-native-instruction", "inspect-native-load-image", "inspect-plist", + "inspect-recorded-crash", "inspect-signature", "inspect-web-event-listeners", "inspect-web-network-capture", @@ -706,15 +708,15 @@ }, "runtime_catalog": { "counts": { - "cli_commands": 93, - "mcp_tools": 138, + "cli_commands": 94, + "mcp_tools": 139, "mcp_prompts": 6 }, "digests": { - "tools_sha256": "38ae770dfb525835175a1e0f931c05c371967cdc814c75b085e865d7db0be23a", + "tools_sha256": "435eebda30d4ca4e15a4830bc18404c0f074ca3fed5473a43aeebd9eca42333b", "prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d", - "combined_sha256": "876d360007323512c42a25736a1ce0524246a9b87aad27564c2b275bc1c34133", - "providers_sha256": "44935c127fd040af492f2da591dfcd782d766770d306b91242216674220047ed" + "combined_sha256": "65ffda040e33bc97166231d7b9c4a43a9732fca8a474ee39cae7b69aee4485b4", + "providers_sha256": "954b499c76514890062d2b6c936d73cb3a9c45412d522a081125a325e81cdec5" } } } diff --git a/docs/recorded-crashes.md b/docs/recorded-crashes.md new file mode 100644 index 0000000000..d68db74fd8 --- /dev/null +++ b/docs/recorded-crashes.md @@ -0,0 +1,88 @@ +# Recorded crash evidence + +`inspect_recorded_crash` / `rea inspect-recorded-crash` inspect a caller-selected +Linux ELF64 x86-64 little-endian core independently of an active binary target. + +```sh +export REA_PWNTOOLS_PYTHON=/absolute/existing/venv/bin/python +rea inspect-recorded-crash ./crash.core --json +``` + +The selected Python needs unchanged pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2. +REA does not install these dependencies. ELFFile/its original ELF structures and +pwntools' original Linux amd64 ctypes provide parsing and ABI layout. The adapter +adds bounds, representation preservation and lifecycle ownership. +The initial Linux note profile accepts System V/unspecified and GNU/Linux OSABI +with ABI version zero. Other declared ABIs return `unsupported_target` before +note interpretation; an unspecified OSABI alone does not prove OS origin. + +## Recorded observations + +- Artifact path, original SHA-256 and size; full normalized result and Evidence + are inline through CLI and MCP. +- Original program headers and recorded segment file/memory ranges. Undumped + memory and current executable/library identity remain unknown. +- Every note's original identity, owner/descriptor bytes and source ranges, + including unfamiliar owners and note kinds. +- Every exact Linux CORE PRSTATUS note as a separate recorded thread, with + historical PID, signal and lossless uint64 registers bound to source bytes. + Each supported note has exactly one interpretation; each thread contains the + complete 27-register ABI set, with names bound to their original ABI offsets. + Physical note ranges cannot overlap or leave segment gaps. ELF and note headers + are independently bound to the snapshot, including extended program counts. +- Signed SIGINFO number/code/errno. Initially only SIGSEGV codes 1/2 establish the + fault-address union; other variants retain raw bytes and unknown meaning. + +Historical PIDs are metadata. They never select a live process or authorize +attach/control. Missing notes do not establish absence at the crash; note +interpretation completeness and signal-thread association remain unknown. + +## Optional debugger mapping context + +```sh +export REA_PWNDBG_GDB=/absolute/existing/gdb +export REA_PWNDBG_GDBINIT=/absolute/unchanged/pwndbg/gdbinit.py +export REA_PWNDBG_VENV_PATH=/absolute/existing/pwndbg-venv +rea inspect-recorded-crash ./crash.core --debugger-context --json +``` + +MCP uses `{ "path": "/artifacts/crash.core", "include_debugger_context": true }`. +Omitting this meaningful facet requests only recorded evidence. Requested but +unavailable context returns an actionable failure; it never silently claims +debugger coverage. GDB defaults to `/usr/bin/gdb`; the plugin and virtual +environment require explicit absolute paths. Initial local verification covers +Linux x64, GDB 13.1 and unchanged pwndbg 2026.09.15. + +The outer provider owns both processes and one stable core snapshot. An owned +Python launcher lowers inherited soft limits before replacing itself with GDB. +GDB starts without initfiles, auto-load, debuginfod or automatic shared-library +loading. Owned empty library search paths and caches are used, with plugin +updates disabled. Fixed package code verifies a core connection and no paired +executable before querying mapping candidates. No caller GDB command language is +exposed. The exact observed GDB/plugin version strings are retained. + +Mapping names are producer display metadata, with unknown current file identity. +Map permissions are derived from reported flags; zero/unfamiliar flags leave +permissions unknown. These maps do not establish page contents or an executable +match. Core source bytes remain available independently of this enrichment. +An Evidence envelope containing these maps is `derived`; inspection without +debugger context is `observed`. + +## Bounds and verification + +Initial Linux x64 limits: 32 MiB input, 64 MiB decoder reply, 1 MiB diagnostics per +stage, 30 seconds owned wall deadline per stage, 3 GiB address-space soft ceiling +and 30 seconds CPU soft ceiling. Tighter inherited soft/hard limits are retained; +hard limits are not raised. Complete oversized results fail explicitly. These +process limits do not constitute a sandbox for caller-supplied engine code. + +Resource exit statuses need matching private failure-branch markers. Unmarked +launcher failures retain their actual status/output and unknown resource cause. +Cancellation, output truncation and independent owned cleanup remain explicit. + +`npm run verify:recorded:crash` uses source-owned fixtures and caller-supplied +GDB/Python/pwndbg/strace. Fixture generation explicitly launches its owned test +program; subsequent inspection only reads the resulting core. Real acceptance +checks recorded threads, high registers, missing/malformed notes, opaque owners, +CLI/MCP parity, optional debugger context and historical PID collision with an +owned live sentinel. Original inputs and host configuration remain unchanged. diff --git a/docs/testing.md b/docs/testing.md index a193736e9a..3e12ff4d04 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -661,6 +661,28 @@ unknowns, malformed carriers and independent cleanup. An optional positional entrypoint verifies a fresh installed package. It acquires no engine, compiler or chain dependency and does not execute a contract on a chain. +### Recorded crash evidence + +`npm run verify:recorded:crash` is a separate Linux x64 lane. It requires GCC, +GDB, absolute `REA_PWNTOOLS_PYTHON` with the offline ELF profile above, +`REA_PWNDBG_GDBINIT` and `REA_PWNDBG_VENV_PATH` with unchanged pwndbg 2026.09.15, +and `REA_VERIFY_STRACE_COMMAND`. Its disposable CI runner installs GDB, checks out the exact upstream +commit and installs its frozen lockfile in isolated runner storage. No developer +host configuration or core-pattern setting changes. + +Fixture generation explicitly runs an owned source-built two-thread program +under GDB to create a recording. Subsequent public CLI/MCP inspection verifies +lossless high registers, signed signals, note source bytes, malformed/missing +notes, unfamiliar owners, optional core-only mapping context and actionable +missing/unsupported plugin errors. A historical-PID collision fixture references +an owned live sentinel; inspection syscall traces reject process attach/memory +access, provider lookups of that PID's `/proc` files and attempted Internet sockets. Traces admit +the observed upstream startup helpers (`iconv -l`, the selected checkout's Git +version lookup) and REA ownership inspection separately from target execution. +This is fixture evidence, not a sandbox claim. Inputs remain unchanged and the +sentinel must stay alive; owned cleanup and empty verifier descendants are required. Pass an +installed package entrypoint as the script's first argument for package coverage. + ## Agent evaluation and conformance records Evaluate native, JavaScript, managed and browser investigation tasks through a diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index 1746247386..53a4777f4b 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,5 +1,5 @@ { - "manifest_id": "ecm_ade478810523ea3415006518b9af2b728996cfa1f437653759f803509ef68e10", + "manifest_id": "ecm_f04aa9af367df047b8a17609e61fb7f9058f4351f389d7e4fd9d8ca9520c4a63", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -13,7 +13,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "a099ce1ed2d19d44f066fda671e4ee627d1c1681376b589bab731daa7e6121b6" + "sha256": "a02e22bf4d93ee7d57d9c4b4712f0b1353ea1458ed48e870a1f2cfad7ea98d04" } ], "claims": [ diff --git a/package.json b/package.json index 7145960bb7..227747e24d 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,15 @@ "bridge/pwntools/layout_impl.py", "third_party/pwntools/README.md", "third_party/evmole/README.md", - "third_party/evmole/LICENSE" + "third_party/evmole/LICENSE", + "bridge/pwntools/decoder_runtime.py", + "bridge/pwntools/recorded_crash.py", + "bridge/pwntools/recorded_crash_impl.py", + "bridge/pwndbg/launch.py", + "bridge/pwndbg/launch_impl.py", + "bridge/pwndbg/core_context.py", + "third_party/pwndbg/README.md", + "third_party/pwndbg/LICENSE.md" ], "scripts": { "deps:check": "node scripts/check-dependency-install.mjs", @@ -203,7 +211,8 @@ "verify:browser:runtime": "npm run build:cached && node scripts/verify-browser-runtime.mjs", "verify:web:network-captures": "npm run build:cached && node scripts/verify-web-network-captures.mjs", "verify:binary:layout": "npm run build:cached && node scripts/verify-binary-layout.mjs", - "verify:evm:interface": "npm run build:cached && node scripts/verify-evm-interface.mjs" + "verify:evm:interface": "npm run build:cached && node scripts/verify-evm-interface.mjs", + "verify:recorded:crash": "npm run build:cached && node scripts/verify-recorded-crash.mjs" }, "lint-staged": { "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}": [ diff --git a/scripts/fixtures/recorded-crash.c b/scripts/fixtures/recorded-crash.c new file mode 100644 index 0000000000..a7baa7da1a --- /dev/null +++ b/scripts/fixtures/recorded-crash.c @@ -0,0 +1,24 @@ +#include +#include +#include +#include +static atomic_int ready; +static void *recorded_thread(void *unused) { + (void)unused; + __asm__ volatile("movabs $0xfedcba9876543210, %%r12" ::: "r12"); + atomic_store(&ready, 1); + for (;;) pause(); + return 0; +} +__attribute__((noinline)) static void fixture_crash(uint64_t value) { + volatile uint64_t *pointer = (volatile uint64_t *)(uintptr_t)0x123; + *pointer = value; +} +int main(void) { + pthread_t thread; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 65536) || pthread_create(&thread, &attr, recorded_thread, 0)) return 1; + while (!atomic_load(&ready)) usleep(1000); + fixture_crash(UINT64_C(0x1122334455667788)); + return 0; +} diff --git a/scripts/lib/recorded-core-fixtures.py b/scripts/lib/recorded-core-fixtures.py new file mode 100644 index 0000000000..cc8d654439 --- /dev/null +++ b/scripts/lib/recorded-core-fixtures.py @@ -0,0 +1,94 @@ +"""Source-owned mutations built with the same unchanged upstream ELF/ABI structures.""" +import ctypes +import io +import json +from pathlib import Path +import sys +from elftools.elf.elffile import ELFFile +from elftools.construct import Container +from pwnlib.elf.datatypes import elf_prstatus_amd64 + +source, directory, sentinel = Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3]) +original = source.read_bytes() +image = ELFFile(io.BytesIO(original)) +segments = list(image.iter_segments()) +notes = [note for segment in segments if segment.header.p_type == "PT_NOTE" for note in segment.iter_notes()] +status = next(note for note in notes if note.n_name == "CORE" and note.n_type == "NT_PRSTATUS") +descriptor_start = lambda note: note.n_offset + image.structs.Elf_Nhdr.sizeof() + ((note.n_namesz + 3) // 4) * 4 +files = {} + + +def save(name, content): + path = directory / (name + ".core") + path.write_bytes(content) + files[name] = str(path) + + +def note_header(content, note, **changes): + header = dict(note) + header.update(changes) + encoded = image.structs.Elf_Nhdr.build(Container(**header)) + content[note.n_offset:note.n_offset + len(encoded)] = encoded + + +save("truncated-header", original[:32]) +for name, osabi, version in [("linux-osabi", 3, 0), ("solaris-osabi", 6, 0), ("freebsd-osabi", 9, 0), ("unsupported-abi-version", 0, 1)]: + content = bytearray(original) + content[7:9] = bytes([osabi, version]) + save(name, content) +content = bytearray(original) +section = Container(sh_name=0, sh_type="SHT_NULL", sh_flags=0, sh_addr=0, sh_offset=0, sh_size=0, sh_link=0, sh_info=len(segments), sh_addralign=0, sh_entsize=0) +content.extend(image.structs.Elf_Shdr.build(section)) +header = Container(**dict(image.header, e_phnum=0xffff, e_shoff=len(original), e_shentsize=image.structs.Elf_Shdr.sizeof(), e_shnum=1, e_shstrndx=0)) +content[:image.structs.Elf_Ehdr.sizeof()] = image.structs.Elf_Ehdr.build(header) +save("extended-program-count", content) +for index, segment in enumerate(segments): + if segment.header.p_type != "PT_NOTE": + continue + content = bytearray(original) + raw_notes = original[segment.header.p_offset:segment.header.p_offset + segment.header.p_filesz] + content.extend(raw_notes + b"\0\0\0") + header = Container(**dict(segment.header, p_offset=len(original), p_filesz=len(raw_notes) + 3)) + start = image.header.e_phoff + index * image.header.e_phentsize + content[start:start + image.structs.Elf_Phdr.sizeof()] = image.structs.Elf_Phdr.build(header) + save("zero-note-tail", content) + content[-1] = 1 + save("nonzero-note-tail", content) + break +content = bytearray(original) +note_header(content, status, n_descsz=1) +save("truncated-status", content) +content = bytearray(original) +note_header(content, status, n_namesz=len(original)) +save("truncated-owner", content) +content = bytearray(original) +content[status.n_offset + image.structs.Elf_Nhdr.sizeof()] = 255 +save("opaque-owner", content) +content = bytearray(original) +for note in notes: + if note.n_type == "NT_SIGINFO": + note_header(content, note, n_type=0x7ffeff) +save("unknown-signals", content) +content = bytearray(original) +for index, segment in enumerate(segments): + if segment.header.p_type == "PT_NOTE": + header = dict(segment.header, p_type="PT_NULL") + encoded = image.structs.Elf_Phdr.build(Container(**header)) + start = image.header.e_phoff + index * image.header.e_phentsize + content[start:start + len(encoded)] = encoded +save("missing-notes", content) +content = bytearray(original) +start = descriptor_start(status) + elf_prstatus_amd64.pr_pid.offset +content[start:start + ctypes.sizeof(ctypes.c_uint32)] = (0xfffffffe).to_bytes(4, "little") +save("negative-pid", content) +content = bytearray(original) +content[start:start + ctypes.sizeof(ctypes.c_uint32)] = sentinel.to_bytes(4, "little") +for note in notes: + if note.n_name == "CORE" and note.n_type == "NT_PRPSINFO": + # Build with unchanged pyelftools; no hard-coded PID offset or host file resolution. + raw = dict(note.n_desc, pr_pid=sentinel) + encoded = image.structs.Elf_Prpsinfo.build(Container(**raw)) + offset = descriptor_start(note) + content[offset:offset + len(encoded)] = encoded +save("historical-pid-collision", content) +print(json.dumps(files)) diff --git a/scripts/verify-binary-layout.mjs b/scripts/verify-binary-layout.mjs index 8c0c886856..8b83d2c293 100644 --- a/scripts/verify-binary-layout.mjs +++ b/scripts/verify-binary-layout.mjs @@ -1032,8 +1032,8 @@ try { "invalid_input", ], ["undersized-program-header", undersizedProgramHeader, "invalid_input"], - ["arm64", unsupported, "unsupported_provider"], - ["core", core, "unsupported_provider"], + ["arm64", unsupported, "unsupported_target"], + ["core", core, "unsupported_target"], ]) { const path = join(root.path, name); await writeFile(path, bytes); diff --git a/scripts/verify-recorded-crash.mjs b/scripts/verify-recorded-crash.mjs new file mode 100644 index 0000000000..eb59749c73 --- /dev/null +++ b/scripts/verify-recorded-crash.mjs @@ -0,0 +1,318 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { writeFile, readFile, copyFile } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { verifyCompleteToolCatalog } from "./lib/verify-package-core.mjs"; +import { connectRecordedCrash } from "./verify/recorded-crash/public.mjs"; +import { + createRecordedCore, + fixtureCommand, + startCoreSentinel, +} from "./verify/recorded-crash/fixtures.mjs"; +import { verifyCoreInspectionTrace } from "./verify/recorded-crash/trace.mjs"; + +const environment = Object.fromEntries( + Object.entries(process.env).filter(([, value]) => typeof value === "string"), +); +environment.REA_PWNDBG_GDB ??= "/usr/bin/gdb"; +for (const key of [ + "REA_PWNTOOLS_PYTHON", + "REA_PWNDBG_GDB", + "REA_PWNDBG_GDBINIT", + "REA_PWNDBG_VENV_PATH", + "REA_VERIFY_STRACE_COMMAND", +]) + if (!isAbsolute(environment[key] ?? "")) + throw new Error( + `verify:recorded:crash requires absolute ${key}; see docs/recorded-crashes.md. No dependencies are installed by this verifier.`, + ); +if (process.platform !== "linux" || process.arch !== "x64") + throw new Error("verify:recorded:crash requires Linux x64."); +const run = createVerifierRun(); +environment.REA_PROCESS_RUN_ID = run.run_id; +const root = await PrivateRuntimeRoot.create({ prefix: "rea-core-verifier-" }); +const options = { root: root.path, environment, runId: run.run_id }; +const entrypoint = + process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const failures = []; +const sessions = []; +let sentinel; +let cases = 0; +let trace; +let catalogTools; +try { + for (const [command, args] of [ + ["gcc", ["--version"]], + [environment.REA_PWNDBG_GDB, ["--version"]], + [environment.REA_VERIFY_STRACE_COMMAND, ["-V"]], + ]) { + try { + await fixtureCommand(command, args, options); + } catch (cause) { + throw new Error( + `verify:recorded:crash prerequisite unavailable: ${command}`, + { cause }, + ); + } + } + const core = await createRecordedCore(options); + sentinel = await startCoreSentinel(options); + sentinel.assertAlive(); + const fixtures = JSON.parse( + await fixtureCommand( + environment.REA_PWNTOOLS_PYTHON, + [ + "-I", + fileURLToPath( + new URL("./lib/recorded-core-fixtures.py", import.meta.url), + ), + core, + root.path, + String(sentinel.pid), + ], + options, + ), + ); + const spaced = join(root.path, "recording with spaces.core"); + await copyFile(core, spaced); + const basic = await connectRecordedCrash({ entrypoint, environment }); + sessions.push(basic); + const advertised = (await basic.client.listTools()).tools; + assert.equal( + advertised.filter((tool) => tool.name === "inspect_recorded_crash").length, + 1, + ); + const catalog = await verifyCompleteToolCatalog(basic.client, { + timeout: 90_000, + }); + for (const name of [ + "inspect_binary_layout", + "inspect_recorded_crash", + "inspect_evm_interface", + ]) + assert.ok(catalog.includes(name)); + catalogTools = catalog.length; + for (const mode of ["cli", "mcp"]) { + const report = await basic.inspect(mode, spaced); + assert.equal(report.threads.length, 2); + assert.ok( + report.threads.some((thread) => + thread.registers.some( + (register) => + register.name === "rdi" && register.value === "0x1122334455667788", + ), + ), + ); + assert.ok( + report.threads.some((thread) => + thread.registers.some( + (register) => + register.name === "r12" && register.value === "0xfedcba9876543210", + ), + ), + ); + assert.ok( + report.signals.some( + (signal) => + signal.number === 11 && + signal.code === 1 && + signal.fault_address === "0x123", + ), + ); + assert.ok( + report.signals.some( + (signal) => signal.code === -6 && signal.fault_address === null, + ), + ); + cases++; + const enriched = await basic.inspect(mode, spaced, { + debuggerContext: true, + }); + assert.deepEqual(enriched.threads, report.threads); + assert.deepEqual(enriched.notes, report.notes); + assert.ok(enriched.debugger.pwndbg_version.startsWith("2026.09.15")); + assert.equal(enriched.debugger.connection, "core"); + assert.equal(enriched.debugger.executable, null); + assert.ok(enriched.debugger.maps.length > 0); + assert.ok( + enriched.debugger.maps.every( + (map) => map.reported_flags !== 0 || map.permissions === null, + ), + ); + assert.ok( + enriched.debugger.maps.every( + (map) => map.current_file_identity === "unknown", + ), + ); + assert.ok( + enriched.debugger.reported_limits.address_space_bytes <= 3 * 1024 ** 3, + ); + assert.ok(enriched.debugger.reported_limits.cpu_seconds <= 30); + assert.ok( + enriched.debugger.reported_limits.file_size_bytes <= 64 * 1024 ** 2, + ); + assert.ok(enriched.diagnostics.stdout.includes("[core decoder]")); + assert.equal(enriched.decoder_diagnostics.truncated, false); + cases++; + for (const name of [ + "truncated-header", + "truncated-status", + "truncated-owner", + "nonzero-note-tail", + ]) { + await basic.inspect(mode, fixtures[name], { category: "invalid_input" }); + cases++; + } + const opaque = await basic.inspect(mode, fixtures["opaque-owner"]); + assert.ok( + opaque.notes.some( + (note) => Buffer.from(note.owner_bytes_base64, "base64")[0] === 255, + ), + ); + assert.equal(opaque.threads.length, 1); + cases++; + const noSignals = await basic.inspect(mode, fixtures["unknown-signals"]); + assert.equal(noSignals.signals.length, 0); + assert.equal(noSignals.threads.length, 2); + cases++; + const noNotes = await basic.inspect(mode, fixtures["missing-notes"]); + assert.equal(noNotes.notes.length, 0); + assert.equal(noNotes.threads.length, 0); + assert.equal(noNotes.note_interpretation_completeness, "unknown"); + cases++; + const negative = await basic.inspect(mode, fixtures["negative-pid"]); + assert.ok(negative.threads.some((thread) => thread.historical_pid === -2)); + cases++; + const linux = await basic.inspect(mode, fixtures["linux-osabi"]); + assert.deepEqual(linux.threads, report.threads); + assert.deepEqual(linux.signals, report.signals); + cases++; + const extended = await basic.inspect( + mode, + fixtures["extended-program-count"], + ); + assert.deepEqual(extended.threads, report.threads); + assert.deepEqual(extended.signals, report.signals); + cases++; + const padded = await basic.inspect(mode, fixtures["zero-note-tail"]); + assert.equal(padded.threads.length, 2); + assert.ok( + padded.note_padding.some((padding) => padding.bytes_base64 === "AAAA"), + ); + cases++; + for (const name of [ + "solaris-osabi", + "freebsd-osabi", + "unsupported-abi-version", + ]) { + await basic.inspect(mode, fixtures[name], { + category: "unsupported_target", + }); + cases++; + } + const collision = await basic.inspect( + mode, + fixtures["historical-pid-collision"], + { debuggerContext: true }, + ); + assert.ok( + collision.threads.some( + (thread) => thread.historical_pid === sentinel.pid, + ), + ); + sentinel.assertAlive(); + cases++; + } + for (const [name, changes] of [ + ["missing", { REA_PWNDBG_GDBINIT: join(root.path, "missing-plugin.py") }], + [ + "wrong-profile", + { REA_PWNDBG_GDBINIT: join(root.path, "wrong-profile.py") }, + ], + ]) { + if (name === "wrong-profile") { + // Source-owned failure injection changes only the in-memory reported version. + // The selected upstream checkout and its installed files remain unchanged. + await writeFile( + changes.REA_PWNDBG_GDBINIT, + `from pathlib import Path\nentry = ${JSON.stringify(environment.REA_PWNDBG_GDBINIT)}\nexec(compile(Path(entry).read_bytes(), entry, 'exec'), {'__name__': '__main__', '__file__': entry})\nimport pwndbg.lib.version\npwndbg.lib.version.__version__ = '0.0.0-source-owned'\n`, + ); + } + const session = await connectRecordedCrash({ + entrypoint, + environment: { ...environment, ...changes }, + }); + sessions.push(session); + for (const mode of ["cli", "mcp"]) { + const error = await session.inspect(mode, core, { + debuggerContext: true, + category: "unsupported_provider", + }); + assert.ok( + JSON.stringify(error).includes( + name === "missing" ? "REA_PWNDBG_GDBINIT" : "source-owned", + ), + ); + cases++; + } + } + const executableCarrier = join(root.path, "executable-as-core"); + const executableBytes = Buffer.from(await readFile(core)); + executableBytes.writeUInt16LE(2, 16); + await writeFile(executableCarrier, executableBytes); + for (const mode of ["cli", "mcp"]) { + const error = await basic.inspect(mode, executableCarrier, { + category: "unsupported_target", + }); + assert.ok(JSON.stringify(error).includes("ET_CORE")); + cases++; + } + trace = await verifyCoreInspectionTrace({ + ...options, + entrypoint, + core: fixtures["historical-pid-collision"], + sentinel, + }); + cases += trace.length; + assert.ok((await readFile(core)).length > 0); +} catch (cause) { + failures.push(cause); +} finally { + for (const close of [ + ...sessions.map((session) => () => session.close()), + ...(sentinel ? [() => sentinel.close()] : []), + () => root.close(), + ]) { + try { + await close(); + } catch (cause) { + failures.push(cause); + } + } +} +const verifier = await completeVerifierRun(run); +try { + assert.equal(verifier.process_lineage.status, "verified"); + assert.deepEqual(verifier.process_lineage.descendants, []); +} catch (cause) { + failures.push(cause); +} +const result = { + status: failures.length === 0 ? "passed" : "failed", + public_cases: cases, + catalog_tools: catalogTools, + profile: + "Linux x64; pwntools4.15.0/pyelftools0.33/Unicorn2.1.2; core-only BYO GDB/pwndbg2026.09.15", + fixture_generation: "owned test program executed", + inspection_trace: trace, + verifier, +}; +console.log(JSON.stringify(result, null, 2)); +if (failures.length > 0) + throw new AggregateError( + failures, + "Recorded crash verification failed; cleanup failures are retained.", + ); diff --git a/scripts/verify/recorded-crash/fixtures.mjs b/scripts/verify/recorded-crash/fixtures.mjs new file mode 100644 index 0000000000..2ad46803f6 --- /dev/null +++ b/scripts/verify/recorded-crash/fixtures.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { runOwnedCommand } from "../../../dist/process/OwnedCommand.js"; +import { + spawnOwnedProviderProcess, + ProviderProcessSupervisor, +} from "../../../dist/process/ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "../../../dist/process/ProcessOwnership.js"; + +/** Execute fixture tooling under an owned deadline; fixture generation deliberately runs its own test program. */ +export async function fixtureCommand( + command, + arguments_, + { root, environment, runId }, +) { + const result = await runOwnedCommand( + { + command, + arguments: arguments_, + cwd: root, + runId, + hostEnvironment: environment, + expectedCommand: null, + }, + { timeoutMs: 45_000, diagnosticBytes: 1024 * 1024 }, + ); + return result.stdout.text; +} + +/** Build and record a source-owned two-thread crash without changing kernel core settings. */ +export async function createRecordedCore(options) { + const executable = join(options.root, "fixture-program"); + const core = join(options.root, "recording.core"); + await fixtureCommand( + "gcc", + [ + "-g", + "-O0", + "-pthread", + "-fno-pie", + "-no-pie", + fileURLToPath( + new URL("../../fixtures/recorded-crash.c", import.meta.url), + ), + "-o", + executable, + ], + options, + ); + // Fixed relative arguments avoid putting selected filesystem paths in GDB command syntax. + await fixtureCommand( + options.environment.REA_PWNDBG_GDB, + [ + "-nx", + "-nh", + "--batch", + "--quiet", + "-iex", + "set auto-load off", + "-iex", + "set debuginfod enabled off", + "-ex", + "set startup-with-shell off", + "-ex", + "set disable-randomization off", + "-ex", + "file ./fixture-program", + "-ex", + "run", + "-ex", + "generate-core-file ./recording.core", + ], + options, + ); + return core; +} + +/** Retain authority over an unrelated live process used only as a historical-PID collision oracle. */ +export async function startCoreSentinel(options) { + const launch = await spawnOwnedProviderProcess({ + command: process.execPath, + arguments: [ + "-e", + "process.stdout.write('ready\\n'); setInterval(() => {}, 1000)", + ], + cwd: options.root, + hostEnvironment: options.environment, + runId: options.runId, + }); + const supervisor = new ProviderProcessSupervisor({ + ...launch, + ownsProcessLifetime: true, + cleanup: () => cleanupOwnedProcessGroup(launch.ownership), + }); + try { + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => finish(new Error("Sentinel readiness timed out")), + 10_000, + ); + const onData = () => { + if (supervisor.snapshot().stdout.text.includes("ready\n")) finish(); + }; + const onExit = () => + finish(new Error("Sentinel exited before readiness")); + function finish(error) { + clearTimeout(timer); + launch.process.stdout.off("data", onData); + launch.process.off("exit", onExit); + if (error) reject(error); + else resolve(); + } + launch.process.stdout.on("data", onData); + launch.process.once("exit", onExit); + onData(); + }); + } catch (cause) { + try { + await supervisor.stop(); + } finally { + supervisor.dispose(); + } + throw cause; + } + return { + pid: launch.process.pid, + assertAlive() { + assert.equal(launch.process.exitCode, null); + assert.equal(launch.process.signalCode, null); + process.kill(launch.process.pid, 0); + }, + async close() { + try { + const stopped = await supervisor.stop(); + assert.notEqual(stopped.status, "incomplete", JSON.stringify(stopped)); + } finally { + supervisor.dispose(); + } + }, + }; +} diff --git a/scripts/verify/recorded-crash/public.mjs b/scripts/verify/recorded-crash/public.mjs new file mode 100644 index 0000000000..144845df7c --- /dev/null +++ b/scripts/verify/recorded-crash/public.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseEvidence } from "../../../dist/domain/evidence.js"; +import { recordedCrashSchema } from "../../../dist/domain/native/recordedCrash.js"; +import { mcpTextValue } from "../../lib/mcp-verifier-results.mjs"; + +const execute = promisify(execFile); + +/** Exercise the public installed CLI and SDK transport with the same selected environment. */ +export async function connectRecordedCrash({ entrypoint, environment }) { + const client = new Client({ name: "recorded-crash-verifier", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: environment, + stderr: "pipe", + }); + try { + await client.connect(transport); + } catch (cause) { + await transport.close(); + throw cause; + } + return { + client, + async inspect(mode, path, { debuggerContext = false, category } = {}) { + const original = await readFile(path); + let evidence; + if (mode === "mcp") { + const response = await client.callTool( + { + name: "inspect_recorded_crash", + arguments: { path, include_debugger_context: debuggerContext }, + }, + { timeout: 90_000 }, + ); + const value = JSON.parse(mcpTextValue(response)); + if (category !== undefined) { + assert.equal(response.isError, true); + assert.equal(value.error.category, category, JSON.stringify(value)); + assert.deepEqual(await readFile(path), original); + return value.error; + } + assert.notEqual(response.isError, true, JSON.stringify(value)); + evidence = parseEvidence(value.evidence); + assert.deepEqual(value.result, evidence.normalized_result); + } else { + assert.equal(mode, "cli"); + let response; + try { + response = await execute( + process.execPath, + [ + entrypoint, + "inspect-recorded-crash", + path, + ...(debuggerContext ? ["--debugger-context"] : []), + "--json", + ], + { env: environment, timeout: 90_000, maxBuffer: 32 * 1024 * 1024 }, + ); + } catch (cause) { + if (category === undefined || typeof cause.code !== "number") + throw cause; + const error = JSON.parse(cause.stdout); + assert.equal(error.category, category, JSON.stringify(error)); + assert.deepEqual(await readFile(path), original); + return error; + } + assert.equal(category, undefined, "Expected an explicit failure"); + evidence = parseEvidence(JSON.parse(response.stdout)); + } + const report = recordedCrashSchema.parse(evidence.normalized_result); + assert.equal( + evidence.confidence, + debuggerContext ? "derived" : "observed", + ); + assert.deepEqual(evidence.raw_result, report); + assert.equal(evidence.subject.local_path, path); + assert.equal(report.artifact.path, path); + assert.equal( + report.artifact.sha256, + createHash("sha256").update(original).digest("hex"), + ); + assert.equal(report.artifact.bytes, original.length); + assert.equal( + evidence.provider.version, + "pwntools@4.15.0;pyelftools@0.33;unicorn@2.1.2", + ); + assert.equal(report.target_execution, "not-performed"); + assert.equal(report.live_process_identity, "unknown"); + assert.equal(report.diagnostics.truncated, false); + assert.equal( + report.debugger.status, + debuggerContext ? "available" : "not-requested", + ); + for (const note of report.notes) { + for (const [encoded, range] of [ + [note.owner_bytes_base64, note.owner_location], + [note.descriptor_bytes_base64, note.descriptor_location], + ]) { + const start = Number(BigInt(range.offset)); + assert.deepEqual( + Buffer.from(encoded, "base64"), + original.subarray(start, start + Number(BigInt(range.bytes))), + ); + } + } + for (const thread of report.threads) { + assert.equal(thread.registers.length, 27); + for (const register of thread.registers) + assert.equal( + BigInt(register.value), + original.readBigUInt64LE(Number(BigInt(register.location.offset))), + ); + } + assert.deepEqual(await readFile(path), original); + return report; + }, + async close() { + try { + await client.close(); + } finally { + await transport.close(); + } + }, + }; +} diff --git a/scripts/verify/recorded-crash/trace-client.mjs b/scripts/verify/recorded-crash/trace-client.mjs new file mode 100644 index 0000000000..f38f1d6cff --- /dev/null +++ b/scripts/verify/recorded-crash/trace-client.mjs @@ -0,0 +1,11 @@ +import { connectRecordedCrash } from "./public.mjs"; + +const session = await connectRecordedCrash({ + entrypoint: process.argv[2], + environment: process.env, +}); +try { + await session.inspect("mcp", process.argv[3], { debuggerContext: true }); +} finally { + await session.close(); +} diff --git a/scripts/verify/recorded-crash/trace.mjs b/scripts/verify/recorded-crash/trace.mjs new file mode 100644 index 0000000000..abe3cecb73 --- /dev/null +++ b/scripts/verify/recorded-crash/trace.mjs @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import { readdir, readFile, realpath } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { isAbsolute, join, resolve } from "node:path"; +import { runOwnedCommand } from "../../../dist/process/OwnedCommand.js"; + +/** Check attempted syscalls, including failed attempts, on a recorded PID that collides with an owned live sentinel. */ +export async function verifyCoreInspectionTrace({ + root, + environment, + entrypoint, + core, + sentinel, + runId, +}) { + const gdb = await realpath(environment.REA_PWNDBG_GDB); + const python = await realpath(environment.REA_PWNTOOLS_PYTHON); + const plugin = resolve(environment.REA_PWNDBG_GDBINIT, ".."); + const helperPaths = (name) => + new Set([ + join(environment.REA_PWNDBG_VENV_PATH, "bin", name), + ...(environment.PATH ?? "") + .split(":") + .filter(isAbsolute) + .map((directory) => join(directory, name)), + ]); + const psPaths = helperPaths("ps"); + const iconvPaths = helperPaths("iconv"); + const gitPaths = helperPaths("git"); + const observations = []; + for (const mode of ["cli", "mcp"]) { + sentinel.assertAlive(); + const prefix = join(root, `inspection-${mode}.trace`); + const invocation = + mode === "cli" + ? [ + entrypoint, + "inspect-recorded-crash", + core, + "--debugger-context", + "--json", + ] + : [ + fileURLToPath(new URL("./trace-client.mjs", import.meta.url)), + entrypoint, + core, + ]; + await runOwnedCommand( + { + command: environment.REA_VERIFY_STRACE_COMMAND, + arguments: [ + "-ff", + "-qq", + "-s", + "8192", + "-e", + "trace=%process,%network,%file,ptrace,process_vm_readv,process_vm_writev", + "-o", + prefix, + process.execPath, + ...invocation, + ], + cwd: root, + hostEnvironment: environment, + expectedCommand: null, + runId, + }, + { timeoutMs: 90_000, diagnosticBytes: 1024 * 1024 }, + ); + const executions = []; + const ownershipReads = []; + let gdbObserved = false; + for (const file of await readdir(root)) { + if (!file.startsWith(`inspection-${mode}.trace.`)) continue; + const text = await readFile(join(root, file), "utf8"); + const ownershipTrace = text + .split("\n") + .some( + (line) => + /^execve\("\/(?:usr\/)?bin\/ps",/u.test(line) && + line.includes( + '["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]', + ) && + line.endsWith(" = 0"), + ); + const pidReads = text + .split("\n") + .filter((line) => line.includes(`/proc/${sentinel.pid}/`)); + if (ownershipTrace) { + for (const line of pidReads) { + assert.ok( + new RegExp( + `/proc/${sentinel.pid}/(?:stat|status|cmdline|wchan|environ)"`, + "u", + ).test(line), + `Unexpected ownership observation of the sentinel: ${line}`, + ); + ownershipReads.push(line); + } + } else + assert.deepEqual( + pidReads, + [], + `Historical PID caused a provider lookup in ${file}: ${pidReads.join("\n")}`, + ); + assert.ok( + !/\bptrace\(/u.test(text), + `Inspection attempted ptrace: ${file}`, + ); + assert.ok( + !/\bprocess_vm_(?:readv|writev)\(/u.test(text), + `Inspection attempted process memory access: ${file}`, + ); + assert.ok( + !/\b(?:socket|connect|sendto|sendmsg)\([^\n]*(?:AF_INET|AF_INET6)/u.test( + text, + ), + `Inspection attempted an Internet socket: ${file}`, + ); + for (const line of text.split("\n")) { + if (!/execve(?:at)?\(/u.test(line)) continue; + const match = /^execve\("([^"]+)"/u.exec(line); + assert.notEqual(match, null, `Unresolved executable identity: ${line}`); + assert.ok( + / = (?:0|-1 [A-Z]+)/u.test(line), + `Incomplete execution observation: ${line}`, + ); + const executable = await realpath(match[1]).catch(() => match[1]); + const ownership = + psPaths.has(match[1]) && + line.includes('["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]'); + const iconv = + iconvPaths.has(match[1]) && line.includes('["iconv", "-l"]'); + const version = + gitPaths.has(match[1]) && + line.includes( + "[" + + [ + "git", + "--git-dir", + join(plugin, ".git"), + "rev-parse", + "--short", + "HEAD", + ] + .map((argument) => JSON.stringify(argument)) + .join(", ") + + "]", + ); + assert.ok( + [process.execPath, python, gdb].includes(executable) || + ownership || + iconv || + version, + `Unexpected attempted host execution: ${line}`, + ); + if (executable === gdb) gdbObserved = true; + executions.push(executable); + } + } + assert.ok(gdbObserved, "Trace must include the actual core-only GDB stage"); + assert.ok(executions.includes(python)); + sentinel.assertAlive(); + observations.push({ + mode, + executions, + historical_pid: sentinel.pid, + sentinel: "still-alive", + ownership_process_table_reads: ownershipReads, + }); + } + return observations; +} diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md index 25690e7ba8..83c0822319 100644 --- a/skills/reverse-engineer-anything/SKILL.md +++ b/skills/reverse-engineer-anything/SKILL.md @@ -2,9 +2,9 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "32" - tool_count: 138 - catalog_digest: "876d360007323512c42a25736a1ce0524246a9b87aad27564c2b275bc1c34133" + version: "33" + tool_count: 139 + catalog_digest: "65ffda040e33bc97166231d7b9c4a43a9732fca8a474ee39cae7b69aee4485b4" --- # REA @@ -109,6 +109,11 @@ explicit path or endpoint and do not need it. caller-supplied pwntools without opening a disassembler database. Preserve its raw locations and inference/coverage limits; see the [offline binary guide](https://github.com/morluto/rea/blob/main/docs/binary-diagnostics.md). +- Supplied Linux x86-64 ELF core: `inspect_recorded_crash`. Read every recorded + thread, raw note and source-bound register without opening a live target. + Historical PIDs do not select live processes. Request `include_debugger_context` + only when core-only mapping candidates help; see the + [recorded crash guide](https://github.com/morluto/rea/blob/main/docs/recorded-crashes.md). - Native executable, library, or analysis database: `open_binary`, then use focused analysis tools directly; call `binary_overview` when metadata or inventory context is useful and available from the selected provider. diff --git a/skills/reverse-engineer-anything/references/native-and-artifacts.md b/skills/reverse-engineer-anything/references/native-and-artifacts.md index 97bd5b29ec..b0674f641d 100644 --- a/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -1,5 +1,21 @@ # Native, managed, and packaged artifacts +## Recorded Linux crashes + +Use `inspect_recorded_crash` with an explicit Linux x86-64 ELF core path to +inspect historical thread registers, signals, raw notes and original file +ranges. It is independent of the active disassembler target. Source-bound +register values and note bytes are observations; missing notes, signal-thread +association and current executable/library identity remain unknown. PIDs are +historical metadata and never authorize live attach or control. + +The optional `include_debugger_context` facet adds mapping candidates through +caller-supplied GDB/pwndbg in an owned core-only session. Display names do not +establish current file identity; zero reported flags leave permissions unknown. +Requested unavailable context fails with setup guidance. See +[recorded crashes](https://github.com/morluto/rea/blob/main/docs/recorded-crashes.md) +for exact upstream profiles, bounds and verification coverage. + ## Native targets After `open_binary`, use focused search, procedure, or function tools directly. diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index ea627693e6..b600de075b 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -35,6 +35,7 @@ export type AvailabilityPolicy = { readonly processCaptureEnabled: boolean; readonly optionalProviderLoadFailures?: OptionalProviderLoadFailures; readonly binaryLayoutEnabled?: boolean; + readonly recordedCrashEnabled?: boolean; readonly firmwareInspectionEnabled?: boolean; readonly firmwareExtractionEnabled?: boolean; readonly androidAnalysisEnabled?: boolean; @@ -298,6 +299,14 @@ const workflowAvailabilityFor = ({ remediation: "The bundled offline EVM interface profile is currently real-verified on Linux x64. No active binary target or chain endpoint is required.", }; + if (name === "inspect_recorded_crash") + return policy.recordedCrashEnabled === true + ? { reason: "available", remediation: null } + : { + reason: "provider_missing", + remediation: + "On Linux x64, set absolute REA_PWNTOOLS_PYTHON to caller-supplied Python with pwntools 4.15.0. No active binary target is required.", + }; if (kind === "firmware-provider") { const enabled = name === "inspect_firmware_regions" diff --git a/src/application/binaryDiagnostics/RecordedCrashPort.ts b/src/application/binaryDiagnostics/RecordedCrashPort.ts new file mode 100644 index 0000000000..3f9551c67d --- /dev/null +++ b/src/application/binaryDiagnostics/RecordedCrashPort.ts @@ -0,0 +1,19 @@ +import type { + ExecutionOptions, + ProviderIdentity, +} from "../AnalysisProvider.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { + InspectRecordedCrashInput, + RecordedCrash, +} from "../../domain/native/recordedCrash.js"; +import type { Result } from "../../domain/result.js"; + +/** Replaceable decoder of supplied recordings; historical PID is never live authority. */ +export interface RecordedCrashPort { + readonly identity: ProviderIdentity; + inspect( + input: InspectRecordedCrashInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/binaryDiagnostics/RecordedCrashService.test.ts b/src/application/binaryDiagnostics/RecordedCrashService.test.ts new file mode 100644 index 0000000000..7390f4f31b --- /dev/null +++ b/src/application/binaryDiagnostics/RecordedCrashService.test.ts @@ -0,0 +1,148 @@ +import { expect, it } from "vitest"; +import { RecordedCrashService } from "./RecordedCrashService.js"; +import { + recordedCrashFixture, + RECORDED_CRASH_TEST_PROVIDER, + recordedCrashDebuggerFixture, +} from "../../../tests/fixtures/binaryDiagnostics/recordedCrash.js"; +import { parseEvidence } from "../../domain/evidence.js"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { err, ok } from "../../domain/result.js"; + +it("binds original bytes, inline observations, unknowns and producing identity", async () => { + const fixture = recordedCrashFixture(); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(fixture)), + }); + const result = await service.inspect({ path: fixture.artifact.path }); + if (!result.ok) throw result.error; + const evidence = parseEvidence(result.value); + expect(evidence.normalized_result).toEqual(fixture); + expect(evidence.raw_result).toEqual(fixture); + expect(evidence.subject?.digest.sha256).toBe(fixture.artifact.sha256); + expect(evidence.provider).toEqual(RECORDED_CRASH_TEST_PROVIDER); + expect(evidence.confidence).toBe("observed"); +}); + +it("labels an envelope containing debugger-derived maps as derived", async () => { + const fixture = recordedCrashFixture(); + fixture.debugger = recordedCrashDebuggerFixture(); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(fixture)), + }); + const result = await service.inspect({ + path: fixture.artifact.path, + include_debugger_context: true, + }); + if (!result.ok) throw result.error; + const evidence = parseEvidence(result.value); + expect(evidence.confidence).toBe("derived"); + expect(evidence.normalized_result).toEqual(fixture); +}); + +it.each([ + { path: "relative.elf" }, + { path: "/selected.core", approval: true }, + { path: 123 }, + {}, +])( + "rejects malformed selection before starting the provider: %j", + async (input) => { + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => { + throw new Error("must not run"); + }, + }); + expect(await service.inspect(input)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +it("preserves a provider's actionable unsupported reason", async () => { + const failure = new AnalysisCapabilityUnavailableError( + "fixture-recorded-crash", + "inspect_recorded_crash", + "ELF core belongs to recorded-crash analysis.", + ); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => Promise.resolve(err(failure)), + }); + expect(await service.inspect({ path: "/source.elf" })).toEqual(err(failure)); +}); + +it("cancels before invocation and after an acquired observation", async () => { + const controller = new AbortController(); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => { + controller.abort(); + return Promise.resolve(ok(recordedCrashFixture())); + }, + }); + expect( + await service.inspect( + { path: "/artifacts/source-owned.core" }, + { signal: controller.signal }, + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + capturedOutput: recordedCrashFixture().diagnostics, + }, + }); + const untouched = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => { + throw new Error("must not run"); + }, + }); + expect( + await untouched.inspect( + { path: "/source.elf" }, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); +}); + +it.each(["different-path", "invalid-digest", "outside-range"])( + "rejects provider evidence at the next boundary: %s", + async (problem) => { + const fixture = recordedCrashFixture(); + if (problem === "different-path") fixture.artifact.path = "/different.elf"; + if (problem === "invalid-digest") fixture.artifact.sha256 = "unobserved"; + if (problem === "outside-range") + fixture.segments.push({ + index: 0, + type: "PT_LOAD", + offset: "0x200", + header_location: { offset: "0x0", bytes: "0x38" }, + file_backing: "file", + file_size: "0x1", + memory_size: "0x1", + virtual_address: "0x20000000000001", + physical_address: "0x0", + alignment: "0x1", + flags: "0x5", + }); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(fixture)), + }); + expect( + await service.inspect({ path: "/artifacts/source-owned.core" }), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + capturedOutput: fixture.diagnostics, + }, + }); + }, +); diff --git a/src/application/binaryDiagnostics/RecordedCrashService.ts b/src/application/binaryDiagnostics/RecordedCrashService.ts new file mode 100644 index 0000000000..69307498a6 --- /dev/null +++ b/src/application/binaryDiagnostics/RecordedCrashService.ts @@ -0,0 +1,97 @@ +import { isAbsolute } from "node:path"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { RecordedCrashPort } from "./RecordedCrashPort.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { + inspectRecordedCrashInputSchema, + recordedCrashSchema, +} from "../../domain/native/recordedCrash.js"; +import { projectInputIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +const OPERATION = "inspect_recorded_crash"; + +/** Shared CLI/MCP validation and Evidence composition for recorded crash inspection. */ +export class RecordedCrashService { + constructor(readonly provider: RecordedCrashPort) {} + + /** Bind complete observations to the caller-selected artifact before creating Evidence. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const input = inspectRecordedCrashInputSchema.safeParse(rawInput); + if (!input.success) + return err( + new AnalysisInputError( + OPERATION, + { cause: input.error }, + projectInputIssues(input.error.issues, rawInput), + ), + ); + if (!isAbsolute(input.data.path)) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + const inspected = await this.provider.inspect(input.data, options); + if (!inspected.ok) return inspected; + const diagnostics = recordedCrashSchema.shape.diagnostics.safeParse( + inspected.value.diagnostics, + ); + const outputOptions = diagnostics.success + ? { capturedOutput: diagnostics.data } + : undefined; + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION, outputOptions)); + const report = recordedCrashSchema.safeParse(inspected.value); + if ( + !report.success || + report.data.artifact.path !== input.data.path || + (report.data.debugger.status === "available") !== + input.data.include_debugger_context + ) + return err( + new AnalysisOutputError( + OPERATION, + "Recorded crash provider returned malformed evidence or changed the selected artifact identity.", + outputOptions, + ), + ); + const value = report.data; + return ok( + createEvidence( + { + path: value.artifact.path, + format: "elf", + architecture: "x86_64", + sha256: value.artifact.sha256, + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + rawResult: value, + confidence: + value.debugger.status === "available" ? "derived" : "observed", + limitations: value.limitations, + locations: [{ kind: "artifact-path", path: value.artifact.path }], + }, + ), + ); + } +} diff --git a/src/application/javascript/ElectronBoundaryAnalysis.ts b/src/application/javascript/ElectronBoundaryAnalysis.ts index 99de494e2d..c1f8423175 100644 --- a/src/application/javascript/ElectronBoundaryAnalysis.ts +++ b/src/application/javascript/ElectronBoundaryAnalysis.ts @@ -1,7 +1,7 @@ import { compareCodePoints } from "../../domain/canonicalOrdering.js"; import type { ElectronIpcFinding } from "../../domain/javascript/electronStaticAnalysisTypes.js"; import type { ElectronBoundarySummary } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js"; @@ -27,7 +27,7 @@ export interface ElectronIpcPairingState { /** Flatten all per-file IPC facts into deterministic application records. */ export const collectElectronIpcRecords = ( - analysis: JavaScriptArtifactAnalysis, + analysis: JavaScriptModuleArtifactAnalysis, ): readonly ElectronIpcRecord[] => analysis.files .flatMap(({ file, javascript }) => @@ -79,7 +79,7 @@ export const unambiguousElectronIpcPairings = ( /** Summarize static Electron findings without treating gaps as absence. */ export const summarizeElectronBoundaries = ( - analysis: JavaScriptArtifactAnalysis, + analysis: JavaScriptModuleArtifactAnalysis, ): ElectronBoundarySummary => { const files = new Map(analysis.files.map(({ file }) => [file.path, file])); const javascript = analysis.files.flatMap(({ javascript: value }) => diff --git a/src/application/javascript/JavaScriptApplicationEvidence.ts b/src/application/javascript/JavaScriptApplicationEvidence.ts index c0ae7551cf..c291f76738 100644 --- a/src/application/javascript/JavaScriptApplicationEvidence.ts +++ b/src/application/javascript/JavaScriptApplicationEvidence.ts @@ -7,7 +7,6 @@ import type { AnalyzeJavaScriptApplicationInput, JavaScriptApplicationAnalysisResult, } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; import { JAVASCRIPT_APPLICATION_PROVIDER } from "../InvestigationProviders.js"; /** Create Evidence for one deterministic local JavaScript application graph. */ @@ -26,7 +25,7 @@ export const createJavaScriptApplicationEvidence = ( predicateType: "rea.javascript-application-analysis", operation: "analyze_javascript_application", parameters: parameters(input), - result: jsonValueSchema.parse(result), + result, rawResult: null, confidence: "derived", authority: "shipped-artifact", diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index b390854e0d..aba362b768 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -1,7 +1,9 @@ import { createHash } from "node:crypto"; +import { setImmediate } from "node:timers/promises"; import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; import { analyzeParsedJavaScriptSemantics } from "../../domain/javascript/javascriptSemanticAnalysis.js"; +import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js"; import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; import { flattenSourceMapLeaves } from "../../domain/sourceMapEnvelope.js"; @@ -18,6 +20,8 @@ import type { import type { AnalyzedJavaScriptArtifactFile, JavaScriptArtifactAnalysis, + JavaScriptModuleArtifactAnalysis, + JavaScriptModuleSemanticIr, JavaScriptHtmlScriptObservation, JavaScriptJsonModuleObservation, JavaScriptPackageObservation, @@ -27,8 +31,10 @@ import type { import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js"; import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js"; -interface MutableArtifactAnalysis { - readonly files: AnalyzedJavaScriptArtifactFile[]; +interface MutableArtifactAnalysis< + SemanticIr extends JavaScriptModuleSemanticIr, +> { + readonly files: AnalyzedJavaScriptArtifactFile[]; readonly packages: JavaScriptPackageObservation[]; readonly jsonModules: JavaScriptJsonModuleObservation[]; readonly htmlScripts: JavaScriptHtmlScriptObservation[]; @@ -40,23 +46,88 @@ interface MutableArtifactAnalysis { truncatedScopes: number; } -interface ArtifactAnalysisContext { - readonly state: MutableArtifactAnalysis; +interface ArtifactAnalysisContext< + SemanticIr extends JavaScriptModuleSemanticIr, +> { + readonly state: MutableArtifactAnalysis; + readonly projectSemantics: ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ) => SemanticIr; } /** Analyze every text source from the artifact without a result quota. */ export const analyzeJavaScriptArtifactFiles = ( fileSet: JavaScriptArtifactFileSet, -): JavaScriptArtifactAnalysis => { - const state = emptyArtifactAnalysis(); - const context = { state }; +): JavaScriptArtifactAnalysis => + analyzeArtifactFiles(fileSet, (_file, ir) => ir); + +/** Project each file's full IR immediately, retaining only later module facts. */ +export const analyzeAndProjectJavaScriptArtifactFiles = async ( + fileSet: JavaScriptArtifactFileSet, + projectSemantics: ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ) => void, + beforeFile?: ( + file: JavaScriptArtifactFile, + completed: number, + total: number, + ) => Promise, +): Promise => { + const state = emptyArtifactAnalysis(); + const projectFileSemantics = ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ): JavaScriptModuleSemanticIr => { + projectSemantics(file, ir); + const programScopes = ir.scopes.filter(({ kind }) => kind === "program"); + const programScopeId = programScopes[0]?.scopeId; + const localNames = new Set( + ir.moduleLinks.map(({ localName }) => localName), + ); + const callableIds = new Set( + ir.moduleLinks.map(({ callableId }) => callableId), + ); + return { + scopes: programScopes, + bindings: ir.bindings.filter( + ({ name, scopeId }) => + scopeId === programScopeId && localNames.has(name), + ), + callables: ir.callables.filter(({ callableId }) => + callableIds.has(callableId), + ), + moduleLinks: ir.moduleLinks, + coverage: ir.coverage, + limitations: ir.limitations, + }; + }; + const context = { state, projectSemantics: projectFileSemantics }; + for (const [index, file] of fileSet.files.entries()) { + // Let progress, cancellation and garbage collection run between files. + await setImmediate(); + await beforeFile?.(file, index, fileSet.files.length); + analyzeArtifactFile(file, context); + } + return finalizeArtifactAnalysis(state); +}; + +const analyzeArtifactFiles = ( + fileSet: JavaScriptArtifactFileSet, + projectSemantics: ArtifactAnalysisContext["projectSemantics"], +): JavaScriptArtifactAnalysis => { + const state = emptyArtifactAnalysis(); + const context = { state, projectSemantics }; for (const file of fileSet.files) analyzeArtifactFile(file, context); return finalizeArtifactAnalysis(state); }; -const finalizeArtifactAnalysis = ( - state: MutableArtifactAnalysis, -): JavaScriptArtifactAnalysis => { +const finalizeArtifactAnalysis = < + SemanticIr extends JavaScriptModuleSemanticIr, +>( + state: MutableArtifactAnalysis, +): JavaScriptArtifactAnalysis => { const truncatedScopes = state.truncatedScopes; return { files: state.files, @@ -79,9 +150,9 @@ const finalizeArtifactAnalysis = ( }; }; -const analyzeArtifactFile = ( +const analyzeArtifactFile = ( file: JavaScriptArtifactFile, - context: ArtifactAnalysisContext, + context: ArtifactAnalysisContext, ): void => { const { state } = context; addStructuredObservations(file, state); @@ -108,7 +179,10 @@ const analyzeArtifactFile = ( state.files.push({ file, javascript: analysis, - semantic: semantics === null ? null : { ir: semantics }, + semantic: + semantics === null + ? null + : { ir: context.projectSemantics(file, semantics) }, }); state.visitedNodes += analysis.visited_ast_nodes; state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0); @@ -121,13 +195,15 @@ const analyzeArtifactFile = ( const addSourceMap = ( file: JavaScriptArtifactFile, - context: ArtifactAnalysisContext, + context: ArtifactAnalysisContext, ): void => { const sourceMap = parseSourceMap(file); context.state.sourceMaps.push(sourceMap); }; -const emptyArtifactAnalysis = (): MutableArtifactAnalysis => ({ +const emptyArtifactAnalysis = < + SemanticIr extends JavaScriptModuleSemanticIr, +>(): MutableArtifactAnalysis => ({ files: [], packages: [], jsonModules: [], @@ -142,7 +218,7 @@ const emptyArtifactAnalysis = (): MutableArtifactAnalysis => ({ const addStructuredObservations = ( file: JavaScriptArtifactFile, - state: MutableArtifactAnalysis, + state: MutableArtifactAnalysis, ): void => { if (file.kind === "package-json") state.packages.push(parsePackage(file)); if (file.kind !== "json") return; diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index d1cdd2a6b5..152b3520af 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -87,18 +87,33 @@ export type JavaScriptJsonModuleObservation = JavaScriptStructuredObservation & } ); +/** Module facts needed after file-local semantic projection has finished. */ +export type JavaScriptModuleSemanticIr = Pick< + JavaScriptSemanticIr, + | "scopes" + | "bindings" + | "callables" + | "moduleLinks" + | "coverage" + | "limitations" +>; + /** One relevant file plus optional AST-only JavaScript facts. */ -export interface AnalyzedJavaScriptArtifactFile { +export interface AnalyzedJavaScriptArtifactFile< + SemanticIr extends JavaScriptModuleSemanticIr = JavaScriptSemanticIr, +> { readonly file: JavaScriptArtifactFile; readonly javascript: JavaScriptStaticAnalysis | null; readonly semantic: { - readonly ir: JavaScriptSemanticIr; + readonly ir: SemanticIr; } | null; } /** Complete static-analysis projection before graph construction. */ -export interface JavaScriptArtifactAnalysis { - readonly files: readonly AnalyzedJavaScriptArtifactFile[]; +export interface JavaScriptArtifactAnalysis< + SemanticIr extends JavaScriptModuleSemanticIr = JavaScriptSemanticIr, +> { + readonly files: readonly AnalyzedJavaScriptArtifactFile[]; readonly packages: readonly JavaScriptPackageObservation[]; readonly json_modules: readonly JavaScriptJsonModuleObservation[]; readonly html_scripts: readonly JavaScriptHtmlScriptObservation[]; @@ -110,3 +125,7 @@ export interface JavaScriptArtifactAnalysis { readonly truncated_scopes: number; readonly limitations: readonly string[]; } + +/** Artifact facts retaining only semantics used by application relationships. */ +export type JavaScriptModuleArtifactAnalysis = + JavaScriptArtifactAnalysis; diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 49f317270d..e910d9cc51 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -3,7 +3,7 @@ import { createJavaScriptApplicationGraph, type JavaScriptApplicationGraph, } from "../../domain/javascript/javascriptApplicationGraph.js"; -import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactFileSet } from "../../domain/javascript/javascriptArtifactFiles.js"; import { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js"; import type { JavaScriptArtifactGraphContext } from "./JavaScriptArtifactGraphContext.js"; @@ -37,7 +37,7 @@ import { export const buildJavaScriptArtifactGraph = ( snapshot: ArtifactInventorySnapshot, fileSet: JavaScriptArtifactFileSet, - analysis: JavaScriptArtifactAnalysis, + analysis: JavaScriptModuleArtifactAnalysis, ): JavaScriptApplicationGraph => { const accumulator = new JavaScriptArtifactGraphAccumulator(); const root = createJavaScriptArtifactRootNode(accumulator, snapshot); diff --git a/src/application/javascript/JavaScriptArtifactGraphContext.ts b/src/application/javascript/JavaScriptArtifactGraphContext.ts index 291dfa46fb..0ef275eeb9 100644 --- a/src/application/javascript/JavaScriptArtifactGraphContext.ts +++ b/src/application/javascript/JavaScriptArtifactGraphContext.ts @@ -6,7 +6,7 @@ import { } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraph.js"; import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; -import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactFile, JavaScriptArtifactFileSet, @@ -29,7 +29,7 @@ export interface JavaScriptArtifactGraphContext { readonly accumulator: JavaScriptArtifactGraphAccumulator; readonly snapshot: ArtifactInventorySnapshot; readonly fileSet: JavaScriptArtifactFileSet; - readonly analysis: JavaScriptArtifactAnalysis; + readonly analysis: JavaScriptModuleArtifactAnalysis; readonly root: ApplicationNode; readonly filesByPath: ReadonlyMap; readonly fileNodes: Map; @@ -209,7 +209,7 @@ export const addUnavailableStaticParseScope = ( /** Coverage for one bounded JavaScript AST analysis. */ export const javascriptAnalysisCoverage = ( analysis: NonNullable< - JavaScriptArtifactAnalysis["files"][number]["javascript"] + JavaScriptModuleArtifactAnalysis["files"][number]["javascript"] >, ): JavaScriptArtifactGraphCoverage => analysis.parse_status === "complete" diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts index 039659f89f..1299f28dc0 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts @@ -1,7 +1,7 @@ import type { ArtifactInventorySnapshot } from "../../artifacts/inventory/ArtifactInventory.js"; import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraph.js"; import { completeApplicationCoverage } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; -import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactContainer, JavaScriptArtifactFile, @@ -270,7 +270,7 @@ export const addJavaScriptPackageNodes = ( const createFileTarget = ( context: JavaScriptArtifactGraphContext, file: JavaScriptArtifactFile, - javascript: JavaScriptArtifactAnalysis["files"][number]["javascript"], + javascript: JavaScriptModuleArtifactAnalysis["files"][number]["javascript"], ): ApplicationNode => { const kind = artifactFileNodeKind(file.kind); const json = context.analysis.json_modules.find( diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index 7f14c78989..57575552f0 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -5,7 +5,7 @@ import { createJavaScriptArtifactReader as createReader } from "../../artifacts/ import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascriptApplicationGraph.js"; import type { JavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { ElectronBoundarySummary } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; +import { analyzeAndProjectJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; import { readJavaScriptArtifactFiles } from "../../artifacts/javascript/JavaScriptArtifactFiles.js"; import { buildJavaScriptArtifactGraph } from "./JavaScriptArtifactGraphBuilder.js"; import { @@ -14,7 +14,7 @@ import { } from "./JavaScriptArtifactReconstructionInput.js"; import { scanCanonicalArtifactInventory } from "../../artifacts/inventory/ArtifactInventory.js"; import { summarizeElectronBoundaries } from "./ElectronBoundaryAnalysis.js"; -import { buildJavaScriptSemanticGraph } from "./JavaScriptSemanticGraphBuilder.js"; +import { createJavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js"; import type { ProgressReporter } from "../ProgressReporter.js"; /** Application-layer result retaining local diagnostics outside the canonical graph. */ @@ -70,9 +70,23 @@ export const reconstructJavaScriptArtifact = async ( const files = await readJavaScriptArtifactFiles(reader, snapshot, signal); await reportPhase( "parse_javascript_sources", - `Parsing ${String(files.files.length)} application source files`, + `Parsing and projecting ${String(files.files.length)} application source files`, + ); + const semanticProjection = createJavaScriptSemanticGraphProjection(); + const analysis = await analyzeAndProjectJavaScriptArtifactFiles( + files, + semanticProjection.projectFile, + async (file, completed, total) => { + abortIfNeeded(signal); + await progress?.report({ + phase: "parse_javascript_source", + completed: 0, + total: 1, + message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`, + }); + abortIfNeeded(signal); + }, ); - const analysis = analyzeJavaScriptArtifactFiles(files); abortIfNeeded(signal); await reportPhase( "build_javascript_application_graph", @@ -81,13 +95,13 @@ export const reconstructJavaScriptArtifact = async ( const graph = buildJavaScriptArtifactGraph(snapshot, files, analysis); await reportPhase( "build_javascript_semantic_graph", - "Constructing static semantic relationships", + "Binding and validating static semantic relationships", ); - const semanticGraph = buildJavaScriptSemanticGraph({ - rootArtifactSha256: snapshot.manifest.root_sha256, - applicationGraph: graph, + const semanticGraph = semanticProjection.finish( + snapshot.manifest.root_sha256, + graph, analysis, - }); + ); return { input_path: path, format, diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index d3b688ca4b..e87d165bdd 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -8,7 +8,6 @@ import { } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import type { JavaScriptModuleOrigin, - JavaScriptSemanticIr, JavaScriptSemanticModuleLink, } from "../../domain/javascript/javascriptSemanticIr.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -28,8 +27,10 @@ import { } from "./JavaScriptArtifactPathResolution.js"; import { projectJavaScriptExportReturnShapes } from "./JavaScriptReturnShapeProjection.js"; +import type { JavaScriptModuleSemanticIr } from "./JavaScriptArtifactAnalysisTypes.js"; + interface SemanticAnalysis { - readonly ir: JavaScriptSemanticIr; + readonly ir: JavaScriptModuleSemanticIr; } interface RelationshipInput { @@ -310,7 +311,7 @@ const unresolvedModuleNode = ( }); const moduleOriginForExport = ( - ir: JavaScriptSemanticIr, + ir: JavaScriptModuleSemanticIr, link: JavaScriptSemanticModuleLink, ): JavaScriptModuleOrigin | null => { if (link.specifier !== null) @@ -374,7 +375,7 @@ const semanticCoverage = ( const moduleFormat = ( path: string, - ir: JavaScriptSemanticIr, + ir: JavaScriptModuleSemanticIr, ): "commonjs" | "esm" | "mixed" | "unknown" => { const extension = posix.extname(path).toLowerCase(); if (extension === ".mjs") return "esm"; diff --git a/src/application/javascript/JavaScriptReturnShapeProjection.ts b/src/application/javascript/JavaScriptReturnShapeProjection.ts index dcd806ef88..f22a5fecaf 100644 --- a/src/application/javascript/JavaScriptReturnShapeProjection.ts +++ b/src/application/javascript/JavaScriptReturnShapeProjection.ts @@ -4,13 +4,13 @@ import { truncatedApplicationCoverage, } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import type { - JavaScriptSemanticIr, JavaScriptSemanticCallable, JavaScriptSemanticModuleLink, } from "../../domain/javascript/javascriptSemanticIr.js"; import { flattenSemanticReturnValue } from "../../domain/javascript/javascriptSemanticReturns.js"; import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactGraphCoverage } from "./JavaScriptArtifactGraphContext.js"; +import type { JavaScriptModuleSemanticIr } from "./JavaScriptArtifactAnalysisTypes.js"; export interface JavaScriptReturnShapeProjection { readonly properties: ReturnType; @@ -21,7 +21,7 @@ export interface JavaScriptReturnShapeProjection { /** Project one exact export/callable link into shallow bounded graph values. */ export const projectJavaScriptExportReturnShapes = (input: { - readonly ir: JavaScriptSemanticIr; + readonly ir: JavaScriptModuleSemanticIr; readonly link: JavaScriptSemanticModuleLink; readonly modulePath: string; readonly baseCoverage: JavaScriptArtifactGraphCoverage; diff --git a/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts b/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts index c44b1cc5ac..8d949e964f 100644 --- a/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphAsyncProjection.ts @@ -5,10 +5,9 @@ import type { JavaScriptSemanticTimerOperation, } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; import { @@ -59,55 +58,41 @@ const addEventNode = ( operation: JavaScriptSemanticEventOperation, eventKey: string, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "event", - roleKey: `event:${eventKey}`, - location: operation.eventName === null ? operation.location : null, - label: operation.eventName, - functionNodeId: null, - properties: { - emitter_key: operation.emitterKey, - event_name: operation.eventName, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "event", + roleKey: `event:${eventKey}`, + location: operation.eventName === null ? operation.location : null, + label: operation.eventName, + functionNodeId: null, + properties: { + emitter_key: operation.emitterKey, + event_name: operation.eventName, + }, + }); const addListenerNode = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticEventOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "listener", - roleKey: `listener:${operation.eventId}`, - location: operation.listenerLocation ?? operation.location, - label: - operation.eventName === null - ? operation.method - : `${operation.method}:${operation.eventName}`, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { - emitter_key: operation.emitterKey, - event_name: operation.eventName, - method: operation.method, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "listener", + roleKey: `listener:${operation.eventId}`, + location: operation.listenerLocation ?? operation.location, + label: + operation.eventName === null + ? operation.method + : `${operation.method}:${operation.eventName}`, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { + emitter_key: operation.emitterKey, + event_name: operation.eventName, + method: operation.method, + }, + }); const eventEmitterNode = ( context: SemanticFlowProjectionContext, @@ -164,24 +149,17 @@ export const projectSemanticTimers = ( operation.ownerCallableId === null ? undefined : context.callableNodes.get(operation.ownerCallableId); - const timer = addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "timer", - roleKey: operation.timerId, - location: operation.location, - label: operation.method, - functionNodeId: owner?.node_id ?? null, - properties: { - delay_milliseconds: operation.delayMilliseconds, - method: operation.method, - }, - }, - context.state, - ), - ); + const timer = retainSemanticGraphNode(context.state, context.file, { + kind: "timer", + roleKey: operation.timerId, + location: operation.location, + label: operation.method, + functionNodeId: owner?.node_id ?? null, + properties: { + delay_milliseconds: operation.delayMilliseconds, + method: operation.method, + }, + }); return timer === null ? [] : [[operation.timerId, timer] as const]; }), ); diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 38a3522343..424ab9fdfd 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -38,16 +38,17 @@ import { type SemanticFlowProjectionContext, } from "./JavaScriptSemanticGraphFlowProjection.js"; import { - owningSemanticCallableNode, + createSemanticCallableOwnerLookup, + createSemanticCallSiteLookup, + createSemanticNodeRangeLookup, semanticFamilyStatus, - semanticNodesWithinRange, } from "./JavaScriptSemanticGraphProjection.js"; import { addSemanticFallbackRoot as addFallbackRoot, - addSemanticGraphNode as addNode, + retainSemanticGraphNode as retainNode, addSemanticGraphRelation as addRelation, - constructSemanticGraphNode as semanticNode, createSemanticGraphProjectionState as emptyState, + bindSemanticGraphApplicationNodes, type SemanticGraphProjectionState as BuilderState, } from "./JavaScriptSemanticGraphConstruction.js"; @@ -72,6 +73,9 @@ interface FileContext extends SemanticFlowProjectionContext { readonly argumentNodes: Map; readonly referenceNodes: JavaScriptSemanticGraphNode[]; readonly callResolutions: ReadonlyMap; + readonly callableOwnerAt: ReturnType< + typeof createSemanticCallableOwnerLookup + >; } /** @@ -108,152 +112,170 @@ export const buildJavaScriptSemanticGraph = ({ applicationGraph, analysis, }: BuilderInput): JavaScriptSemanticGraph => { - const state = emptyState(applicationGraph); - const fingerprints: JavaScriptSemanticGraph["fingerprints"][number][] = []; - let truncatedFiles = 0; + const projection = createJavaScriptSemanticGraphProjection(); for (const analyzed of analysis.files) { - if (analyzed.semantic === null) continue; - if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { - truncatedFiles += 1; - continue; - } - const remainingTreeBudget = SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size; - state.fileNodeBudget = Math.min( - SEMANTIC_GRAPH_FILE_NODE_CEILING, - remainingTreeBudget, - ); - state.fileNodesDropped = false; - fingerprints.push( - ...projectFile(analyzed.file, analyzed.semantic.ir, state), - ); - // Nodes were dropped only if the budget actually blocked creation. A file - // that exactly fills its share ends with a zero budget without dropping - // anything, so the remaining budget alone cannot decide truncation. - if (state.fileNodesDropped) truncatedFiles += 1; - state.fileNodeBudget = null; + if (analyzed.semantic !== null) + projection.projectFile(analyzed.file, analyzed.semantic.ir); } - if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); - const unknowns = [...state.unknowns.values()]; - return createJavaScriptSemanticGraph({ - schema: "JavaScriptSemanticRelationGraph", - root_artifact_sha256: rootArtifactSha256, - application_graph_id: applicationGraph.graph_id, - root_node_ids: [...state.roots], - nodes: [...state.nodes.values()], - relations: [...state.relations.values()], - fingerprints, - unknowns, - coverage: { - status: truncatedFiles > 0 ? "partial" : "unknown", - truncated: truncatedFiles > 0, - omitted_nodes: truncatedFiles > 0 ? null : 0, - omitted_relations: truncatedFiles > 0 ? null : 0, - limits: - truncatedFiles > 0 - ? [ - { - name: "semantic_graph_node_ceiling", - value: SEMANTIC_GRAPH_NODE_CEILING, - unit: "items" as const, - }, - ] - : [], - families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ - family, - status: semanticFamilyStatus(family, analysis), - retained_relations: [...state.relations.values()].filter( - (relation) => - JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation] === family, - ).length, - omitted_relations: truncatedFiles > 0 ? null : 0, - unknown_ids: unknowns - .filter((unknown) => unknown.family === family) - .map(({ unknown_id: identifier }) => identifier), - })), - }, - limitations: [ - "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", - "Local data flow does not claim control-flow-sensitive reaching definitions or arbitrary dynamic property resolution.", - "Promise ownership covers explicit unshadowed Promise construction, static factories, aggregation, chaining, and await syntax only.", - "Function fingerprints are static candidates; equal digests can remain ambiguous and do not prove behavioral equivalence.", - "Event extraction covers EventEmitter-style literal registrations, removals, and dispatch candidates; dynamic names remain unknown.", - "Timer extraction covers global or node:timers scheduling and exact local-handle cancellation.", - "Child-process extraction covers asynchronous node:child_process creation, literal argv/env/stdio options, exit/error listeners, and kill signals.", - "Configuration extraction covers process.env, process.argv, node:fs reads, and direct logical defaults.", - "Request extraction covers fetch, WebSocket, node:http/node:https construction, direct option fields, and exact local response consumers.", - "Boundary extraction covers unshadowed JSON/global coercions plus parse and validation method candidates.", - "Resource extraction covers built-in filesystem/network acquisition and exact local close/destroy/end handles.", - ], - }); + return projection.finish(rootArtifactSha256, applicationGraph, analysis); }; +/** File-local semantic projection that does not retain consumed source IR. */ +export interface JavaScriptSemanticGraphProjection { + readonly projectFile: ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ) => void; + readonly finish: ( + rootArtifactSha256: string, + applicationGraph: BuilderInput["applicationGraph"], + analysis: Pick, + ) => JavaScriptSemanticGraph; +} + +/** Accumulate semantic nodes while each source IR is still file-local. */ +export const createJavaScriptSemanticGraphProjection = + (): JavaScriptSemanticGraphProjection => { + const state = emptyState({ nodes: [] }); + const fingerprints: JavaScriptSemanticGraph["fingerprints"][number][] = []; + let truncatedFiles = 0; + const projectSource = ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ): void => { + if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { + truncatedFiles += 1; + return; + } + const remainingTreeBudget = + SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size; + state.fileNodeBudget = Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + remainingTreeBudget, + ); + state.fileNodesDropped = false; + fingerprints.push(...projectFile(file, ir, state)); + // Nodes were dropped only if the budget actually blocked creation. A file + // that exactly fills its share ends with a zero budget without dropping + // anything, so the remaining budget alone cannot decide truncation. + if (state.fileNodesDropped) truncatedFiles += 1; + state.fileNodeBudget = null; + }; + return { + projectFile: projectSource, + finish: (rootArtifactSha256, applicationGraph, analysis) => { + bindSemanticGraphApplicationNodes(state, applicationGraph); + if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); + const unknowns = [...state.unknowns.values()]; + const graph = createJavaScriptSemanticGraph({ + schema: "JavaScriptSemanticRelationGraph", + root_artifact_sha256: rootArtifactSha256, + application_graph_id: applicationGraph.graph_id, + root_node_ids: [...state.roots], + nodes: [...state.nodes.values()], + relations: [...state.relations.values()], + fingerprints, + unknowns, + coverage: { + status: truncatedFiles > 0 ? "partial" : "unknown", + truncated: truncatedFiles > 0, + omitted_nodes: truncatedFiles > 0 ? null : 0, + omitted_relations: truncatedFiles > 0 ? null : 0, + limits: + truncatedFiles > 0 + ? [ + { + name: "semantic_graph_node_ceiling", + value: SEMANTIC_GRAPH_NODE_CEILING, + unit: "items" as const, + }, + ] + : [], + families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ + family, + status: semanticFamilyStatus(family, analysis), + retained_relations: [...state.relations.values()].filter( + (relation) => + JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation] === + family, + ).length, + omitted_relations: truncatedFiles > 0 ? null : 0, + unknown_ids: unknowns + .filter((unknown) => unknown.family === family) + .map(({ unknown_id: identifier }) => identifier), + })), + }, + limitations: [ + "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", + "Local data flow does not claim control-flow-sensitive reaching definitions or arbitrary dynamic property resolution.", + "Promise ownership covers explicit unshadowed Promise construction, static factories, aggregation, chaining, and await syntax only.", + "Function fingerprints are static candidates; equal digests can remain ambiguous and do not prove behavioral equivalence.", + "Event extraction covers EventEmitter-style literal registrations, removals, and dispatch candidates; dynamic names remain unknown.", + "Timer extraction covers global or node:timers scheduling and exact local-handle cancellation.", + "Child-process extraction covers asynchronous node:child_process creation, literal argv/env/stdio options, exit/error listeners, and kill signals.", + "Configuration extraction covers process.env, process.argv, node:fs reads, and direct logical defaults.", + "Request extraction covers fetch, WebSocket, node:http/node:https construction, direct option fields, and exact local response consumers.", + "Boundary extraction covers unshadowed JSON/global coercions plus parse and validation method candidates.", + "Resource extraction covers built-in filesystem/network acquisition and exact local close/destroy/end handles.", + ], + }); + state.nodes.clear(); + state.relations.clear(); + state.unknowns.clear(); + state.roots.clear(); + fingerprints.length = 0; + return graph; + }, + }; + }; + const projectFile = ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, state: BuilderState, ): JavaScriptSemanticGraph["fingerprints"][number][] => { - const moduleNode = addNode( - state, - semanticNode( - file, - { - kind: "module", - roleKey: "module", - location: null, - label: file.path, - functionNodeId: null, - }, - state, - ), - ); + const moduleNode = retainNode(state, file, { + kind: "module", + roleKey: "module", + location: null, + label: file.path, + functionNodeId: null, + }); if (moduleNode === null) return []; state.roots.add(moduleNode.node_id); const callableNodes = new Map( ir.callables.flatMap((callable) => { - const node = addNode( - state, - semanticNode( - file, - { - kind: "function", - roleKey: `callable:${callable.callableId}`, - location: callable.location, - label: callable.name, - functionNodeId: null, - // The label is display text; keep the exact name, which may be "". - properties: { name: callable.name }, - }, - state, - ), - ); + const node = retainNode(state, file, { + kind: "function", + roleKey: `callable:${callable.callableId}`, + location: callable.location, + label: callable.name, + functionNodeId: null, + // The label is display text; keep the exact name, which may be "". + properties: { name: callable.name }, + }); return node === null ? [] : [[callable.callableId, node] as const]; }), ); + const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes); const bindingNodes = new Map( ir.bindings.flatMap((binding) => { const location = binding.definitions[0]?.location ?? null; const kind = binding.kind === "parameter" ? "parameter" : "binding"; - const owner = owningSemanticCallableNode(location, ir, callableNodes); - const node = addNode( - state, - semanticNode( - file, - { - kind, - roleKey: `binding:${binding.bindingId}`, - location, - label: binding.name, - functionNodeId: owner?.node_id ?? null, - }, - state, - ), - ); + const owner = callableOwnerAt(location); + const node = retainNode(state, file, { + kind, + roleKey: `binding:${binding.bindingId}`, + location, + label: binding.name, + functionNodeId: owner?.node_id ?? null, + }); return node === null ? [] : [[binding.bindingId, node] as const]; }), ); const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state); const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state); - const context: FileContext = { + const definitions: Omit = { file, ir, state, @@ -270,8 +292,16 @@ const projectFile = ( call.resolution === "exact" ? "resolved" : "candidate", ]), ), + callableOwnerAt, + callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes), + }; + projectDefinitionsAndReferences(definitions); + const context: FileContext = { + ...definitions, + referenceNodesWithin: createSemanticNodeRangeLookup( + definitions.referenceNodes, + ), }; - projectDefinitionsAndReferences(context); projectSemanticValues(context); projectSemanticObjects(context); projectCalls(context); @@ -300,20 +330,13 @@ const createReturnSiteNodes = ( if (owner === undefined) continue; for (const site of callable.returnSites) { const identifier = site.returnSiteId; - const node = addNode( - state, - semanticNode( - file, - { - kind: "return-site", - roleKey: identifier, - location: site.location, - label: "return", - functionNodeId: owner.node_id, - }, - state, - ), - ); + const node = retainNode(state, file, { + kind: "return-site", + roleKey: identifier, + location: site.location, + label: "return", + functionNodeId: owner.node_id, + }); if (node !== null) result.set(identifier, node); } } @@ -332,44 +355,32 @@ const createCallSiteNodes = ( call.callerCallableId === null ? null : (callables.get(call.callerCallableId)?.node_id ?? null); - const node = addNode( - state, - semanticNode( - file, - { - kind: "call-site", - roleKey: `call:${call.callSiteId}`, - location: call.location, - label: call.kind, - functionNodeId: owner, - }, - state, - ), - ); + const node = retainNode(state, file, { + kind: "call-site", + roleKey: `call:${call.callSiteId}`, + location: call.location, + label: call.kind, + functionNodeId: owner, + }); if (node !== null) result.set(call.callSiteId, node); } return result; }; -const projectDefinitionsAndReferences = (context: FileContext): void => { +const projectDefinitionsAndReferences = ( + context: Omit, +): void => { for (const binding of context.ir.bindings) { const bindingNode = context.bindingNodes.get(binding.bindingId); if (bindingNode === undefined) continue; for (const [index, definition] of binding.definitions.entries()) { - const definitionNode = addNode( - context.state, - semanticNode( - context.file, - { - kind: "expression", - roleKey: `definition:${binding.bindingId}:${String(index)}`, - location: definition.location, - label: definition.kind, - functionNodeId: bindingNode.function_node_id, - }, - context.state, - ), - ); + const definitionNode = retainNode(context.state, context.file, { + kind: "expression", + roleKey: `definition:${binding.bindingId}:${String(index)}`, + location: definition.location, + label: definition.kind, + functionNodeId: bindingNode.function_node_id, + }); addRelation(context.state, { source: definitionNode, target: bindingNode, @@ -379,25 +390,14 @@ const projectDefinitionsAndReferences = (context: FileContext): void => { } } for (const [index, reference] of context.ir.references.entries()) { - const expression = addNode( - context.state, - semanticNode( - context.file, - { - kind: "expression", - roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`, - location: reference.location, - label: reference.name, - functionNodeId: - owningSemanticCallableNode( - reference.location, - context.ir, - context.callableNodes, - )?.node_id ?? null, - }, - context.state, - ), - ); + const expression = retainNode(context.state, context.file, { + kind: "expression", + roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`, + location: reference.location, + label: reference.name, + functionNodeId: + context.callableOwnerAt(reference.location)?.node_id ?? null, + }); const binding = reference.bindingId === null ? undefined @@ -447,10 +447,7 @@ const projectCalls = (context: FileContext): void => { `${call.callSiteId}\u0000${String(argument.index)}`, argumentNode, ); - const references = semanticNodesWithinRange( - context.referenceNodes, - argument.location, - ); + const references = context.referenceNodesWithin(argument.location); for (const reference of references) addRelation(context.state, { source: reference, @@ -499,19 +496,12 @@ const createArgumentNode = ( callSiteId: string, argument: JavaScriptSemanticCallArgument, ): JavaScriptSemanticGraphNode | null => - addNode( - context.state, - semanticNode( - context.file, - { - kind: "expression", - roleKey: `argument:${callSiteId}:${String(argument.index)}`, - location: argument.location, - label: argument.spread - ? "spread argument" - : `argument ${String(argument.index)}`, - functionNodeId: callNode.function_node_id, - }, - context.state, - ), - ); + retainNode(context.state, context.file, { + kind: "expression", + roleKey: `argument:${callSiteId}:${String(argument.index)}`, + location: argument.location, + label: argument.spread + ? "spread argument" + : `argument ${String(argument.index)}`, + functionNodeId: callNode.function_node_id, + }); diff --git a/src/application/javascript/JavaScriptSemanticGraphChildProcessProjection.ts b/src/application/javascript/JavaScriptSemanticGraphChildProcessProjection.ts index 450ecff565..600d0390bc 100644 --- a/src/application/javascript/JavaScriptSemanticGraphChildProcessProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphChildProcessProjection.ts @@ -5,10 +5,9 @@ import type { JavaScriptSemanticChildProcessSpawn, } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; import { @@ -53,31 +52,23 @@ const addChildProcessNode = ( context: SemanticFlowProjectionContext, spawn: JavaScriptSemanticChildProcessSpawn, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "child-process", - roleKey: spawn.processId, - location: spawn.location, - label: spawn.command ?? spawn.method, - functionNodeId: - spawn.ownerCallableId === null - ? null - : (context.callableNodes.get(spawn.ownerCallableId)?.node_id ?? - null), - properties: { - argv_count: spawn.argvCount, - command: spawn.command, - environment_supplied: spawn.environmentSupplied, - method: spawn.method, - stdio_mode: spawn.stdioMode, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "child-process", + roleKey: spawn.processId, + location: spawn.location, + label: spawn.command ?? spawn.method, + functionNodeId: + spawn.ownerCallableId === null + ? null + : (context.callableNodes.get(spawn.ownerCallableId)?.node_id ?? null), + properties: { + argv_count: spawn.argvCount, + command: spawn.command, + environment_supplied: spawn.environmentSupplied, + method: spawn.method, + stdio_mode: spawn.stdioMode, + }, + }); const projectSpawn = ( context: SemanticFlowProjectionContext, @@ -141,21 +132,14 @@ const addStdioNode = ( spawn: JavaScriptSemanticChildProcessSpawn, child: JavaScriptSemanticGraphNode, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "stdio", - roleKey: `stdio:${spawn.processId}`, - location: spawn.location, - label: spawn.stdioMode, - functionNodeId: child.function_node_id, - properties: { mode: spawn.stdioMode }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "stdio", + roleKey: `stdio:${spawn.processId}`, + location: spawn.location, + label: spawn.stdioMode, + functionNodeId: child.function_node_id, + properties: { mode: spawn.stdioMode }, + }); const projectInteraction = ( context: SemanticFlowProjectionContext, @@ -204,52 +188,38 @@ const addChildListenerNode = ( context: SemanticFlowProjectionContext, interaction: JavaScriptSemanticChildProcessInteraction, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "listener", - roleKey: interaction.interactionId, - location: interaction.listenerLocation ?? interaction.location, - label: interaction.eventName, - functionNodeId: - interaction.ownerCallableId === null - ? null - : (context.callableNodes.get(interaction.ownerCallableId) - ?.node_id ?? null), - properties: { - event_name: interaction.eventName, - method: interaction.method, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "listener", + roleKey: interaction.interactionId, + location: interaction.listenerLocation ?? interaction.location, + label: interaction.eventName, + functionNodeId: + interaction.ownerCallableId === null + ? null + : (context.callableNodes.get(interaction.ownerCallableId)?.node_id ?? + null), + properties: { + event_name: interaction.eventName, + method: interaction.method, + }, + }); const addSignalNode = ( context: SemanticFlowProjectionContext, interaction: JavaScriptSemanticChildProcessInteraction, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "signal", - roleKey: interaction.interactionId, - location: interaction.location, - label: interaction.signalName, - functionNodeId: - interaction.ownerCallableId === null - ? null - : (context.callableNodes.get(interaction.ownerCallableId) - ?.node_id ?? null), - properties: { signal_name: interaction.signalName }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "signal", + roleKey: interaction.interactionId, + location: interaction.location, + label: interaction.signalName, + functionNodeId: + interaction.ownerCallableId === null + ? null + : (context.callableNodes.get(interaction.ownerCallableId)?.node_id ?? + null), + properties: { signal_name: interaction.signalName }, + }); const addChildUnknown = (input: ChildUnknownInput): void => { const { context, node, location, relationKinds, detail, candidateNodeIds } = diff --git a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts index d69d9e7367..bb7322783a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts +++ b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts @@ -4,6 +4,7 @@ import { javascriptDisplayText } from "../../domain/javascript/javascriptAstValu import { createJavaScriptSemanticGraphNode, createJavaScriptSemanticGraphRelation, + javaScriptSemanticNodeId, type JavaScriptSemanticGraphNode, } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { ApplicationGraphEvidence } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; @@ -80,6 +81,28 @@ export const createSemanticGraphProjectionState = ( }; }; +/** Bind retained semantic identities after all application observations exist. */ +export const bindSemanticGraphApplicationNodes = ( + state: SemanticGraphProjectionState, + applicationGraph: Pick, +): void => { + const index = indexApplicationNodes(applicationGraph.nodes).identifiers; + for (const [identifier, node] of state.nodes) { + state.nodes.set(identifier, { + ...node, + application_node_ids: [ + ...(index.get( + applicationLocationKey( + node.identity.artifact_sha256, + node.identity.module_path, + node.identity.source_range, + ), + ) ?? []), + ], + }); + } +}; + /** Construct one canonical semantic node backed by an exact artifact file. */ export const constructSemanticGraphNode = ( file: JavaScriptArtifactFile, @@ -101,6 +124,33 @@ export const constructSemanticGraphNode = ( evidence: observedSemanticEvidence(file, input.location), }); +/** Reuse retained identities and reject exhausted budgets before allocation. */ +export const retainSemanticGraphNode = ( + state: SemanticGraphProjectionState, + file: JavaScriptArtifactFile, + input: SemanticNodeConstructionInput, +): JavaScriptSemanticGraphNode | null => { + const identifier = javaScriptSemanticNodeId({ + kind: input.kind, + identity: { + artifact_sha256: file.sha256, + module_path: file.path, + source_range: input.location, + role_key: input.roleKey, + }, + }); + const existing = state.nodes.get(identifier); + if (existing !== undefined) return existing; + if (typeof state.fileNodeBudget === "number" && state.fileNodeBudget <= 0) { + state.fileNodesDropped = true; + return null; + } + return addSemanticGraphNode( + state, + constructSemanticGraphNode(file, input, state), + ); +}; + /** Retain one canonical semantic node. */ export const addSemanticGraphNode = ( state: SemanticGraphProjectionState, diff --git a/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts b/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts index 8ebd024b89..761eba0e32 100644 --- a/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphDataProjection.ts @@ -6,10 +6,9 @@ import type { JavaScriptSemanticRequestOperation, } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; import { @@ -70,29 +69,22 @@ const addConfigurationNode = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticConfigurationOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "config-source", - roleKey: operation.configId, - location: operation.location, - label: operation.key ?? operation.kind, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { - key: operation.key, - source_kind: operation.kind, - value: operation.value, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "config-source", + roleKey: operation.configId, + location: operation.location, + label: operation.key ?? operation.kind, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { + key: operation.key, + source_kind: operation.kind, + value: operation.value, + }, + }); const configurationRelation = ( operation: JavaScriptSemanticConfigurationOperation, @@ -144,29 +136,22 @@ const addRequestNode = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticRequestOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: operation.kind === "request" ? "request" : "response", - roleKey: operation.requestId, - location: operation.location, - label: operation.endpoint ?? operation.method, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { - endpoint: operation.endpoint, - method: operation.method, - operation_kind: operation.kind, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: operation.kind === "request" ? "request" : "response", + roleKey: operation.requestId, + location: operation.location, + label: operation.endpoint ?? operation.method, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { + endpoint: operation.endpoint, + method: operation.method, + operation_kind: operation.kind, + }, + }); const projectRequestConstruction = ( context: SemanticFlowProjectionContext, @@ -265,28 +250,21 @@ const addBoundaryNode = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticBoundaryOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "boundary", - roleKey: operation.boundaryId, - location: operation.location, - label: operation.method, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { - boundary_kind: operation.kind, - method: operation.method, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "boundary", + roleKey: operation.boundaryId, + location: operation.location, + label: operation.method, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { + boundary_kind: operation.kind, + method: operation.method, + }, + }); const bindingOrOwner = ( context: SemanticFlowProjectionContext, diff --git a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts index fba83df6b0..b8c685bc54 100644 --- a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts @@ -5,17 +5,15 @@ import { sourceRangesEqual } from "../../domain/javascript/javascriptStaticAnaly import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, type SemanticGraphProjectionState, } from "./JavaScriptSemanticGraphConstruction.js"; import { inferredSemanticEvidenceAt, unknownSemanticEvidence, } from "./JavaScriptSemanticGraphEvidence.js"; -import { semanticNodesWithinRange } from "./JavaScriptSemanticGraphProjection.js"; /** File-local graph state needed by return, capture, and frontier projection. */ export interface SemanticFlowProjectionContext { @@ -28,6 +26,12 @@ export interface SemanticFlowProjectionContext { readonly callSiteNodes: ReadonlyMap; readonly returnSiteNodes: ReadonlyMap; readonly referenceNodes: readonly JavaScriptSemanticGraphNode[]; + readonly referenceNodesWithin: ( + range: JavaScriptSourceRange, + ) => JavaScriptSemanticGraphNode[]; + readonly callSiteAt: ( + range: JavaScriptSourceRange, + ) => JavaScriptSemanticGraphNode | undefined; } /** Find the retained call-site node at one exact semantic source range. */ @@ -35,12 +39,7 @@ export const semanticCallSiteAt = ( context: SemanticFlowProjectionContext, location: JavaScriptSourceRange, ): JavaScriptSemanticGraphNode | undefined => { - const call = context.ir.callSites.find(({ location: candidate }) => - sourceRangesEqual(candidate, location), - ); - return call === undefined - ? undefined - : context.callSiteNodes.get(call.callSiteId); + return context.callSiteAt(location); }; /** Project explicit static Promise/task ownership without runtime claims. */ @@ -53,26 +52,19 @@ export const projectSemanticPromises = ( operation.ownerCallableId === null ? undefined : context.callableNodes.get(operation.ownerCallableId); - const node = addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: operation.kind === "awaited-expression" ? "task" : "promise", - roleKey: operation.promiseId, - location: operation.location, - label: promiseLabel(operation), - functionNodeId: owner?.node_id ?? null, - properties: { - method: operation.method, - operation_kind: operation.kind, - ownership: operation.ownership, - source_resolution: operation.sourceResolution, - }, - }, - context.state, - ), - ); + const node = retainSemanticGraphNode(context.state, context.file, { + kind: operation.kind === "awaited-expression" ? "task" : "promise", + roleKey: operation.promiseId, + location: operation.location, + label: promiseLabel(operation), + functionNodeId: owner?.node_id ?? null, + properties: { + method: operation.method, + operation_kind: operation.kind, + ownership: operation.ownership, + source_resolution: operation.sourceResolution, + }, + }); return node === null ? [] : [[operation.promiseId, node] as const]; }), ); @@ -217,10 +209,7 @@ export const projectSemanticReturnValues = ( for (const callable of context.ir.callables) for (const site of callable.returnSites) { const returnNode = context.returnSiteNodes.get(site.returnSiteId); - const references = semanticNodesWithinRange( - context.referenceNodes, - site.location, - ); + const references = context.referenceNodesWithin(site.location); for (const reference of references) addSemanticGraphRelation(context.state, { source: reference, diff --git a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts index e0a5d8890b..e0e3b78e8a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts @@ -2,10 +2,9 @@ import { createJavaScriptSemanticGraphUnknown } from "../../domain/javascript/ja import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { JavaScriptSemanticObjectOperation } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlowProjection.js"; @@ -72,31 +71,24 @@ const addObjectOccurrence = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticObjectOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "expression", - roleKey: operation.objectOperationId, - location: operation.location, - label: - operation.propertyName === null - ? operation.kind - : `${operation.kind}:${operation.propertyName}`, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { - operation_kind: operation.kind, - property_name: operation.propertyName, - }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "expression", + roleKey: operation.objectOperationId, + location: operation.location, + label: + operation.propertyName === null + ? operation.kind + : `${operation.kind}:${operation.propertyName}`, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { + operation_kind: operation.kind, + property_name: operation.propertyName, + }, + }); const addObjectUnknown = ( context: SemanticFlowProjectionContext, diff --git a/src/application/javascript/JavaScriptSemanticGraphProjection.test.ts b/src/application/javascript/JavaScriptSemanticGraphProjection.test.ts new file mode 100644 index 0000000000..086319bdca --- /dev/null +++ b/src/application/javascript/JavaScriptSemanticGraphProjection.test.ts @@ -0,0 +1,259 @@ +import { expect, it } from "vitest"; + +import { compareCodePoints } from "../../domain/canonicalOrdering.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { analyzeJavaScriptSemantics } from "../../domain/javascript/javascriptSemanticAnalysis.js"; +import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraph.js"; +import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; +import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import { + constructSemanticGraphNode, + createSemanticGraphProjectionState, +} from "./JavaScriptSemanticGraphConstruction.js"; +import { + createSemanticCallableOwnerLookup, + createSemanticNodeRangeLookup, + createSemanticCallSiteLookup, +} from "./JavaScriptSemanticGraphProjection.js"; +import { sourceRangesEqual } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js"; + +const fixtureFile: JavaScriptArtifactFile = { + path: "app.js", + container_sha256: "a".repeat(64), + sha256: "b".repeat(64), + bytes: 0, + inventory_artifact_id: "fixture-app", + kind: "javascript", + unpacked: false, + text: { included: true, value: "" }, +}; + +const nodesFor = (ir: JavaScriptSemanticIr) => { + const state = createSemanticGraphProjectionState({ nodes: [] }); + + return new Map( + ir.callables.map((callable) => [ + callable.callableId, + constructSemanticGraphNode( + fixtureFile, + { + kind: "function", + roleKey: callable.callableId, + label: callable.name, + location: callable.location, + functionNodeId: null, + }, + state, + ), + ]), + ); +}; + +// Reference behavior before the optimization, including stable ties and +// fallback when the most deeply nested callable has no retained graph node. +const previousOwner = ( + location: JavaScriptSourceRange | null, + ir: JavaScriptSemanticIr, + nodes: ReadonlyMap, +) => { + if (location === null) return undefined; + const compare = ( + left: JavaScriptSourceRange["start"], + right: JavaScriptSourceRange["start"], + ) => left.line - right.line || left.column - right.column; + const key = (range: JavaScriptSourceRange) => + `${String(range.start.line).padStart(12, "0")}:${String(range.start.column).padStart(12, "0")}`; + return ir.callables + .filter( + (callable) => + compare(callable.location.start, location.start) <= 0 && + compare(callable.location.end, location.end) >= 0, + ) + .sort((left, right) => + compareCodePoints(key(right.location), key(left.location)), + ) + .map((callable) => nodes.get(callable.callableId)) + .find((node) => node !== undefined); +}; + +const rangesFor = (ir: JavaScriptSemanticIr) => [ + null, + ...ir.callables.map((callable) => callable.location), + ...ir.bindings.flatMap((binding) => + binding.definitions.map((definition) => definition.location), + ), + ...ir.references.map((reference) => reference.location), + { start: { line: 999, column: 0 }, end: { line: 999, column: 1 } }, +]; + +it("matches the prior owner selection across nested functions, arrows and methods", () => { + const ir = analyzeJavaScriptSemantics(` + const outside = 1; + function outer(parameter) { + function inner(value) { return value + parameter; } + const callback = argument => inner(argument); + return callback(outside); + } + class Example { method(value) { return () => outer(value); } } + `); + const nodes = nodesFor(ir); + const lookup = createSemanticCallableOwnerLookup(ir, nodes); + for (const range of rangesFor(ir)) + expect(lookup(range)).toBe(previousOwner(range, ir, nodes)); +}); + +it("reads retained node membership at query time and keeps file indexes independent", () => { + const ir = analyzeJavaScriptSemantics( + "function outer() { function inner() { return 1; } return inner(); }", + ); + const inner = ir.callables.find((callable) => callable.name === "inner"); + if (inner === undefined) throw new Error("Missing inner fixture callable"); + const nodes = nodesFor(ir); + const otherNodes = nodesFor(ir); + const lookup = createSemanticCallableOwnerLookup(ir, nodes); + const otherLookup = createSemanticCallableOwnerLookup(ir, otherNodes); + const before = lookup(inner.location); + expect(before?.label).toBe("inner"); + nodes.delete(inner.callableId); + expect(lookup(inner.location)?.label).toBe("outer"); + expect(otherLookup(inner.location)?.label).toBe("inner"); + if (before === undefined) throw new Error("Missing retained fixture node"); + nodes.set(inner.callableId, before); + expect(lookup(inner.location)).toBe(before); +}); + +it("preserves stable ties and non-nested overlapping ranges", () => { + const original = analyzeJavaScriptSemantics( + "function first() { return 1; } function second() { return 2; } function third() { return 3; }", + ); + const range = (start: number, end: number): JavaScriptSourceRange => ({ + start: { line: 1, column: start }, + end: { line: 1, column: end }, + }); + const ir = { + ...original, + callables: original.callables.map((callable, index) => ({ + ...callable, + location: index === 0 ? range(0, 90) : range(20, index === 1 ? 60 : 80), + })), + }; + const nodes = nodesFor(ir); + const lookup = createSemanticCallableOwnerLookup(ir, nodes); + for (const query of [ + range(25, 30), + range(25, 70), + range(0, 80), + range(80, 90), + range(91, 92), + ]) + expect(lookup(query)).toBe(previousOwner(query, ir, nodes)); +}); + +it("matches the prior result across a compiled-style collection of sibling callables", () => { + const ir = analyzeJavaScriptSemantics( + Array.from( + { length: 300 }, + (_, index) => + `function f${String(index)}(value) { return value + ${String(index)}; }`, + ).join(""), + ); + const nodes = nodesFor(ir); + const lookup = createSemanticCallableOwnerLookup(ir, nodes); + for (const range of rangesFor(ir)) + expect(lookup(range)).toBe(previousOwner(range, ir, nodes)); +}); + +it("returns contained nodes in original order, including repeated and overlapping entries", () => { + const ir = analyzeJavaScriptSemantics(` + function outer(value) { function inner() { return value; } return inner(); } + const sibling = () => outer(1); + `); + const nodes = [...nodesFor(ir).values()].reverse(); + const repeated = nodes[0]; + if (repeated === undefined) throw new Error("Missing range fixture node"); + nodes.push( + repeated, + constructSemanticGraphNode( + fixtureFile, + { + kind: "module", + roleKey: "module", + location: null, + label: "module", + functionNodeId: null, + }, + createSemanticGraphProjectionState({ nodes: [] }), + ), + ); + const lookup = createSemanticNodeRangeLookup(nodes); + const compare = ( + left: JavaScriptSourceRange["start"], + right: JavaScriptSourceRange["start"], + ) => left.line - right.line || left.column - right.column; + for (const range of rangesFor(ir)) { + if (range === null) continue; + const expected = nodes.filter( + ({ identity }) => + identity.source_range !== null && + compare(range.start, identity.source_range.start) <= 0 && + compare(range.end, identity.source_range.end) >= 0, + ); + expect(lookup(range)).toEqual(expected); + } + expect( + createSemanticNodeRangeLookup([])( + repeated.identity.source_range ?? { + start: { line: 1, column: 0 }, + end: { line: 1, column: 1 }, + }, + ), + ).toEqual([]); +}); + +it("keeps first-call matching and current retained membership for exact locations", () => { + const original = analyzeJavaScriptSemantics( + "function target(value) { return value; } target(1); target(2);", + ); + const first = original.callSites[0]; + const node = [...nodesFor(original).values()][0]; + if (first === undefined || node === undefined) + throw new Error("Missing call fixture"); + const duplicate = { ...first, callSiteId: `${first.callSiteId}:duplicate` }; + const ir = { + ...original, + callSites: [first, duplicate, ...original.callSites.slice(1)], + }; + const nodes = new Map( + ir.callSites.map((call) => [ + call.callSiteId, + constructSemanticGraphNode( + fixtureFile, + { + kind: "call-site", + roleKey: call.callSiteId, + location: call.location, + label: call.callSiteId, + functionNodeId: null, + }, + createSemanticGraphProjectionState({ nodes: [] }), + ), + ]), + ); + const lookup = createSemanticCallSiteLookup(ir, nodes); + const previous = (range: JavaScriptSourceRange) => { + const call = ir.callSites.find((candidate) => + sourceRangesEqual(candidate.location, range), + ); + return call === undefined ? undefined : nodes.get(call.callSiteId); + }; + for (const call of ir.callSites) + expect(lookup(call.location)).toBe(previous(call.location)); + nodes.delete(first.callSiteId); + expect(lookup(first.location)).toBeUndefined(); + expect(nodes.has(duplicate.callSiteId)).toBe(true); + nodes.set(first.callSiteId, node); + expect(lookup(first.location)).toBe(node); + expect( + lookup({ start: { line: 99, column: 0 }, end: { line: 99, column: 1 } }), + ).toBeUndefined(); +}); diff --git a/src/application/javascript/JavaScriptSemanticGraphProjection.ts b/src/application/javascript/JavaScriptSemanticGraphProjection.ts index 4d09591e4a..ac0b4e519c 100644 --- a/src/application/javascript/JavaScriptSemanticGraphProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphProjection.ts @@ -5,40 +5,173 @@ import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSem import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; -/** Find the innermost callable that contains one exact source range. */ -export const owningSemanticCallableNode = ( - location: JavaScriptSourceRange | null, +interface SourceInterval { + readonly range: JavaScriptSourceRange; + readonly order: number; +} + +interface CallableInterval extends SourceInterval { + readonly callableId: string; + readonly priority: string; +} + +interface SourceIntervalTree { + readonly entry: Interval; + readonly minimumStart: JavaScriptSourceRange["start"]; + readonly maximumStart: JavaScriptSourceRange["start"]; + readonly minimumEnd: JavaScriptSourceRange["end"]; + readonly maximumEnd: JavaScriptSourceRange["end"]; + readonly left: SourceIntervalTree | null; + readonly right: SourceIntervalTree | null; +} + +/** Index one immutable file IR; read retained callable membership at query time. */ +export const createSemanticCallableOwnerLookup = ( ir: JavaScriptSemanticIr, nodes: ReadonlyMap, -): JavaScriptSemanticGraphNode | undefined => { - if (location === null) return undefined; - return ir.callables - .filter((callable) => contains(callable.location, location)) - .sort((left, right) => - compareCodePoints( - positionKey(right.location), - positionKey(left.location), - ), - ) - .map(({ callableId }) => nodes.get(callableId)) - .find((node) => node !== undefined); +): (( + location: JavaScriptSourceRange | null, +) => JavaScriptSemanticGraphNode | undefined) => { + const entries = ir.callables + .map((callable, order): CallableInterval => ({ + callableId: callable.callableId, + range: callable.location, + order, + priority: positionKey(callable.location), + })) + .sort( + (left, right) => + pointCompare(left.range.start, right.range.start) || + left.order - right.order, + ); + const tree = buildSourceIntervalTree(entries, 0, entries.length); + return (location) => { + if (location === null || tree === null || nodes.size === 0) + return undefined; + const pending = [tree]; + let best: CallableInterval | undefined; + let result: JavaScriptSemanticGraphNode | undefined; + while (pending.length > 0) { + const current = pending.pop(); + if ( + current === undefined || + pointCompare(current.minimumStart, location.start) > 0 || + pointCompare(current.maximumEnd, location.end) < 0 + ) + continue; + const candidate = current.entry; + if (contains(candidate.range, location)) { + const node = nodes.get(candidate.callableId); + const rank = + best === undefined + ? 1 + : compareCodePoints(candidate.priority, best.priority) || + best.order - candidate.order; + if (node !== undefined && rank > 0) { + best = candidate; + result = node; + } + } + if (current.left !== null) pending.push(current.left); + if (current.right !== null) pending.push(current.right); + } + return result; + }; }; -/** Find retained semantic nodes whose exact ranges are within one expression. */ -export const semanticNodesWithinRange = ( +const buildSourceIntervalTree = ( + entries: readonly Interval[], + start: number, + end: number, +): SourceIntervalTree | null => { + if (start >= end) return null; + const middle = Math.floor((start + end) / 2); + const entry = entries[middle]; + if (entry === undefined) return null; + // This recursion follows a balanced index, not the producer's nesting depth. + const left = buildSourceIntervalTree(entries, start, middle); + const right = buildSourceIntervalTree(entries, middle + 1, end); + let minimumEnd = entry.range.end; + let maximumEnd = entry.range.end; + for (const child of [left, right]) { + if (child !== null && pointCompare(child.minimumEnd, minimumEnd) < 0) + minimumEnd = child.minimumEnd; + if (child !== null && pointCompare(child.maximumEnd, maximumEnd) > 0) + maximumEnd = child.maximumEnd; + } + return { + entry, + minimumStart: left?.minimumStart ?? entry.range.start, + maximumStart: right?.maximumStart ?? entry.range.start, + minimumEnd, + maximumEnd, + left, + right, + }; +}; + +/** Index an immutable node list; return contained ranges in producer order. */ +export const createSemanticNodeRangeLookup = ( nodes: readonly JavaScriptSemanticGraphNode[], +): ((range: JavaScriptSourceRange) => JavaScriptSemanticGraphNode[]) => { + const entries = nodes + .flatMap((node, order) => { + const range = node.identity.source_range; + return range === null ? [] : [{ node, range, order }]; + }) + .sort( + (left, right) => + pointCompare(left.range.start, right.range.start) || + left.order - right.order, + ); + const tree = buildSourceIntervalTree(entries, 0, entries.length); + return (range) => { + if (tree === null) return []; + const pending = [tree]; + const matches: (typeof entries)[number][] = []; + while (pending.length > 0) { + const current = pending.pop(); + if ( + current === undefined || + pointCompare(current.maximumStart, range.start) < 0 || + pointCompare(current.minimumEnd, range.end) > 0 + ) + continue; + if (contains(range, current.entry.range)) matches.push(current.entry); + if (current.left !== null) pending.push(current.left); + if (current.right !== null) pending.push(current.right); + } + return matches + .sort((left, right) => left.order - right.order) + .map(({ node }) => node); + }; +}; + +/** Index exact call ranges, preserving the first producer occurrence. */ +export const createSemanticCallSiteLookup = ( + ir: JavaScriptSemanticIr, + nodes: ReadonlyMap, +): (( range: JavaScriptSourceRange, -): JavaScriptSemanticGraphNode[] => - nodes.filter( - (node) => - node.identity.source_range !== null && - contains(range, node.identity.source_range), - ); +) => JavaScriptSemanticGraphNode | undefined) => { + const identifiers = new Map(); + for (const call of ir.callSites) { + const key = rangeKey(call.location); + if (!identifiers.has(key)) identifiers.set(key, call.callSiteId); + } + return (range) => { + const identifier = identifiers.get(rangeKey(range)); + return identifier === undefined ? undefined : nodes.get(identifier); + }; +}; + +const rangeKey = (range: JavaScriptSourceRange): string => + `${range.start.line}:${range.start.column}-${range.end.line}:${range.end.column}`; /** Report extractor support without treating missing families as absence. */ export const semanticFamilyStatus = ( family: (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number], - analysis: JavaScriptArtifactAnalysis, + analysis: Pick, ): "complete" | "partial" | "unknown" | "unsupported" => { if ( ![ diff --git a/src/application/javascript/JavaScriptSemanticGraphResourceProjection.ts b/src/application/javascript/JavaScriptSemanticGraphResourceProjection.ts index 22ca8aff6b..8689cc6739 100644 --- a/src/application/javascript/JavaScriptSemanticGraphResourceProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphResourceProjection.ts @@ -2,10 +2,9 @@ import { createJavaScriptSemanticGraphUnknown } from "../../domain/javascript/ja import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { JavaScriptSemanticResourceOperation } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, addSemanticGraphUnknown, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; import { @@ -39,25 +38,18 @@ const addResourceNode = ( context: SemanticFlowProjectionContext, operation: JavaScriptSemanticResourceOperation, ): JavaScriptSemanticGraphNode | null => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "resource", - roleKey: operation.resourceId, - location: operation.location, - label: operation.method, - functionNodeId: - operation.ownerCallableId === null - ? null - : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? - null), - properties: { method: operation.method }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "resource", + roleKey: operation.resourceId, + location: operation.location, + label: operation.method, + functionNodeId: + operation.ownerCallableId === null + ? null + : (context.callableNodes.get(operation.ownerCallableId)?.node_id ?? + null), + properties: { method: operation.method }, + }); const projectRelease = ( context: SemanticFlowProjectionContext, diff --git a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts index 96298f3496..bcda8680e5 100644 --- a/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphValueProjection.ts @@ -4,9 +4,8 @@ import type { JavaScriptSemanticValue, } from "../../domain/javascript/javascriptSemanticIr.js"; import { - addSemanticGraphNode, + retainSemanticGraphNode, addSemanticGraphRelation, - constructSemanticGraphNode, } from "./JavaScriptSemanticGraphConstruction.js"; import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlowProjection.js"; @@ -85,22 +84,15 @@ const addLiteralNode = ( value: string | number | boolean | null, role: string, ) => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "literal", - roleKey: `literal:${binding.bindingId}:${role}:${JSON.stringify(value)}`, - location: binding.definitions[0]?.location ?? null, - label: JSON.stringify(value), - functionNodeId: - context.bindingNodes.get(binding.bindingId)?.function_node_id ?? null, - properties: { value }, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "literal", + roleKey: `literal:${binding.bindingId}:${role}:${JSON.stringify(value)}`, + location: binding.definitions[0]?.location ?? null, + label: JSON.stringify(value), + functionNodeId: + context.bindingNodes.get(binding.bindingId)?.function_node_id ?? null, + properties: { value }, + }); /** Create one canonical property slot for a binding/property identity. */ export const semanticPropertySlot = ( @@ -108,22 +100,15 @@ export const semanticPropertySlot = ( objectBindingId: string, name: string, ) => - addSemanticGraphNode( - context.state, - constructSemanticGraphNode( - context.file, - { - kind: "property-slot", - roleKey: `property:${objectBindingId}:${name}`, - location: null, - label: name, - functionNodeId: - context.bindingNodes.get(objectBindingId)?.function_node_id ?? null, - properties: { - name, - object_binding_id: objectBindingId, - } satisfies Readonly>, - }, - context.state, - ), - ); + retainSemanticGraphNode(context.state, context.file, { + kind: "property-slot", + roleKey: `property:${objectBindingId}:${name}`, + location: null, + label: name, + functionNodeId: + context.bindingNodes.get(objectBindingId)?.function_node_id ?? null, + properties: { + name, + object_binding_id: objectBindingId, + } satisfies Readonly>, + }); diff --git a/src/cli/binaryDiagnosticsCommands.ts b/src/cli/binaryDiagnosticsCommands.ts index e6f63f7efe..560a119356 100644 --- a/src/cli/binaryDiagnosticsCommands.ts +++ b/src/cli/binaryDiagnosticsCommands.ts @@ -1,6 +1,9 @@ import { z } from "incur"; import { resolve } from "node:path"; -import { createBinaryLayoutService } from "../composition/binaryDiagnostics.js"; +import { + createRecordedCrashService, + createBinaryLayoutService, +} from "../composition/binaryDiagnostics.js"; import { CLI_COMMANDS } from "../cliCommandNames.js"; import { logCliCommand } from "../cliLogging.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -32,4 +35,29 @@ export const registerBinaryDiagnosticsCommands = ( }), ), }); + const recorded = createRecordedCrashService(environment); + cli.command(CLI_COMMANDS.inspectRecordedCrash, { + description: + "Inspect supplied core registers/signals and optional debugger mapping evidence", + args: z.object({ path: z.string().describe("Local Linux amd64 ELF core") }), + options: z.object({ + debuggerContext: z + .boolean() + .default(false) + .describe("Add BYO core-only GDB/pwndbg mapping candidates"), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectRecordedCrash, async () => { + const result = await recorded.inspect( + { + path: resolve(args.path), + include_debugger_context: options.debuggerContext, + }, + { signal }, + ); + return result.ok ? result.value : projectAnalysisError(result.error); + }), + ), + }); }; diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts index 0e7e755039..cc401ee321 100644 --- a/src/cli/jsonOutput.ts +++ b/src/cli/jsonOutput.ts @@ -61,6 +61,10 @@ function* bufferedParts(parts: Iterable): Generator { } function* encodeString(value: string): Generator { + if (value.length <= 8192) { + yield JSON.stringify(value); + return; + } yield '"'; for (let start = 0; start < value.length;) { let end = Math.min(start + 8192, value.length); diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 83d2194460..61a85bb25a 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -39,6 +39,7 @@ export const CLI_COMMANDS = Object.freeze({ resolveNativeCallTargets: "resolve-native-call-targets", inspectAssetCatalog: "inspect-asset-catalog", inspectManagedArtifact: "inspect-managed-artifact", + inspectRecordedCrash: "inspect-recorded-crash", inspectBinaryLayout: "inspect-binary-layout", inspectFirmwareRegions: "inspect-firmware-regions", extractFirmware: "extract-firmware", diff --git a/src/composition/binaryDiagnostics.ts b/src/composition/binaryDiagnostics.ts index cece28a5e1..7fab8d73a2 100644 --- a/src/composition/binaryDiagnostics.ts +++ b/src/composition/binaryDiagnostics.ts @@ -1,3 +1,5 @@ +import { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; +import { PwntoolsRecordedCrashProvider } from "../native/pwntools/PwntoolsRecordedCrashProvider.js"; import { BinaryLayoutService } from "../application/binaryDiagnostics/BinaryLayoutService.js"; import { PwntoolsLayoutProvider } from "../native/pwntools/PwntoolsLayoutProvider.js"; @@ -6,3 +8,9 @@ export const createBinaryLayoutService = ( environment: Readonly = process.env, ): BinaryLayoutService => new BinaryLayoutService(new PwntoolsLayoutProvider(environment)); + +/** Compose recorded-core inspection without acquiring a debugger or toolchain. */ +export const createRecordedCrashService = ( + environment: Readonly = process.env, +): RecordedCrashService => + new RecordedCrashService(new PwntoolsRecordedCrashProvider(environment)); diff --git a/src/contracts/native/binaryDiagnosticsToolContracts.ts b/src/contracts/native/binaryDiagnosticsToolContracts.ts index c49780c03a..28d199a8b8 100644 --- a/src/contracts/native/binaryDiagnosticsToolContracts.ts +++ b/src/contracts/native/binaryDiagnosticsToolContracts.ts @@ -1,3 +1,7 @@ +import { + inspectRecordedCrashInputSchema, + recordedCrashSchema, +} from "../../domain/native/recordedCrash.js"; import { inspectBinaryLayoutInputSchema, binaryLayoutSchema, @@ -23,4 +27,26 @@ export const BINARY_DIAGNOSTICS_TOOL_CONTRACTS = [ }, ], }, + { + name: "inspect_recorded_crash", + ...toolContractMetadata("inspect_recorded_crash"), + kind: "native-provider", + description: + "Inspect a supplied Linux ELF64 amd64 little-endian core without launching its target or attaching to any live PID. Returns all recorded segments, raw note owners/descriptors and source ranges, every Linux CORE PRSTATUS thread/register and SIGINFO fields inline with SHA-256 Evidence. Historical PID is recorded metadata; current process/file identity, note coverage and signal-thread associations stay unknown. Optionally add derived core-only GDB/pwndbg mapping candidates through explicit BYO configuration; zero map flags mean unknown permissions. Uses the same stable snapshot for both stages, bounded complete output and owned cleanup. Initial Linux x64 profile: unchanged pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2; optional unchanged pwndbg 2026.09.15. No implicit executable/library pairing, GDB command language, host configuration changes or dependency installation.", + inputSchema: inspectRecordedCrashInputSchema, + outputSchema: evidenceResultOf(recordedCrashSchema), + examples: [ + { + title: "Inspect recorded registers and signal evidence", + input: { path: "/artifacts/crash.core" }, + }, + { + title: "Add core-only debugger mapping candidates", + input: { + path: "/artifacts/crash.core", + include_debugger_context: true, + }, + }, + ], + }, ] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index fd416c7c6c..558f4551fa 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -70,6 +70,11 @@ export const TOOL_EFFECTS: Readonly> = { launchesProcess: true, accessesNetwork: true, }), + inspect_recorded_crash: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + }), inspect_binary_layout: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index 29a629d3d6..d5173c1946 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -17,6 +17,31 @@ const TARGET: BinaryTarget = { const PROVIDER = { id: "fixture", name: "Fixture provider", version: "1" }; const PROFILE = createAnalysisProfile(PROVIDER, { loader: "default" }); +it("snapshots caller-owned payloads before deriving their identity", () => { + const result = { nested: { values: ["observed"] } }; + const rawResult = { entries: [{ value: "raw" }] }; + const parameters = { selected: ["target"] }; + const evidence = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters, + result, + rawResult, + }); + result.nested.values.push("later"); + const rawEntry = rawResult.entries[0]; + if (rawEntry !== undefined) rawEntry.value = "changed"; + parameters.selected.push("other"); + expect(evidence.normalized_result).toEqual({ + nested: { values: ["observed"] }, + }); + expect(evidence.raw_result).toEqual({ entries: [{ value: "raw" }] }); + expect(evidence.parameters).toEqual({ selected: ["target"] }); + expect(parseEvidence(evidence)).toEqual(evidence); + expect(() => + parseEvidence({ ...evidence, normalized_result: result }), + ).toThrow(/semantic identifier/u); +}); + describe("analysis evidence identity", () => { it("normalizes prototype-named parameter keys", () => { const evidence = createEvidence(TARGET, PROVIDER, { diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 6348fc38a8..844c9e4882 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -274,13 +274,23 @@ export const createEvidence = ( locations: [...(observation.locations ?? [])], evidence_links: [...(observation.evidenceLinks ?? [])], } satisfies JsonValue; - const normalized = evidenceRecordSchema.parse({ + // Select the known envelope before parsing. A union otherwise traverses and + // clones a legacy result for the profiled branch before rejecting its absent + // profile and trying the legacy branch. + const schema = + observation.analysisProfile === undefined + ? evidenceBaseSchema + : profiledEvidenceSchema; + const normalized = schema.parse({ ...semantic, evidence_id: `ev_${"0".repeat(64)}`, subject, }); - return parseEvidence({ + // The envelope has already been parsed into an independent snapshot. Only + // its derived identifier changes here; parsing again clones the full payload + // and recomputes the same digest while the previous snapshot is still live. + return { ...normalized, evidence_id: computeEvidenceId(normalized), - }); + }; }; diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index a7af45a995..86884cba7c 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -38,7 +38,8 @@ const normalizeEvidence = ( evidence_ids: uniqueSorted(evidence.evidence_ids), }); -const nodeId = ( +/** Derive a semantic node ID from its kind and exact artifact identity. */ +export const javaScriptSemanticNodeId = ( node: Pick, ): string => `jsrg_node_${canonicalDigest({ kind: node.kind, identity: node.identity }, "JavaScript semantic graph")}`; @@ -59,7 +60,7 @@ export const createJavaScriptSemanticGraphNode = ( }; return javaScriptSemanticNodeSchema.parse({ ...semantic, - node_id: nodeId(semantic), + node_id: javaScriptSemanticNodeId(semantic), }); }; @@ -129,11 +130,21 @@ export const createJavaScriptSemanticFingerprint = (input: unknown) => { type GraphRecord = z.infer; +type SemanticGraphIssue = { + readonly code: "custom"; + readonly path: PropertyKey[]; + readonly message: string; +}; + +interface GraphIssueReporter { + readonly addIssue: (issue: SemanticGraphIssue) => void; +} + const sortedUniqueIssue = ( values: readonly string[], path: PropertyKey[], label: string, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (let index = 1; index < values.length; index += 1) { if (compareCodePoints(values[index - 1] ?? "", values[index] ?? "") < 0) @@ -147,7 +158,10 @@ const sortedUniqueIssue = ( } }; -const checkCoverage = (graph: GraphRecord, context: z.RefinementCtx): void => { +const checkCoverage = ( + graph: GraphRecord, + context: GraphIssueReporter, +): void => { const families = graph.coverage.families.map(({ family }) => family); if ( canonicalJson(families, "JavaScript semantic graph") !== @@ -221,7 +235,7 @@ const checkCoverage = (graph: GraphRecord, context: z.RefinementCtx): void => { const checkCanonicalOrder = ( graph: GraphRecord, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { sortedUniqueIssue( graph.root_node_ids, @@ -259,10 +273,10 @@ const checkCanonicalOrder = ( const checkNodes = ( graph: GraphRecord, nodes: ReadonlyMap, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (const [index, node] of graph.nodes.entries()) { - if (node.node_id !== nodeId(node)) + if (node.node_id !== javaScriptSemanticNodeId(node)) context.addIssue({ code: "custom", path: ["nodes", index, "node_id"], @@ -283,7 +297,7 @@ const checkNodes = ( const checkRoots = ( graph: GraphRecord, nodes: ReadonlyMap, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (const root of graph.root_node_ids) if (!nodes.has(root)) @@ -297,7 +311,7 @@ const checkRoots = ( const checkRelations = ( graph: GraphRecord, nodes: ReadonlyMap, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (const [index, relation] of graph.relations.entries()) { const { relation_id: identifier, ...semantic } = relation; @@ -340,7 +354,7 @@ const checkRelations = ( const checkUnknowns = ( graph: GraphRecord, nodes: ReadonlyMap, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (const [index, unknown] of graph.unknowns.entries()) { const { unknown_id: identifier, ...semantic } = unknown; @@ -386,7 +400,7 @@ const checkUnknowns = ( const checkUnknownReferences = ( graph: GraphRecord, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { const unknownIds = new Set( graph.unknowns.map(({ unknown_id }) => unknown_id), @@ -403,7 +417,7 @@ const checkUnknownReferences = ( const checkFingerprints = ( graph: GraphRecord, nodes: ReadonlyMap, - context: z.RefinementCtx, + context: GraphIssueReporter, ): void => { for (const [index, fingerprint] of graph.fingerprints.entries()) { if (nodes.get(fingerprint.function_node_id)?.kind !== "function") @@ -449,7 +463,10 @@ const checkFingerprints = ( } }; -const checkGraph = (graph: GraphRecord, context: z.RefinementCtx): void => { +const checkGraphContent = ( + graph: GraphRecord, + context: GraphIssueReporter, +): void => { checkCanonicalOrder(graph, context); const nodes = new Map(graph.nodes.map((node) => [node.node_id, node])); checkNodes(graph, nodes, context); @@ -465,6 +482,10 @@ const checkGraph = (graph: GraphRecord, context: z.RefinementCtx): void => { path: ["limitations"], message: "Non-complete graph coverage requires a limitation", }); +}; + +const checkGraph = (graph: GraphRecord, context: GraphIssueReporter): void => { + checkGraphContent(graph, context); const { graph_id: identifier, ...semantic } = graph; if ( identifier !== @@ -523,10 +544,21 @@ export const createJavaScriptSemanticGraph = ( }, limitations: uniqueSorted(parsed.limitations), }; - return javaScriptSemanticGraphSchema.parse({ + // The input schema already cloned and validated every field. Validate the + // normalized relationships in place, then derive the only added field. The + // public schema still verifies arbitrary records and their commitments. + const record: GraphRecord = { ...semantic, graph_id: `jsrg_${canonicalDigest(semantic, "JavaScript semantic graph")}`, + }; + const issues: SemanticGraphIssue[] = []; + checkGraphContent(record, { + addIssue: (issue) => { + issues.push(issue); + }, }); + if (issues.length > 0) throw new z.ZodError(issues); + return record; }; export type { JavaScriptSemanticGraphNode, JavaScriptSemanticGraphRelation }; diff --git a/src/domain/native/recordedCrash.test.ts b/src/domain/native/recordedCrash.test.ts new file mode 100644 index 0000000000..7a050d4170 --- /dev/null +++ b/src/domain/native/recordedCrash.test.ts @@ -0,0 +1,284 @@ +import { expect, it } from "vitest"; +import { + inspectRecordedCrashInputSchema, + recordedCrashSchema, +} from "./recordedCrash.js"; +import { + recordedCrashFixture, + recordedCrashSignalFixture, +} from "../../../tests/fixtures/binaryDiagnostics/recordedCrash.js"; + +it("preserves high uint64 registers and historical thread identity", () => { + const value = recordedCrashSchema.parse(recordedCrashFixture()); + expect( + value.threads[0]?.registers.find(({ name }) => name === "rdi")?.value, + ).toBe("0x1122334455667788"); + expect(value.live_process_identity).toBe("unknown"); + expect( + inspectRecordedCrashInputSchema.parse({ path: "/selected.core" }) + .include_debugger_context, + ).toBe(false); +}); + +it("preserves a short zero-filled note-segment tail", () => { + const value = recordedCrashSignalFixture(); + value.segments = value.segments.map((segment) => ({ + ...segment, + file_size: "0x19b", + })); + value.note_padding.push({ + segment_index: 0, + location: { offset: "0x218", bytes: "0x3" }, + bytes_base64: "AAAA", + }); + expect(recordedCrashSchema.safeParse(value).success).toBe(true); +}); + +it.each([ + "missing-thread", + "duplicate-thread", + "missing-signal", + "duplicate-signal", + "missing-register", + "unknown-register", + "duplicated-physical-thread", + "duplicated-physical-signal", + "omitted-physical-note", + "expanded-note-span", + "forged-padding", +])("rejects an incomplete or duplicate interpretation: %s", (scenario) => { + const value = recordedCrashSignalFixture(); + if ( + scenario === "duplicated-physical-thread" || + scenario === "duplicated-physical-signal" + ) { + const noteIndex = scenario === "duplicated-physical-thread" ? 0 : 1; + const note = value.notes[noteIndex]; + if (note === undefined) throw new Error("missing fixture note"); + value.notes.push({ ...note, index: 2 }); + if (noteIndex === 0) + value.threads.push( + ...value.threads.map((thread) => ({ ...thread, note_index: 2 })), + ); + else + value.signals.push( + ...value.signals.map((signal) => ({ ...signal, note_index: 2 })), + ); + } + if ( + ["omitted-physical-note", "expanded-note-span", "forged-padding"].includes( + scenario, + ) + ) { + value.notes = value.notes.filter((note) => note.index === 0); + value.signals = []; + } + if (scenario === "expanded-note-span") + value.notes = value.notes.map((note) => ({ + ...note, + location: { ...note.location, bytes: "0x198" }, + })); + if (scenario === "forged-padding") + value.note_padding.push({ + segment_index: 0, + location: { offset: "0x1e4", bytes: "0x34" }, + bytes_base64: Buffer.alloc(52).toString("base64"), + }); + if (scenario === "missing-thread") value.threads = []; + if (scenario === "duplicate-thread") + value.threads = [...value.threads, ...value.threads]; + if (scenario === "missing-signal") value.signals = []; + if (scenario === "duplicate-signal") + value.signals = [...value.signals, ...value.signals]; + if (scenario === "missing-register") + value.threads = value.threads.map((thread) => ({ + ...thread, + registers: thread.registers.slice(1), + })); + if (scenario === "unknown-register") + value.threads = value.threads.map((thread) => ({ + ...thread, + registers: thread.registers.map((register) => + register.name === "rdi" + ? { ...register, name: "unrecognized" } + : register, + ), + })); + expect(recordedCrashSchema.safeParse(value).success).toBe(false); +}); + +it.each([ + [ + "unsafe register number", + (v: ReturnType) => ({ + ...v, + threads: [ + { + ...v.threads[0], + registers: [ + { + name: "rdi", + value: 9007199254740992, + location: { offset: "0x104", bytes: "0x8" }, + }, + ], + }, + ], + }), + ], + [ + "source outside artifact", + (v: ReturnType) => ({ + ...v, + artifact: { ...v.artifact, bytes: 120 }, + }), + ], + [ + "invalid note reference", + (v: ReturnType) => ({ + ...v, + threads: [{ ...v.threads[0], note_index: 1 }], + }), + ], + [ + "register outside descriptor", + (v: ReturnType) => ({ + ...v, + threads: [ + { + ...v.threads[0], + registers: [ + { + name: "rdi", + value: "0x0", + location: { offset: "0x1e4", bytes: "0x8" }, + }, + ], + }, + ], + }), + ], + [ + "raw descriptor length", + (v: ReturnType) => ({ + ...v, + notes: v.notes.map((n) => ({ ...n, descriptor_bytes_base64: "AA==" })), + }), + ], + [ + "different owner ABI", + (v: ReturnType) => ({ + ...v, + notes: v.notes.map((n) => ({ ...n, owner_display: "OTHER" })), + }), + ], + [ + "raw owner ABI hidden by a CORE display label", + (v: ReturnType) => ({ + ...v, + notes: v.notes.map((n) => ({ + ...n, + owner_bytes_base64: Buffer.from("EVIL\0").toString("base64"), + })), + }), + ], + [ + "duplicate register", + (v: ReturnType) => ({ + ...v, + threads: v.threads.map((t) => ({ + ...t, + registers: [...t.registers, ...t.registers], + })), + }), + ], +] as const)("rejects %s before Evidence serialization", (_name, change) => + expect( + recordedCrashSchema.safeParse(change(recordedCrashFixture())).success, + ).toBe(false), +); + +it("does not interpret a signal through a mislabeled raw owner ABI", () => { + const value = recordedCrashFixture(); + expect( + recordedCrashSchema.safeParse({ + ...value, + threads: [], + notes: value.notes.map((note) => ({ + ...note, + type: "NT_SIGINFO", + owner_bytes_base64: Buffer.from("EVIL\0").toString("base64"), + })), + signals: [ + { + note_index: 0, + number: 11, + code: 1, + errno: 0, + fault_address: "0x123", + fault_address_meaning: "recorded-sigsegv-address", + thread_association: "unknown", + }, + ], + }).success, + ).toBe(false); +}); + +it("keeps absent map flags unknown and validates reported permission bits", () => { + const value = recordedCrashFixture(); + const context = { + status: "available", + gdb_version: "fixture", + pwndbg_version: "fixture", + connection: "core", + executable: null, + historical_pid: 123, + reported_limits: { + address_space_bytes: 3221225472, + cpu_seconds: 30, + file_size_bytes: 67108864, + }, + maps: [ + { + start: "0x1000", + end: "0x2000", + offset: "0x0", + reported_flags: 0, + permissions: null, + pathname_display: "recorded pathname", + current_file_identity: "unknown", + }, + ], + confidence: "derived", + completeness: "unknown", + diagnostics: value.diagnostics, + } as const; + expect( + recordedCrashSchema.safeParse({ ...value, debugger: context }).success, + ).toBe(true); + expect( + recordedCrashSchema.safeParse({ + ...value, + debugger: { + ...context, + maps: context.maps.map((m) => ({ + ...m, + permissions: { read: false, write: false, execute: false }, + })), + }, + }).success, + ).toBe(false); + expect( + recordedCrashSchema.safeParse({ + ...value, + debugger: { + ...context, + maps: context.maps.map((m) => ({ + ...m, + reported_flags: 4, + permissions: { read: false, write: false, execute: false }, + })), + }, + }).success, + ).toBe(false); +}); diff --git a/src/domain/native/recordedCrash.ts b/src/domain/native/recordedCrash.ts new file mode 100644 index 0000000000..7668e63ce2 --- /dev/null +++ b/src/domain/native/recordedCrash.ts @@ -0,0 +1,362 @@ +import { z } from "zod"; +import { recordedNoteCoverageIssues } from "./recordedCrashNoteCoverage.js"; + +const hex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/); +const index = z.number().int().nonnegative(); +const signed32 = z.number().int().min(-2147483648).max(2147483647); +const base64 = z + .string() + .regex( + /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$/, + ); +const range = z.strictObject({ offset: hex, bytes: hex }); +const diagnostics = z.strictObject({ + stdout: z.string(), + stderr: z.string(), + truncated: z.boolean(), +}); +// Exact b"CORE\0"; a lossy display label cannot establish the Linux note ABI. +const LINUX_CORE_OWNER_BASE64 = "Q09SRQA="; + +/** Complete register identities in the supported Linux amd64 PRSTATUS ABI order. */ +export const RECORDED_LINUX_AMD64_REGISTERS: readonly string[] = [ + "r15", + "r14", + "r13", + "r12", + "rbp", + "rbx", + "r11", + "r10", + "r9", + "r8", + "rax", + "rcx", + "rdx", + "rsi", + "rdi", + "orig_rax", + "rip", + "cs", + "eflags", + "rsp", + "ss", + "fs_base", + "gs_base", + "ds", + "es", + "fs", + "gs", +]; +const supportedRegisters = new Set(RECORDED_LINUX_AMD64_REGISTERS); + +/** Select an explicit recording and optionally request debugger-derived mapping context. */ +export const inspectRecordedCrashInputSchema = z.strictObject({ + path: z + .string() + .min(1) + .describe("Absolute filesystem path to a supplied Linux ELF core"), + include_debugger_context: z + .boolean() + .default(false) + .describe( + "Add core-only mapping candidates through caller-supplied GDB/pwndbg", + ), +}); + +/** Debugger map candidates preserve reported flags; zero does not establish absent permissions. */ +export const recordedCrashDebuggerSchema = z.discriminatedUnion("status", [ + z.strictObject({ status: z.literal("not-requested") }), + z.strictObject({ + status: z.literal("available"), + gdb_version: z.string().min(1), + pwndbg_version: z.string().min(1), + connection: z.literal("core"), + executable: z.null(), + historical_pid: signed32, + reported_limits: z.strictObject({ + address_space_bytes: index, + cpu_seconds: index, + file_size_bytes: index, + }), + maps: z.array( + z.strictObject({ + start: hex, + end: hex, + offset: hex, + reported_flags: index, + permissions: z + .strictObject({ + read: z.boolean(), + write: z.boolean(), + execute: z.boolean(), + }) + .nullable(), + pathname_display: z.string(), + current_file_identity: z.literal("unknown"), + }), + ), + confidence: z.literal("derived"), + completeness: z.literal("unknown"), + diagnostics, + }), +]); + +/** Portable recorded observations; registers and source coordinates remain lossless. */ +const recordedCrashObjectSchema = z.strictObject({ + artifact: z.strictObject({ + path: z.string().min(1), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + bytes: index, + }), + format: z.literal("elf-core"), + architecture: z.literal("x86_64-little-endian"), + target_execution: z.literal("not-performed"), + live_process_identity: z.literal("unknown"), + segments: z.array( + z.strictObject({ + index, + type: z.union([z.string(), index]), + header_location: range, + offset: hex, + file_size: hex, + memory_size: hex, + virtual_address: hex, + physical_address: hex, + alignment: hex, + flags: hex, + file_backing: z.enum(["file", "none"]), + }), + ), + notes: z.array( + z.strictObject({ + index, + segment_index: index, + type: z.union([z.string(), index]), + location: range, + owner_location: range, + owner_bytes_base64: base64, + owner_display: z.string().nullable(), + descriptor_location: range, + descriptor_bytes_base64: base64, + }), + ), + note_padding: z.array( + z.strictObject({ + segment_index: index, + location: range, + bytes_base64: base64, + }), + ), + threads: z.array( + z.strictObject({ + note_index: index, + historical_pid: signed32, + recorded_current_signal: z.number().int().min(-32768).max(32767), + registers: z.array( + z.strictObject({ + name: z.string().min(1), + value: hex, + location: range, + }), + ), + }), + ), + signals: z.array( + z.strictObject({ + note_index: index, + number: signed32, + code: signed32, + errno: signed32, + fault_address: hex.nullable(), + fault_address_meaning: z.enum(["recorded-sigsegv-address", "unknown"]), + thread_association: z.literal("unknown"), + }), + ), + debugger: recordedCrashDebuggerSchema, + note_interpretation_completeness: z.literal("unknown"), + decoder_diagnostics: diagnostics, + diagnostics, + limitations: z.array(z.string()), +}); + +/** Provider payload excludes identity and diagnostics bound by the owned snapshot boundary. */ +export const recordedCrashPayloadSchema = recordedCrashObjectSchema.omit({ + artifact: true, + diagnostics: true, + debugger: true, + decoder_diagnostics: true, +}); + +/** Validate the next consumer's file ranges, note links and register representations. */ +export const recordedCrashSchema = recordedCrashObjectSchema.superRefine( + (value, context) => { + const fail = (message: string) => + context.addIssue({ code: "custom", message }); + const within = ( + item: z.output, + start = 0n, + end = BigInt(value.artifact.bytes), + ) => { + const offset = BigInt(item.offset); + const bytes = BigInt(item.bytes); + return offset >= start && offset <= end && bytes <= end - offset; + }; + for (const issue of recordedNoteCoverageIssues( + value.segments, + value.notes, + value.note_padding, + )) + fail(issue); + for (const [position, segment] of value.segments.entries()) { + if (position !== segment.index || !within(segment.header_location)) + fail("Invalid original segment identity or header range."); + if ( + segment.file_backing === "file" && + !within({ offset: segment.offset, bytes: segment.file_size }) + ) + fail("Recorded segment exceeds artifact bytes."); + if ( + segment.type === "PT_LOAD" && + BigInt(segment.file_size) > BigInt(segment.memory_size) + ) + fail("Recorded PT_LOAD file size exceeds memory size."); + } + for (const [position, note] of value.notes.entries()) { + const segment = value.segments[note.segment_index]; + if (position !== note.index || segment?.type !== "PT_NOTE") { + fail("Invalid original note identity or segment reference."); + continue; + } + const decodedLength = (raw: string) => + BigInt( + (raw.length / 4) * 3 - + (raw.endsWith("==") ? 2 : raw.endsWith("=") ? 1 : 0), + ); + if ( + decodedLength(note.owner_bytes_base64) !== + BigInt(note.owner_location.bytes) || + decodedLength(note.descriptor_bytes_base64) !== + BigInt(note.descriptor_location.bytes) + ) + fail("Raw note lengths differ from their source ranges."); + const start = BigInt(segment.offset); + const end = start + BigInt(segment.file_size); + if ( + !within(note.location, start, end) || + !within( + note.owner_location, + BigInt(note.location.offset), + BigInt(note.location.offset) + BigInt(note.location.bytes), + ) || + !within( + note.descriptor_location, + BigInt(note.location.offset), + BigInt(note.location.offset) + BigInt(note.location.bytes), + ) + ) + fail("Note source ranges exceed their recorded segment."); + } + for (const padding of value.note_padding) { + const segment = value.segments[padding.segment_index]; + if ( + segment?.type !== "PT_NOTE" || + !within( + padding.location, + BigInt(segment.offset), + BigInt(segment.offset) + BigInt(segment.file_size), + ) + ) + fail("Invalid recorded note padding range."); + } + for (const [kind, rows] of [ + ["NT_PRSTATUS", value.threads], + ["NT_SIGINFO", value.signals], + ] as const) { + const matchingNotes = value.notes.filter( + (note) => + note.type === kind && + note.owner_bytes_base64 === LINUX_CORE_OWNER_BASE64, + ); + if ( + rows.length !== matchingNotes.length || + new Set(rows.map(({ note_index }) => note_index)).size !== rows.length + ) + fail( + "Each supported raw note requires exactly one corresponding interpretation.", + ); + } + for (const thread of value.threads) { + const note = value.notes[thread.note_index]; + if ( + note?.type !== "NT_PRSTATUS" || + note.owner_display !== "CORE" || + note.owner_bytes_base64 !== LINUX_CORE_OWNER_BASE64 + ) { + fail("Thread references a different note kind."); + continue; + } + if ( + thread.registers.length !== RECORDED_LINUX_AMD64_REGISTERS.length || + !thread.registers.every(({ name }) => supportedRegisters.has(name)) || + new Set(thread.registers.map(({ name }) => name)).size !== + thread.registers.length + ) + fail( + "Each recorded thread requires every supported register exactly once.", + ); + for (const register of thread.registers) + if ( + BigInt(register.location.bytes) !== 8n || + !within( + register.location, + BigInt(note.descriptor_location.offset), + BigInt(note.descriptor_location.offset) + + BigInt(note.descriptor_location.bytes), + ) + ) + fail("Register source exceeds its descriptor."); + } + for (const signal of value.signals) { + if ( + value.notes[signal.note_index]?.type !== "NT_SIGINFO" || + value.notes[signal.note_index]?.owner_display !== "CORE" || + value.notes[signal.note_index]?.owner_bytes_base64 !== + LINUX_CORE_OWNER_BASE64 + ) + fail("Signal references a different note kind."); + const known = + signal.number === 11 && (signal.code === 1 || signal.code === 2); + if ( + known !== (signal.fault_address !== null) || + (signal.fault_address !== null) !== + (signal.fault_address_meaning === "recorded-sigsegv-address") + ) + fail("Signal union meaning does not match recorded SIGSEGV code."); + } + if (value.debugger.status === "available") + for (const map of value.debugger.maps) { + if (BigInt(map.end) < BigInt(map.start)) + fail("Debugger mapping end precedes its start."); + if ( + map.permissions !== null && + (map.permissions.read !== Boolean(map.reported_flags & 4) || + map.permissions.write !== Boolean(map.reported_flags & 2) || + map.permissions.execute !== Boolean(map.reported_flags & 1)) + ) + fail("Mapping permissions differ from reported flags."); + if ( + (map.reported_flags === 0 || map.reported_flags > 7) !== + (map.permissions === null) + ) + fail( + "Unreported or unfamiliar map flags require unknown permissions.", + ); + } + }, +); + +export type InspectRecordedCrashInput = z.output< + typeof inspectRecordedCrashInputSchema +>; +export type RecordedCrash = z.output; diff --git a/src/domain/native/recordedCrashNoteCoverage.ts b/src/domain/native/recordedCrashNoteCoverage.ts new file mode 100644 index 0000000000..dbe7e51cd4 --- /dev/null +++ b/src/domain/native/recordedCrashNoteCoverage.ts @@ -0,0 +1,111 @@ +type Range = { readonly offset: string; readonly bytes: string }; +type Segment = { + readonly index: number; + readonly type: string | number; + readonly offset: string; + readonly file_size: string; +}; +type Note = { + readonly segment_index: number; + readonly location: Range; + readonly owner_location: Range; + readonly descriptor_location: Range; +}; +type Padding = { + readonly segment_index: number; + readonly location: Range; + readonly bytes_base64: string; +}; +const align = (size: bigint) => ((size + 3n) / 4n) * 4n; +const layoutMatches = (note: Note): boolean => { + const start = BigInt(note.location.offset); + const owner = BigInt(note.owner_location.bytes); + const descriptor = BigInt(note.descriptor_location.bytes); + return ( + BigInt(note.owner_location.offset) === start + 12n && + BigInt(note.descriptor_location.offset) === start + 12n + align(owner) && + BigInt(note.location.bytes) === 12n + align(owner) + align(descriptor) + ); +}; +const paddingMatches = ( + item: Padding, + segment: Segment | undefined, +): boolean => { + const bytes = BigInt(item.location.bytes); + const decodedLength = + (item.bytes_base64.length / 4) * 3 - + (item.bytes_base64.endsWith("==") + ? 2 + : item.bytes_base64.endsWith("=") + ? 1 + : 0); + return ( + bytes > 0n && + bytes < 12n && + BigInt(decodedLength) === bytes && + /^A*={0,2}$/.test(item.bytes_base64) && + segment !== undefined && + BigInt(item.location.offset) + bytes === + BigInt(segment.offset) + BigInt(segment.file_size) + ); +}; + +/** Require exact aligned note layouts and one complete, nonoverlapping physical coverage. */ +export function recordedNoteCoverageIssues( + segments: readonly Segment[], + notes: readonly Note[], + padding: readonly Padding[], +): readonly string[] { + const issues: string[] = []; + const bySegment = new Map(); + const all: Range[] = []; + const add = (segment: number, range: Range) => { + const ranges = bySegment.get(segment) ?? []; + ranges.push(range); + bySegment.set(segment, ranges); + all.push(range); + }; + for (const note of notes) { + if (!layoutMatches(note)) + issues.push( + "Recorded note layout differs from its aligned header/owner/descriptor ranges.", + ); + add(note.segment_index, note.location); + } + for (const item of padding) { + if (!paddingMatches(item, segments[item.segment_index])) + issues.push( + "Recorded note padding must be a short zero-filled segment tail.", + ); + add(item.segment_index, item.location); + } + const compare = (left: Range, right: Range) => + BigInt(left.offset) < BigInt(right.offset) + ? -1 + : BigInt(left.offset) > BigInt(right.offset) + ? 1 + : 0; + let end = 0n; + for (const range of all.sort(compare)) { + const start = BigInt(range.offset); + if (start < end) + issues.push("Physical recorded note/padding ranges overlap or repeat."); + if (start + BigInt(range.bytes) > end) end = start + BigInt(range.bytes); + } + for (const segment of segments) { + if (segment.type !== "PT_NOTE") continue; + let cursor = BigInt(segment.offset); + for (const range of (bySegment.get(segment.index) ?? []).sort(compare)) { + if (BigInt(range.offset) !== cursor) + issues.push( + "Recorded note/padding ranges leave a gap or overlap in their segment.", + ); + cursor = BigInt(range.offset) + BigInt(range.bytes); + } + if (cursor !== BigInt(segment.offset) + BigInt(segment.file_size)) + issues.push( + "Recorded note/padding ranges do not completely cover their segment.", + ); + } + return issues; +} diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index d56f777c43..0840aaf24a 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "32", + skillVersion: "33", } as const; diff --git a/src/native/pwndbg/PwndbgCoreContext.ts b/src/native/pwndbg/PwndbgCoreContext.ts new file mode 100644 index 0000000000..48f940549d --- /dev/null +++ b/src/native/pwndbg/PwndbgCoreContext.ts @@ -0,0 +1,232 @@ +import { access, mkdir, stat, writeFile } from "node:fs/promises"; +import { constants } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; +import type { AnalysisCapturedOutput } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { + recordedCrashDebuggerSchema, + type RecordedCrash, +} from "../../domain/native/recordedCrash.js"; +import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; +import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { + capturedPwntoolsOutput, + pwntoolsDecoderFailure, +} from "../pwntools/PwntoolsFailures.js"; +import { PWNTOOLS_LIMITS } from "../pwntools/PwntoolsRelease.js"; +import { readPwntoolsFailureEvidence } from "../pwntools/PwntoolsResourceLimits.js"; +import type { PwntoolsLauncher } from "../pwntools/PwntoolsDecoder.js"; + +const OPERATION = "inspect_recorded_crash"; +const replySchema = z.discriminatedUnion("ok", [ + z.strictObject({ ok: z.literal(true), value: recordedCrashDebuggerSchema }), + z.strictObject({ + ok: z.literal(false), + reason: z.enum([ + "unavailable", + "unsupported", + "decoder", + "format", + "output-limit", + ]), + message: z.string(), + }), +]); + +/** Add mapping candidates from an owned core-only debugger, using the existing snapshot. */ +export const inspectPwndbgCore = async ({ + environment, + rootPath, + diagnostics, + options, + launcher, +}: { + readonly environment: Readonly; + readonly rootPath: string; + readonly diagnostics: AnalysisCapturedOutput; + readonly options: ExecutionOptions | undefined; + readonly launcher?: PwntoolsLauncher; +}): Promise => { + const gdb = environment.REA_PWNDBG_GDB ?? "/usr/bin/gdb"; + const entry = environment.REA_PWNDBG_GDBINIT ?? ""; + const venv = environment.REA_PWNDBG_VENV_PATH ?? ""; + const unavailable = (message: string, cause?: unknown) => + new AnalysisCapabilityUnavailableError("pwndbg-core", OPERATION, message, { + ...(cause === undefined ? {} : { cause }), + userMessage: message, + capturedOutput: diagnostics, + }); + for (const [key, path, kind] of [ + ["REA_PWNDBG_GDB", gdb, "file"], + ["REA_PWNDBG_GDBINIT", entry, "file"], + ["REA_PWNDBG_VENV_PATH", venv, "directory"], + ] as const) { + try { + if (!isAbsolute(path)) + throw new Error("Expected an absolute caller-supplied path."); + await access( + path, + key === "REA_PWNDBG_GDB" ? constants.X_OK : constants.R_OK, + ); + const information = await stat(path); + if (kind === "file" ? !information.isFile() : !information.isDirectory()) + throw new Error(`Expected a ${kind}.`); + } catch (cause: unknown) { + throw unavailable( + `${key} is unavailable at ${path}: ${cause instanceof Error ? cause.message : String(cause)}. Configure the requested core-only context, or omit include_debugger_context for basic recorded evidence.`, + cause, + ); + } + } + const stage = join(rootPath, "debugger"); + await mkdir(stage, { mode: 0o700 }); + const requestPath = join(stage, "request.json"); + const replyPath = join(stage, "reply.json"); + await writeFile( + requestPath, + JSON.stringify({ + snapshot_path: join(rootPath, "object.snapshot"), + reply_path: replyPath, + gdb, + gdbinit: entry, + venv, + }), + { flag: "wx", mode: 0o600 }, + ); + let output: AnalysisCapturedOutput | undefined; + try { + options?.signal?.throwIfAborted(); + const execution = await runOwnedCommand( + { + command: environment.REA_PWNTOOLS_PYTHON ?? "", + arguments: [ + "-I", + fileURLToPath( + new URL("../../../bridge/pwndbg/launch.py", import.meta.url), + ), + join(stage, "resource.failure"), + requestPath, + ], + cwd: stage, + runId: `rea-recorded-core-debugger-${randomUUID()}`, + hostEnvironment: { + ...environment, + PWNDBG_NO_AUTOUPDATE: "1", + PWNDBG_VENV_PATH: venv, + PWNLIB_NOTERM: "1", + TERM: "dumb", + HOME: stage, + XDG_CACHE_HOME: join(stage, "cache"), + XDG_CONFIG_HOME: join(stage, "config"), + PYTHONNOUSERSITE: "1", + OPENBLAS_NUM_THREADS: "1", + OMP_NUM_THREADS: "1", + }, + }, + { + timeoutMs: PWNTOOLS_LIMITS.timeoutMs, + diagnosticBytes: PWNTOOLS_LIMITS.diagnosticBytes, + }, + { + ...(options?.signal === undefined ? {} : { signal: options.signal }), + ...(launcher === undefined ? {} : { launcher }), + }, + ); + output = capturedPwntoolsOutput(execution); + let reply: z.output; + try { + const raw = await readStableArtifact( + replyPath, + PWNTOOLS_LIMITS.outputBytes, + options?.signal, + ); + reply = replySchema.parse(JSON.parse(raw.bytes.toString("utf8"))); + } catch (cause: unknown) { + if (options?.signal?.aborted) + throw new AnalysisCancelledError(OPERATION, { + capturedOutput: recordedCrashStageOutput(diagnostics, output), + }); + throw new AnalysisOutputError( + OPERATION, + `Core-only debugger reply failed: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + capturedOutput: recordedCrashStageOutput(diagnostics, output), + }, + ); + } + if (!reply.ok) { + if (reply.reason === "unavailable" || reply.reason === "unsupported") + throw new AnalysisCapabilityUnavailableError( + "pwndbg-core", + OPERATION, + reply.message, + { + userMessage: reply.message, + capturedOutput: recordedCrashStageOutput(diagnostics, output), + }, + ); + if (reply.reason === "format" || reply.reason === "output-limit") + throw new AnalysisOutputError(OPERATION, reply.message, { + capturedOutput: recordedCrashStageOutput(diagnostics, output), + }); + throw new ProviderAdapterError("pwndbg-core", OPERATION, { + diagnostics: { + stage: "debugger", + reason: reply.message, + captured_output: { ...recordedCrashStageOutput(diagnostics, output) }, + }, + }); + } + if (reply.value.status !== "available") + throw new AnalysisOutputError( + OPERATION, + "Requested debugger context returned no core evidence.", + { capturedOutput: recordedCrashStageOutput(diagnostics, output) }, + ); + return { ...reply.value, diagnostics: output }; + } catch (cause: unknown) { + if ( + options?.signal?.aborted && + (cause === options.signal.reason || + (cause instanceof Error && cause.name === "AbortError")) + ) + throw new AnalysisCancelledError(OPERATION, { + capturedOutput: recordedCrashStageOutput(diagnostics, output), + }); + throw pwntoolsDecoderFailure( + cause, + "debugger", + gdb, + environment.REA_PWNTOOLS_PYTHON ?? "", + await readPwntoolsFailureEvidence(cause, stage), + { + operation: OPERATION, + providerId: "pwndbg-core", + precedingOutput: diagnostics, + }, + ); + } +}; + +/** Preserve both stages' exact text and their independently observed truncation state. */ +export const recordedCrashStageOutput = ( + basic: AnalysisCapturedOutput, + debuggerOutput?: AnalysisCapturedOutput, +): AnalysisCapturedOutput => + debuggerOutput === undefined + ? basic + : { + stdout: `[core decoder]\n${basic.stdout}\n[debugger]\n${debuggerOutput.stdout}`, + stderr: `[core decoder]\n${basic.stderr}\n[debugger]\n${debuggerOutput.stderr}`, + truncated: basic.truncated || debuggerOutput.truncated, + }; diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts new file mode 100644 index 0000000000..eacc2d63e5 --- /dev/null +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -0,0 +1,333 @@ +import { + pwntoolsDecoderFailure, + pwntoolsUnavailable, + capturedPwntoolsOutput, +} from "./PwntoolsFailures.js"; +import { randomUUID } from "node:crypto"; +import { access, stat, writeFile } from "node:fs/promises"; +import { constants } from "node:fs"; +import { isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; +import type { ProviderIdentity } from "../../application/AnalysisProvider.js"; +import type { AnalysisCapturedOutput } from "../../domain/analysisErrorBase.js"; +import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisUnsupportedTargetError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { + pwntoolsResourceLimitsSchema, + readPwntoolsFailureEvidence, +} from "./PwntoolsResourceLimits.js"; +import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; +import { + PWNTOOLS_PROVIDER_IDENTITY, + PWNTOOLS_LIMITS, +} from "./PwntoolsRelease.js"; + +/** Owned launcher seam shared by the offline pwntools artifact adapters. */ +export type PwntoolsLauncher = NonNullable< + Parameters[2] +>["launcher"]; +interface DecoderReport { + readonly diagnostics: AnalysisCapturedOutput; +} +/** Adapter-selected semantics around a shared owned pwntools process boundary. */ +export interface PwntoolsDecoderProfile< + Input extends { readonly path: string }, + Payload, + Report extends DecoderReport, +> { + readonly identity: ProviderIdentity; + readonly operation: string; + readonly bridge: URL; + readonly payloadSchema: z.ZodType; + readonly project: (context: { + readonly input: Input; + readonly payload: Payload; + readonly snapshot: { readonly bytes: Buffer; readonly sha256: string }; + readonly rootPath: string; + readonly diagnostics: AnalysisCapturedOutput; + readonly options: ExecutionOptions | undefined; + }) => Report | Promise; +} + +/** Thin owned-process adapter; all ELF parsing and heuristics come from unchanged upstream code. */ +export class PwntoolsDecoder< + Input extends { readonly path: string }, + Payload, + Report extends DecoderReport, +> { + readonly identity: ProviderIdentity; + constructor( + readonly profile: PwntoolsDecoderProfile, + readonly environment: Readonly, + readonly launcher?: PwntoolsLauncher, + readonly createRuntime: () => Promise< + Pick + > = () => PrivateRuntimeRoot.create({ prefix: "rea-artifact-decoder-" }), + ) { + this.identity = profile.identity; + } + + /** Snapshot one explicit artifact and return complete static observations without launching the selected object as a host process. */ + async inspect( + input: Input, + options?: ExecutionOptions, + ): Promise> { + const OPERATION = this.profile.operation; + const context = { operation: OPERATION, providerId: this.identity.id }; + const replySchema = z.discriminatedUnion("ok", [ + z.strictObject({ + ok: z.literal(true), + profile: z.literal(PWNTOOLS_PROVIDER_IDENTITY.version), + value: this.profile.payloadSchema, + }), + z.strictObject({ + ok: z.literal(false), + reason: z.enum([ + "format", + "unsupported", + "unsupported-target", + "unavailable", + "resource-limit", + "output-limit", + "decoder", + ]), + message: z.string(), + reported_limits: pwntoolsResourceLimitsSchema.nullable().optional(), + }), + ]); + let root: Pick | undefined; + let result: Result; + let phase: "configuration" | "artifact-read" | "decoder" = "configuration"; + const executablePath = this.environment.REA_PWNTOOLS_PYTHON ?? ""; + let selectedPath = executablePath; + try { + if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); + if (process.platform !== "linux" || process.arch !== "x64") + throw new AnalysisCapabilityUnavailableError( + this.identity.id, + OPERATION, + "The initial real-verified pwntools artifact profile supports Linux x64 only.", + ); + if (!isAbsolute(selectedPath)) + throw pwntoolsUnavailable( + "Set REA_PWNTOOLS_PYTHON to an absolute caller-supplied Python executable with pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2; REA never installs Python or packages.", + selectedPath, + undefined, + undefined, + context, + ); + await access(selectedPath, constants.X_OK); + if (!(await stat(selectedPath)).isFile()) + throw pwntoolsUnavailable( + "Selected Python executable must be a regular executable file, not a directory: " + + selectedPath, + selectedPath, + undefined, + undefined, + context, + ); + phase = "artifact-read"; + selectedPath = input.path; + const snapshot = await readStableArtifact( + input.path, + PWNTOOLS_LIMITS.inputBytes, + options?.signal, + ); + phase = "decoder"; + root = await this.createRuntime(); + const requestPath = join(root.path, "request.json"); + const snapshotPath = join(root.path, "object.snapshot"); + const replyPath = join(root.path, "reply.json"); + await writeFile(snapshotPath, snapshot.bytes, { + flag: "wx", + mode: 0o600, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + await writeFile( + requestPath, + JSON.stringify({ snapshot_path: snapshotPath, reply_path: replyPath }), + { flag: "wx", mode: 0o600 }, + ); + const execution = await runOwnedCommand( + { + command: this.environment.REA_PWNTOOLS_PYTHON ?? "", + arguments: [ + "-I", + fileURLToPath(this.profile.bridge), + join(root.path, "resource.failure"), + requestPath, + ], + cwd: root.path, + runId: `rea-artifact-decoder-${randomUUID()}`, + hostEnvironment: { + ...this.environment, + PWNLIB_NOTERM: "1", + OPENBLAS_NUM_THREADS: "1", + OMP_NUM_THREADS: "1", + }, + }, + { + timeoutMs: PWNTOOLS_LIMITS.timeoutMs, + diagnosticBytes: PWNTOOLS_LIMITS.diagnosticBytes, + }, + { + ...(options?.signal === undefined ? {} : { signal: options.signal }), + ...(this.launcher === undefined ? {} : { launcher: this.launcher }), + }, + ); + const capturedOutput = capturedPwntoolsOutput(execution); + let reply: z.output; + try { + const file = await readStableArtifact( + replyPath, + PWNTOOLS_LIMITS.outputBytes, + options?.signal, + ); + reply = replySchema.parse(JSON.parse(file.bytes.toString("utf8"))); + } catch (cause: unknown) { + if (options?.signal?.aborted) + throw new AnalysisCancelledError(OPERATION, { capturedOutput }); + throw new AnalysisOutputError( + OPERATION, + `Owned pwntools decoder reply failed for ${input.path}: ${cause instanceof z.ZodError ? cause.issues[0]?.message + " at " + (cause.issues[0]?.path.map(String).join(".") ?? "root") : cause instanceof Error ? cause.message : String(cause)}`, + { cause, capturedOutput }, + ); + } + if (!reply.ok) { + if (reply.reason === "unsupported-target") + throw new AnalysisUnsupportedTargetError( + OPERATION, + input.path, + reply.message, + { capturedOutput }, + ); + if (reply.reason === "resource-limit") + throw new AnalysisResourceConstraintError( + OPERATION, + "memory", + reply.message, + reply.reported_limits ?? null, + { capturedOutput }, + ); + if (reply.reason === "format") + throw new AnalysisInputError(OPERATION, { capturedOutput }, [ + { + path: ["path"], + reason: "invalid_format", + message: reply.message, + }, + ]); + if (reply.reason === "unavailable") + throw pwntoolsUnavailable( + reply.message, + this.environment.REA_PWNTOOLS_PYTHON ?? "", + undefined, + capturedOutput, + context, + ); + if (reply.reason === "unsupported") + throw new AnalysisCapabilityUnavailableError( + this.identity.id, + OPERATION, + reply.message, + { userMessage: reply.message, capturedOutput }, + ); + if (reply.reason === "output-limit") + throw new AnalysisOutputError(OPERATION, reply.message, { + capturedOutput, + }); + throw new ProviderAdapterError(this.identity.id, OPERATION, { + diagnostics: { + phase, + failure_kind: reply.reason, + reason: reply.message, + stdout: execution.stdout.text, + stderr: execution.stderr.text, + captured_output: { ...capturedOutput }, + }, + }); + } + result = ok( + await this.profile.project({ + input, + payload: reply.value, + snapshot, + rootPath: root.path, + diagnostics: capturedOutput, + options, + }), + ); + } catch (cause: unknown) { + const evidence = await readPwntoolsFailureEvidence(cause, root?.path); + result = err( + options?.signal?.aborted && + (cause === options.signal.reason || + (cause instanceof Error && cause.name === "AbortError")) + ? new AnalysisCancelledError(OPERATION) + : pwntoolsDecoderFailure( + cause, + phase, + selectedPath, + executablePath, + evidence, + context, + ), + ); + } + if (root !== undefined) { + try { + await root.close(); + } catch (cause: unknown) { + return err( + new ProviderCleanupError( + this.identity.id, + [root.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + previous_error: result.ok + ? null + : projectAnalysisError(result.error), + ...(result.ok + ? { + captured_output: { + stdout: result.value.diagnostics.stdout, + stderr: result.value.diagnostics.stderr, + truncated: result.value.diagnostics.truncated, + }, + } + : {}), + }, + { operation: OPERATION }, + ), + ); + } + } + return result.ok && options?.signal?.aborted + ? err( + new AnalysisCancelledError(OPERATION, { + capturedOutput: { + stdout: result.value.diagnostics.stdout, + stderr: result.value.diagnostics.stderr, + truncated: result.value.diagnostics.truncated, + }, + }), + ) + : result; + } +} diff --git a/src/native/pwntools/PwntoolsFailures.ts b/src/native/pwntools/PwntoolsFailures.ts index 5e595f6de0..f5e1b26545 100644 --- a/src/native/pwntools/PwntoolsFailures.ts +++ b/src/native/pwntools/PwntoolsFailures.ts @@ -25,33 +25,58 @@ import { PWNTOOLS_FILE_SIZE_FAILURE_EXIT, } from "./PwntoolsRelease.js"; -const OPERATION = "inspect_binary_layout"; +/** Operation identity supplied by each adapter sharing the pwntools boundary. */ +export interface PwntoolsFailureContext { + readonly operation: string; + readonly providerId: string; + readonly precedingOutput?: AnalysisCapturedOutput; +} +const DEFAULT_CONTEXT = { + operation: "inspect_binary_layout", + providerId: PWNTOOLS_PROVIDER_IDENTITY.id, +}; /** Keep read, engine, output and owned-lifecycle failure reasons distinct. */ -export const pwntoolsLayoutFailure = ( +export const pwntoolsDecoderFailure = ( cause: unknown, phase: string, path: string, executablePath = path, failureEvidence: PwntoolsFailureEvidence = {}, + context: PwntoolsFailureContext = DEFAULT_CONTEXT, ): AnalysisError => { const limitReport = failureEvidence.limits; if (cause instanceof AnalysisError) return cause; if (cause instanceof OwnedCommandFailure) { - const outputOptions = + const currentOutput = cause.snapshot === null ? undefined - : { capturedOutput: capturedPwntoolsOutput(cause.snapshot) }; + : capturedPwntoolsOutput(cause.snapshot); + const combinedOutput = + context.precedingOutput === undefined + ? currentOutput + : currentOutput === undefined + ? context.precedingOutput + : { + stdout: `[core decoder]\n${context.precedingOutput.stdout}\n[debugger]\n${currentOutput.stdout}`, + stderr: `[core decoder]\n${context.precedingOutput.stderr}\n[debugger]\n${currentOutput.stderr}`, + truncated: + context.precedingOutput.truncated || currentOutput.truncated, + }; + const outputOptions = + combinedOutput === undefined + ? undefined + : { capturedOutput: combinedOutput }; if (cause.cleanupFailure !== null) return new ProviderCleanupError( - PWNTOOLS_PROVIDER_IDENTITY.id, + context.providerId, cause.resources, { reason: cause.cleanupFailure, - ...(cause.snapshot === null + ...(combinedOutput === undefined ? {} : { - captured_output: { ...capturedPwntoolsOutput(cause.snapshot) }, + captured_output: { ...combinedOutput }, }), previous_error: { failure_kind: cause.reason, @@ -62,18 +87,22 @@ export const pwntoolsLayoutFailure = ( stderr: cause.snapshot?.stderr.text ?? null, }, }, - { operation: OPERATION, cause }, + { operation: context.operation, cause }, ); if (cause.reason === "cancelled") - return new AnalysisCancelledError(OPERATION, outputOptions); + return new AnalysisCancelledError(context.operation, outputOptions); if (cause.reason === "timeout") return new AnalysisTimeoutError( - OPERATION, + context.operation, PWNTOOLS_LIMITS.timeoutMs, outputOptions, ); if (cause.reason === "output-limit") - return new AnalysisOutputError(OPERATION, cause.message, outputOptions); + return new AnalysisOutputError( + context.operation, + cause.message, + outputOptions, + ); if ( cause.reason === "process" && (cause.snapshot?.signal === "SIGXFSZ" || @@ -82,7 +111,7 @@ export const pwntoolsLayoutFailure = ( cause.snapshot.signal === null)) ) return new AnalysisResourceConstraintError( - OPERATION, + context.operation, "file-size", (cause.snapshot.signal === "SIGXFSZ" ? "Owned Python terminated with SIGXFSZ; the exact signal cause is unknown." @@ -95,7 +124,7 @@ export const pwntoolsLayoutFailure = ( ); if (cause.reason === "process" && cause.snapshot?.signal === "SIGXCPU") return new AnalysisResourceConstraintError( - OPERATION, + context.operation, "cpu", "Owned Python terminated with SIGXCPU; the exact signal cause is unknown." + (limitReport?.failure === null || limitReport === undefined @@ -111,7 +140,7 @@ export const pwntoolsLayoutFailure = ( cause.snapshot.signal === null ) return new AnalysisResourceConstraintError( - OPERATION, + context.operation, "memory", "The owned Python bridge reported a memory allocation failure without a structured reply; the exact allocation cause is unknown." + (limitReport?.failure === null || limitReport === undefined @@ -132,20 +161,27 @@ export const pwntoolsLayoutFailure = ( `Selected Python could not launch (${String(cause.code)}): ${executablePath}. Check the configured executable, its interpreter and host execute access.`, executablePath, String(cause.code), + undefined, + context, ); if (cause instanceof ArtifactReaderFailure) { if (cause.reason === "integrity") - return new AnalysisArtifactChangedError(OPERATION, path, cause.message, { - cause, - }); + return new AnalysisArtifactChangedError( + context.operation, + path, + cause.message, + { + cause, + }, + ); if (cause.reason === "cancelled") - return new AnalysisCancelledError(OPERATION); + return new AnalysisCancelledError(context.operation); if ( cause.reason === "limit" || cause.reason === "path" || cause.reason === "format" ) - return new AnalysisInputError(OPERATION, { cause }, [ + return new AnalysisInputError(context.operation, { cause }, [ { path: ["path"], reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", @@ -159,19 +195,26 @@ export const pwntoolsLayoutFailure = ( `Selected Python executable is unavailable (${String(cause.code)}): ${path}. Check the caller-selected path and execute access.`, path, String(cause.code), + undefined, + context, ); if ( phase === "artifact-read" && (cause.code === "EACCES" || cause.code === "EPERM") ) - return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { - cause, - }); + return new AnalysisAccessDeniedError( + context.operation, + path, + cause.code, + { + cause, + }, + ); if ( phase === "artifact-read" && (cause.code === "ENOENT" || cause.code === "ENOTDIR") ) - return new AnalysisInputError(OPERATION, { cause }, [ + return new AnalysisInputError(context.operation, { cause }, [ { path: ["path"], reason: "invalid_value", @@ -179,11 +222,14 @@ export const pwntoolsLayoutFailure = ( }, ]); } - return new ProviderAdapterError(PWNTOOLS_PROVIDER_IDENTITY.id, OPERATION, { + return new ProviderAdapterError(context.providerId, context.operation, { cause, diagnostics: { phase, path, + ...(context.precedingOutput === undefined + ? {} + : { preceding_output: { ...context.precedingOutput } }), reason: cause instanceof Error ? cause.message : String(cause), ...(failureEvidence.marker === undefined ? {} @@ -214,16 +260,17 @@ export const pwntoolsUnavailable = ( path: string, systemCode?: string, capturedOutput?: AnalysisCapturedOutput, + context: PwntoolsFailureContext = DEFAULT_CONTEXT, ): ProviderSelectionError => new ProviderSelectionError({ ...(capturedOutput === undefined ? {} : { capturedOutput }), - operation: OPERATION, + operation: context.operation, reason: "provider_unavailable", - requestedProviderId: PWNTOOLS_PROVIDER_IDENTITY.id, - candidateIds: [PWNTOOLS_PROVIDER_IDENTITY.id], + requestedProviderId: context.providerId, + candidateIds: [context.providerId], rejections: [ { - providerId: PWNTOOLS_PROVIDER_IDENTITY.id, + providerId: context.providerId, code: "provider_unavailable", reason, diagnostics: { diff --git a/src/native/pwntools/PwntoolsLayoutProvider.ts b/src/native/pwntools/PwntoolsLayoutProvider.ts index 4e543d105c..f9346e397b 100644 --- a/src/native/pwntools/PwntoolsLayoutProvider.ts +++ b/src/native/pwntools/PwntoolsLayoutProvider.ts @@ -1,381 +1,74 @@ -import { - pwntoolsLayoutFailure, - pwntoolsUnavailable, - capturedPwntoolsOutput, -} from "./PwntoolsFailures.js"; -import { randomUUID } from "node:crypto"; -import { access, stat, writeFile } from "node:fs/promises"; -import { constants } from "node:fs"; -import { isAbsolute, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { z } from "zod"; -import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; -import type { BinaryLayoutPort } from "../../application/binaryDiagnostics/BinaryLayoutPort.js"; -import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; -import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import { - AnalysisCancelledError, - AnalysisCapabilityUnavailableError, - AnalysisInputError, - AnalysisOutputError, - AnalysisResourceConstraintError, -} from "../../domain/analysisErrorCore.js"; -import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; -import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; -import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; -import { err, ok, type Result } from "../../domain/result.js"; import { binaryLayoutPayloadSchema, binaryLayoutSchema, type BinaryLayout, type InspectBinaryLayoutInput, } from "../../domain/native/binaryLayout.js"; -import { runOwnedCommand } from "../../process/OwnedCommand.js"; -import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; -import { - pwntoolsResourceLimitsSchema, - type PwntoolsLimitReport, - type PwntoolsFailureEvidence, -} from "./PwntoolsResourceLimits.js"; +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { PwntoolsDecoder, type PwntoolsLauncher } from "./PwntoolsDecoder.js"; +import { PWNTOOLS_PROVIDER_IDENTITY } from "./PwntoolsRelease.js"; import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; -import { - PWNTOOLS_PROVIDER_IDENTITY, - PWNTOOLS_LIMITS, - PWNTOOLS_FILE_SIZE_FAILURE_EXIT, - PWNTOOLS_MEMORY_FAILURE_EXIT, -} from "./PwntoolsRelease.js"; +import type { BinaryLayoutPort } from "../../application/binaryDiagnostics/BinaryLayoutPort.js"; +import type { z } from "zod"; -const OPERATION = "inspect_binary_layout"; -const decoded = binaryLayoutPayloadSchema; -const replySchema = z.discriminatedUnion("ok", [ - z.strictObject({ - ok: z.literal(true), - profile: z.literal(PWNTOOLS_PROVIDER_IDENTITY.version), - value: decoded, - }), - z.strictObject({ - ok: z.literal(false), - reason: z.enum([ - "format", - "unsupported", - "unavailable", - "resource-limit", - "output-limit", - "decoder", - ]), - message: z.string(), - reported_limits: pwntoolsResourceLimitsSchema.nullable().optional(), - }), -]); - -type Launcher = NonNullable[2]>["launcher"]; - -/** Thin owned-process adapter; all ELF parsing and heuristics come from unchanged upstream code. */ -export class PwntoolsLayoutProvider implements BinaryLayoutPort { - readonly identity = PWNTOOLS_PROVIDER_IDENTITY; +/** ELF-specific projection around the shared owned pwntools decoder boundary. */ +export class PwntoolsLayoutProvider + extends PwntoolsDecoder< + InspectBinaryLayoutInput, + z.output, + BinaryLayout + > + implements BinaryLayoutPort +{ constructor( - readonly environment: Readonly, - readonly launcher?: Launcher, - readonly createRuntime: () => Promise< + environment: Readonly, + launcher?: PwntoolsLauncher, + createRuntime: () => Promise< Pick > = () => PrivateRuntimeRoot.create({ prefix: "rea-elf-layout-" }), - ) {} - - /** Snapshot one explicit artifact and return complete static observations without launching the selected object as a host process. */ - async inspect( - input: InspectBinaryLayoutInput, - options?: ExecutionOptions, - ): Promise> { - let root: Pick | undefined; - let result: Result; - let phase: "configuration" | "artifact-read" | "decoder" = "configuration"; - const executablePath = this.environment.REA_PWNTOOLS_PYTHON ?? ""; - let selectedPath = executablePath; - try { - if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); - if (process.platform !== "linux" || process.arch !== "x64") - throw new AnalysisCapabilityUnavailableError( - this.identity.id, - OPERATION, - "The real-verified pwntools layout profile currently supports Linux x64 only.", - ); - if (!isAbsolute(selectedPath)) - throw pwntoolsUnavailable( - "Set REA_PWNTOOLS_PYTHON to an absolute caller-supplied Python executable with pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2; REA never installs Python or packages.", - selectedPath, - ); - await access(selectedPath, constants.X_OK); - if (!(await stat(selectedPath)).isFile()) - throw pwntoolsUnavailable( - "Selected Python executable must be a regular executable file, not a directory: " + - selectedPath, - selectedPath, - ); - phase = "artifact-read"; - selectedPath = input.path; - const snapshot = await readStableArtifact( - input.path, - PWNTOOLS_LIMITS.inputBytes, - options?.signal, - ); - phase = "decoder"; - root = await this.createRuntime(); - const requestPath = join(root.path, "request.json"); - const snapshotPath = join(root.path, "object.snapshot"); - const replyPath = join(root.path, "reply.json"); - await writeFile(snapshotPath, snapshot.bytes, { - flag: "wx", - mode: 0o600, - ...(options?.signal === undefined ? {} : { signal: options.signal }), - }); - await writeFile( - requestPath, - JSON.stringify({ snapshot_path: snapshotPath, reply_path: replyPath }), - { flag: "wx", mode: 0o600 }, - ); - const execution = await runOwnedCommand( - { - command: this.environment.REA_PWNTOOLS_PYTHON ?? "", - arguments: [ - "-I", - fileURLToPath( - new URL("../../../bridge/pwntools/layout.py", import.meta.url), - ), - join(root.path, "resource.failure"), - requestPath, - ], - cwd: root.path, - runId: `rea-elf-layout-${randomUUID()}`, - hostEnvironment: { - ...this.environment, - PWNLIB_NOTERM: "1", - OPENBLAS_NUM_THREADS: "1", - OMP_NUM_THREADS: "1", - }, - }, - { - timeoutMs: PWNTOOLS_LIMITS.timeoutMs, - diagnosticBytes: PWNTOOLS_LIMITS.diagnosticBytes, - }, - { - ...(options?.signal === undefined ? {} : { signal: options.signal }), - ...(this.launcher === undefined ? {} : { launcher: this.launcher }), - }, - ); - const capturedOutput = capturedPwntoolsOutput(execution); - let reply: z.output; - try { - const file = await readStableArtifact( - replyPath, - PWNTOOLS_LIMITS.outputBytes, - options?.signal, - ); - reply = replySchema.parse(JSON.parse(file.bytes.toString("utf8"))); - } catch (cause: unknown) { - if (options?.signal?.aborted) - throw new AnalysisCancelledError(OPERATION, { capturedOutput }); - throw new AnalysisOutputError( - OPERATION, - `Owned ELF decoder reply failed for ${input.path}: ${cause instanceof z.ZodError ? cause.issues[0]?.message + " at " + (cause.issues[0]?.path.map(String).join(".") ?? "root") : cause instanceof Error ? cause.message : String(cause)}`, - { cause, capturedOutput }, - ); - } - if (!reply.ok) { - if (reply.reason === "resource-limit") - throw new AnalysisResourceConstraintError( - OPERATION, - "memory", - reply.message, - reply.reported_limits ?? null, - { capturedOutput }, - ); - if (reply.reason === "format") - throw new AnalysisInputError(OPERATION, { capturedOutput }, [ - { - path: ["path"], - reason: "invalid_format", - message: reply.message, + ) { + super( + { + identity: PWNTOOLS_PROVIDER_IDENTITY, + operation: "inspect_binary_layout", + bridge: new URL("../../../bridge/pwntools/layout.py", import.meta.url), + payloadSchema: binaryLayoutPayloadSchema, + project: ({ input, payload, snapshot, diagnostics }) => { + const value = binaryLayoutSchema.safeParse({ + ...payload, + artifact: { + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, }, - ]); - if (reply.reason === "unavailable") - throw pwntoolsUnavailable( - reply.message, - this.environment.REA_PWNTOOLS_PYTHON ?? "", - undefined, - capturedOutput, - ); - if (reply.reason === "unsupported") - throw new AnalysisCapabilityUnavailableError( - this.identity.id, - OPERATION, - reply.message, - { userMessage: reply.message, capturedOutput }, - ); - if (reply.reason === "output-limit") - throw new AnalysisOutputError(OPERATION, reply.message, { - capturedOutput, + diagnostics, }); - throw new ProviderAdapterError(this.identity.id, OPERATION, { - diagnostics: { - phase, - failure_kind: reply.reason, - reason: reply.message, - stdout: execution.stdout.text, - stderr: execution.stderr.text, - captured_output: { ...capturedOutput }, - }, - }); - } - const validated = binaryLayoutSchema.safeParse({ - ...reply.value, - artifact: { - path: input.path, - sha256: snapshot.sha256, - bytes: snapshot.bytes.length, - }, - diagnostics: { - stdout: execution.stdout.text, - stderr: execution.stderr.text, - truncated: capturedOutput.truncated, - }, - }); - if (!validated.success) - throw new AnalysisOutputError( - OPERATION, - `ELF reply contains invalid source ranges or value meanings: ${validated.error.issues[0]?.message ?? "schema mismatch"}`, - { capturedOutput }, - ); - for (const table of validated.data.packed_relative_relocations) { - const start = Number(BigInt(table.location.offset)); - const length = Number(BigInt(table.location.bytes)); - if ( - !Buffer.from(table.encoded_bytes_base64, "base64").equals( - snapshot.bytes.subarray(start, start + length), - ) - ) - throw new AnalysisOutputError( - OPERATION, - "Reported packed relocation bytes differ from the selected snapshot at their original file range.", - { capturedOutput }, - ); - } - result = ok(validated.data); - } catch (cause: unknown) { - let limitReport: PwntoolsLimitReport | undefined; - let marker: PwntoolsFailureEvidence["marker"]; - if ( - cause instanceof OwnedCommandFailure && - cause.snapshot?.signal === null && - (cause.snapshot.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT || - cause.snapshot.exitCode === PWNTOOLS_FILE_SIZE_FAILURE_EXIT) && - root !== undefined - ) { - try { - const reported = await readStableArtifact( - join(root.path, "resource.failure"), - 1, - ); - const expected = - cause.snapshot.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT - ? "M" - : "F"; - if (!reported.bytes.equals(Buffer.from(expected, "ascii"))) - throw new Error( - "Private bridge failure marker does not match the observed exit status.", + if (!value.success) + throw new AnalysisOutputError( + "inspect_binary_layout", + `ELF reply contains invalid source ranges or value meanings: ${value.error.issues[0]?.message ?? "schema mismatch"}`, + { capturedOutput: diagnostics }, ); - marker = { - resource: expected === "M" ? "memory" : "file-size", - failure: null, - }; - } catch (markerFailure: unknown) { - marker = { - resource: null, - failure: - markerFailure instanceof Error - ? markerFailure.message - : String(markerFailure), - }; - } - } - if ( - cause instanceof OwnedCommandFailure && - (cause.snapshot?.signal === "SIGXCPU" || - cause.snapshot?.signal === "SIGXFSZ" || - ((cause.snapshot?.exitCode === PWNTOOLS_FILE_SIZE_FAILURE_EXIT || - cause.snapshot?.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT) && - cause.snapshot.signal === null)) && - root !== undefined - ) { - try { - const reported = await readStableArtifact( - join(root.path, "limits.json"), - 4096, - ); - limitReport = { - limits: pwntoolsResourceLimitsSchema.parse( - JSON.parse(reported.bytes.toString("utf8")), - ), - failure: null, - }; - } catch (reportFailure: unknown) { - limitReport = { - limits: null, - failure: - reportFailure instanceof Error - ? reportFailure.message - : String(reportFailure), - }; - } - } - result = err( - options?.signal?.aborted && - (cause === options.signal.reason || - (cause instanceof Error && cause.name === "AbortError")) - ? new AnalysisCancelledError(OPERATION) - : pwntoolsLayoutFailure(cause, phase, selectedPath, executablePath, { - ...(limitReport === undefined ? {} : { limits: limitReport }), - ...(marker === undefined ? {} : { marker }), - }), - ); - } - if (root !== undefined) { - try { - await root.close(); - } catch (cause: unknown) { - return err( - new ProviderCleanupError( - this.identity.id, - [root.path], - { - reason: cause instanceof Error ? cause.message : String(cause), - previous_error: result.ok - ? null - : projectAnalysisError(result.error), - ...(result.ok - ? { - captured_output: { - stdout: result.value.diagnostics.stdout, - stderr: result.value.diagnostics.stderr, - truncated: result.value.diagnostics.truncated, - }, - } - : {}), - }, - { operation: OPERATION }, - ), - ); - } - } - return result.ok && options?.signal?.aborted - ? err( - new AnalysisCancelledError(OPERATION, { - capturedOutput: { - stdout: result.value.diagnostics.stdout, - stderr: result.value.diagnostics.stderr, - truncated: result.value.diagnostics.truncated, - }, - }), - ) - : result; + for (const table of value.data.packed_relative_relocations) { + const start = Number(BigInt(table.location.offset)); + const length = Number(BigInt(table.location.bytes)); + if ( + !Buffer.from(table.encoded_bytes_base64, "base64").equals( + snapshot.bytes.subarray(start, start + length), + ) + ) + throw new AnalysisOutputError( + "inspect_binary_layout", + "Reported packed relocation bytes differ from the selected snapshot at their original file range.", + { capturedOutput: diagnostics }, + ); + } + return value.data; + }, + }, + environment, + launcher, + createRuntime, + ); } } diff --git a/src/native/pwntools/PwntoolsRecordedCrashProvider.ts b/src/native/pwntools/PwntoolsRecordedCrashProvider.ts new file mode 100644 index 0000000000..e0d1ea8d2b --- /dev/null +++ b/src/native/pwntools/PwntoolsRecordedCrashProvider.ts @@ -0,0 +1,144 @@ +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + recordedCrashPayloadSchema, + recordedCrashSchema, + type InspectRecordedCrashInput, + type RecordedCrash, +} from "../../domain/native/recordedCrash.js"; +import type { RecordedCrashPort } from "../../application/binaryDiagnostics/RecordedCrashPort.js"; +import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; +import { PwntoolsDecoder, type PwntoolsLauncher } from "./PwntoolsDecoder.js"; +import { PWNTOOLS_PROVIDER_IDENTITY } from "./PwntoolsRelease.js"; +import { validateRecordedCrashSources } from "./RecordedCrashSourceBindings.js"; +import { validateRecordedCrashStructure } from "./RecordedCrashStructureBindings.js"; +import { + inspectPwndbgCore, + recordedCrashStageOutput, +} from "../pwndbg/PwndbgCoreContext.js"; +import type { z } from "zod"; + +/** Preserve every recorded thread while optionally enriching the same owned snapshot. */ +export class PwntoolsRecordedCrashProvider + extends PwntoolsDecoder< + InspectRecordedCrashInput, + z.output, + RecordedCrash + > + implements RecordedCrashPort +{ + constructor( + environment: Readonly, + launcher?: PwntoolsLauncher, + createRuntime: () => Promise< + Pick + > = () => PrivateRuntimeRoot.create({ prefix: "rea-recorded-crash-" }), + ) { + super( + { + identity: PWNTOOLS_PROVIDER_IDENTITY, + operation: "inspect_recorded_crash", + bridge: new URL( + "../../../bridge/pwntools/recorded_crash.py", + import.meta.url, + ), + payloadSchema: recordedCrashPayloadSchema, + project: async ({ + input, + payload, + snapshot, + rootPath, + diagnostics, + options, + }) => { + const validate = (debuggerContext: RecordedCrash["debugger"]) => + recordedCrashSchema.safeParse({ + ...payload, + artifact: { + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }, + debugger: debuggerContext, + decoder_diagnostics: diagnostics, + diagnostics: + debuggerContext.status === "available" + ? recordedCrashStageOutput( + diagnostics, + debuggerContext.diagnostics, + ) + : diagnostics, + }); + const checked = validate({ status: "not-requested" }); + if (!checked.success) + throw new AnalysisOutputError( + "inspect_recorded_crash", + `Recorded core reply is invalid: ${checked.error.issues[0]?.message ?? "schema mismatch"}`, + { capturedOutput: diagnostics }, + ); + const value = checked.data; + validateRecordedCrashStructure(value, snapshot.bytes, diagnostics); + const checkBytes = ( + raw: string, + range: { readonly offset: string; readonly bytes: string }, + ) => { + const start = Number(BigInt(range.offset)); + const length = Number(BigInt(range.bytes)); + if ( + !Buffer.from(raw, "base64").equals( + snapshot.bytes.subarray(start, start + length), + ) + ) + throw new AnalysisOutputError( + "inspect_recorded_crash", + "Reported raw note bytes differ from the selected snapshot.", + { capturedOutput: diagnostics }, + ); + }; + for (const note of value.notes) { + checkBytes(note.owner_bytes_base64, note.owner_location); + checkBytes(note.descriptor_bytes_base64, note.descriptor_location); + } + for (const padding of value.note_padding) + checkBytes(padding.bytes_base64, padding.location); + validateRecordedCrashSources(value, snapshot.bytes, diagnostics); + for (const thread of value.threads) + for (const register of thread.registers) { + const start = Number(BigInt(register.location.offset)); + if ( + snapshot.bytes.readBigUInt64LE(start) !== BigInt(register.value) + ) + throw new AnalysisOutputError( + "inspect_recorded_crash", + "Reported register value differs from its recorded source bytes.", + { capturedOutput: diagnostics }, + ); + } + if (!input.include_debugger_context) return value; + const context = await inspectPwndbgCore({ + environment, + rootPath, + diagnostics, + options, + ...(launcher === undefined ? {} : { launcher }), + }); + const enriched = validate(context); + if (!enriched.success) + throw new AnalysisOutputError( + "inspect_recorded_crash", + `Recorded debugger context is invalid: ${enriched.error.issues[0]?.message ?? "schema mismatch"}`, + { + capturedOutput: + context.status === "available" + ? recordedCrashStageOutput(diagnostics, context.diagnostics) + : diagnostics, + }, + ); + return enriched.data; + }, + }, + environment, + launcher, + createRuntime, + ); + } +} diff --git a/src/native/pwntools/PwntoolsResourceLimits.ts b/src/native/pwntools/PwntoolsResourceLimits.ts index ad00d92b47..dcbe91d5ab 100644 --- a/src/native/pwntools/PwntoolsResourceLimits.ts +++ b/src/native/pwntools/PwntoolsResourceLimits.ts @@ -1,3 +1,10 @@ +import { join } from "node:path"; +import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { + PWNTOOLS_MEMORY_FAILURE_EXIT, + PWNTOOLS_FILE_SIZE_FAILURE_EXIT, +} from "./PwntoolsRelease.js"; import { z } from "zod"; import { PWNTOOLS_LIMITS } from "./PwntoolsRelease.js"; @@ -30,3 +37,78 @@ export interface PwntoolsFailureEvidence { readonly failure: string | null; }; } + +/** Read only bounded owned reports; unverified reserved statuses stay unclassified. */ +export const readPwntoolsFailureEvidence = async ( + cause: unknown, + rootPath?: string, +): Promise => { + let limitReport: PwntoolsLimitReport | undefined; + let marker: PwntoolsFailureEvidence["marker"]; + if ( + cause instanceof OwnedCommandFailure && + cause.snapshot?.signal === null && + (cause.snapshot.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT || + cause.snapshot.exitCode === PWNTOOLS_FILE_SIZE_FAILURE_EXIT) && + rootPath !== undefined + ) { + try { + const reported = await readStableArtifact( + join(rootPath, "resource.failure"), + 1, + ); + const expected = + cause.snapshot.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT ? "M" : "F"; + if (!reported.bytes.equals(Buffer.from(expected, "ascii"))) + throw new Error( + "Private bridge failure marker does not match the observed exit status.", + ); + marker = { + resource: expected === "M" ? "memory" : "file-size", + failure: null, + }; + } catch (markerFailure: unknown) { + marker = { + resource: null, + failure: + markerFailure instanceof Error + ? markerFailure.message + : String(markerFailure), + }; + } + } + if ( + cause instanceof OwnedCommandFailure && + (cause.snapshot?.signal === "SIGXCPU" || + cause.snapshot?.signal === "SIGXFSZ" || + ((cause.snapshot?.exitCode === PWNTOOLS_FILE_SIZE_FAILURE_EXIT || + cause.snapshot?.exitCode === PWNTOOLS_MEMORY_FAILURE_EXIT) && + cause.snapshot.signal === null)) && + rootPath !== undefined + ) { + try { + const reported = await readStableArtifact( + join(rootPath, "limits.json"), + 4096, + ); + limitReport = { + limits: pwntoolsResourceLimitsSchema.parse( + JSON.parse(reported.bytes.toString("utf8")), + ), + failure: null, + }; + } catch (reportFailure: unknown) { + limitReport = { + limits: null, + failure: + reportFailure instanceof Error + ? reportFailure.message + : String(reportFailure), + }; + } + } + return { + ...(limitReport === undefined ? {} : { limits: limitReport }), + ...(marker === undefined ? {} : { marker }), + }; +}; diff --git a/src/native/pwntools/RecordedCrashSourceBindings.ts b/src/native/pwntools/RecordedCrashSourceBindings.ts new file mode 100644 index 0000000000..4de2555d2a --- /dev/null +++ b/src/native/pwntools/RecordedCrashSourceBindings.ts @@ -0,0 +1,68 @@ +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + RECORDED_LINUX_AMD64_REGISTERS, + type RecordedCrash, +} from "../../domain/native/recordedCrash.js"; + +const registerOffsets = new Map( + RECORDED_LINUX_AMD64_REGISTERS.map((name, index) => [name, 112 + index * 8]), +); + +/** Bind interpreted values and register identities to Linux amd64 source bytes. */ +export function validateRecordedCrashSources( + report: RecordedCrash, + snapshot: Buffer, + diagnostics: RecordedCrash["decoder_diagnostics"], +): void { + const fail = () => { + throw new AnalysisOutputError( + "inspect_recorded_crash", + "Reported PID or signal value differs from its recorded Linux amd64 source bytes.", + { capturedOutput: diagnostics }, + ); + }; + const descriptor = (noteIndex: number, minimumBytes: number): Buffer => { + const note = report.notes[noteIndex]; + if (note === undefined) return fail(); + const start = Number(BigInt(note.descriptor_location.offset)); + const length = Number(BigInt(note.descriptor_location.bytes)); + if (length < minimumBytes) return fail(); + return snapshot.subarray(start, start + length); + }; + // Fixed offsets independently checked against unchanged pwntools 4.15.0 + // elf_prstatus_amd64 / elf_siginfo_64; this is the sole supported note ABI. + for (const thread of report.threads) { + const bytes = descriptor(thread.note_index, 336); + if ( + thread.historical_pid !== bytes.readInt32LE(32) || + thread.recorded_current_signal !== bytes.readInt16LE(12) + ) + fail(); + const note = report.notes[thread.note_index]; + if (note === undefined) return fail(); + for (const register of thread.registers) { + const offset = registerOffsets.get(register.name); + if ( + offset === undefined || + BigInt(register.location.offset) !== + BigInt(note.descriptor_location.offset) + BigInt(offset) + ) + throw new AnalysisOutputError( + "inspect_recorded_crash", + "Reported register source does not match its Linux amd64 register identity.", + { capturedOutput: diagnostics }, + ); + } + } + for (const signal of report.signals) { + const bytes = descriptor(signal.note_index, 32); + if ( + signal.number !== bytes.readInt32LE(0) || + signal.errno !== bytes.readInt32LE(4) || + signal.code !== bytes.readInt32LE(8) || + (signal.fault_address !== null && + BigInt(signal.fault_address) !== bytes.readBigUInt64LE(16)) + ) + fail(); + } +} diff --git a/src/native/pwntools/RecordedCrashStructureBindings.ts b/src/native/pwntools/RecordedCrashStructureBindings.ts new file mode 100644 index 0000000000..4f80ef60d9 --- /dev/null +++ b/src/native/pwntools/RecordedCrashStructureBindings.ts @@ -0,0 +1,133 @@ +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import type { RecordedCrash } from "../../domain/native/recordedCrash.js"; + +// Exact amd64/core enum projections in the pinned pyelftools 0.33 profile. +const segmentKinds = new Map([ + [0, "PT_NULL"], + [1, "PT_LOAD"], + [2, "PT_DYNAMIC"], + [3, "PT_INTERP"], + [4, "PT_NOTE"], + [5, "PT_SHLIB"], + [6, "PT_PHDR"], + [7, "PT_TLS"], + [0x60000000, "PT_LOOS"], + [0x6fffffff, "PT_HIOS"], + [0x6474e550, "PT_GNU_EH_FRAME"], + [0x6474e551, "PT_GNU_STACK"], + [0x6474e552, "PT_GNU_RELRO"], + [0x6474e553, "PT_GNU_PROPERTY"], +]); +const noteKinds = new Map([ + [1, "NT_PRSTATUS"], + [2, "NT_FPREGSET"], + [3, "NT_PRPSINFO"], + [4, "NT_TASKSTRUCT"], + [6, "NT_AUXV"], + [0x53494749, "NT_SIGINFO"], + [0x46494c45, "NT_FILE"], +]); +const matchesKind = ( + raw: number, + reported: string | number, + kinds: ReadonlyMap, +): boolean => { + const known = kinds.get(raw); + return typeof reported === "string" + ? reported === known + : known === undefined && reported === raw; +}; + +const segmentMatches = ( + segment: RecordedCrash["segments"][number], + raw: Buffer, +): boolean => { + const type = raw.readUInt32LE(0); + return ( + matchesKind(type, segment.type, segmentKinds) && + BigInt(segment.flags) === BigInt(raw.readUInt32LE(4)) && + segment.file_backing === + (type === 0 || raw.readBigUInt64LE(32) === 0n ? "none" : "file") && + ( + [ + [segment.offset, 8], + [segment.virtual_address, 16], + [segment.physical_address, 24], + [segment.file_size, 32], + [segment.memory_size, 40], + [segment.alignment, 48], + ] as const + ).every( + ([field, relative]) => BigInt(field) === raw.readBigUInt64LE(relative), + ) + ); +}; + +/** Bind coverage-bearing ELF and note headers to the same snapshot as their payloads. */ +export function validateRecordedCrashStructure( + report: RecordedCrash, + snapshot: Buffer, + diagnostics: RecordedCrash["decoder_diagnostics"], +): void { + const fail = () => { + throw new AnalysisOutputError( + "inspect_recorded_crash", + "Reported core structure differs from its original ELF/note headers.", + { capturedOutput: diagnostics }, + ); + }; + const bytesAt = (offset: bigint, bytes: number): Buffer => { + if (offset < 0n || offset > BigInt(snapshot.length - bytes)) return fail(); + return snapshot.subarray(Number(offset), Number(offset) + bytes); + }; + const header = bytesAt(0n, 64); + if ( + !header + .subarray(0, 6) + .equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46, 2, 1])) || + ![0, 3].includes(header[7] ?? -1) || + header[8] !== 0 || + header.readUInt16LE(16) !== 4 || + header.readUInt16LE(18) !== 62 + ) + fail(); + const table = header.readBigUInt64LE(32); + const entryBytes = header.readUInt16LE(54); + let count = header.readUInt16LE(56); + if (count === 0xffff) { + if (header.readUInt16LE(58) < 64) return fail(); + count = bytesAt(header.readBigUInt64LE(40), 64).readUInt32LE(44); + } + if (count !== report.segments.length || (count > 0 && entryBytes < 56)) + fail(); + for (const segment of report.segments) { + const offset = table + BigInt(segment.index) * BigInt(entryBytes); + if ( + BigInt(segment.header_location.offset) !== offset || + BigInt(segment.header_location.bytes) !== BigInt(entryBytes) + ) + fail(); + const raw = bytesAt(offset, entryBytes); + if (!segmentMatches(segment, raw)) fail(); + } + for (const note of report.notes) { + const raw = bytesAt(BigInt(note.location.offset), 12); + if ( + BigInt(note.owner_location.bytes) !== BigInt(raw.readUInt32LE(0)) || + BigInt(note.descriptor_location.bytes) !== BigInt(raw.readUInt32LE(4)) || + !matchesKind(raw.readUInt32LE(8), note.type, noteKinds) + ) + fail(); + const owner = bytesAt( + BigInt(note.owner_location.offset), + raw.readUInt32LE(0), + ); + const terminator = owner.indexOf(0); + const display = + owner.length === 0 + ? null + : owner.subarray(0, terminator).toString("utf8"); + if ((owner.length > 0 && terminator < 0) || note.owner_display !== display) + fail(); + } +} diff --git a/src/server/createServer.ts b/src/server/createServer.ts index e0291e09f5..d011e5ac85 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -1,6 +1,9 @@ import type { EvmInterfaceService } from "../application/evm/EvmInterfaceService.js"; import { createEvmInterfaceService } from "../composition/evm.js"; import { registerEvmTools } from "./registerEvmTools.js"; +import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; +import { createRecordedCrashService } from "../composition/binaryDiagnostics.js"; +import type { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; import { registerBinaryDiagnosticsTools } from "./registerBinaryDiagnosticsTools.js"; import { createBinaryLayoutService } from "../composition/binaryDiagnostics.js"; import type { BinaryLayoutService } from "../application/binaryDiagnostics/BinaryLayoutService.js"; @@ -68,6 +71,7 @@ export interface CreateServerOptions { readonly evmInterface?: EvmInterfaceService; readonly logger?: Logger; readonly binaryLayout?: BinaryLayoutService; + readonly recordedCrash?: RecordedCrashService; readonly firmwareAnalysis?: FirmwareAnalysisPort; readonly javascriptRecovery?: JavaScriptRecoveryPort; readonly webModuleTrace?: WebModuleTraceService; @@ -99,6 +103,11 @@ const installSessionToolAvailability = ( webModuleResolutionEnabled: options.webModuleTrace !== undefined || isAbsolute(process.env.REA_BROWSER_EXECUTABLE ?? ""), + recordedCrashEnabled: + options.recordedCrash !== undefined || + (process.platform === "linux" && + process.arch === "x64" && + isAbsolute(process.env.REA_PWNTOOLS_PYTHON ?? "")), binaryLayoutEnabled: options.binaryLayout !== undefined || (process.platform === "linux" && @@ -218,6 +227,12 @@ export const createServer = ( toolLogger, recordEvidence, ); + registerRecordedCrashTools( + server, + options.recordedCrash ?? createRecordedCrashService(), + toolLogger, + recordEvidence, + ); registerFirmwareTools( server, new FirmwareAnalysisService( diff --git a/src/server/registerRecordedCrashTools.ts b/src/server/registerRecordedCrashTools.ts new file mode 100644 index 0000000000..0381689a0f --- /dev/null +++ b/src/server/registerRecordedCrashTools.ts @@ -0,0 +1,32 @@ +import type { McpServer } from "@modelcontextprotocol/server"; +import type { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "../logger.js"; +import { logToolExecution } from "./toolLogging.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { toCallToolResult } from "./toolResult.js"; + +/** Bind recorded crash inspection to its named contract and session Evidence owner. */ +export const registerRecordedCrashTools = ( + server: McpServer, + service: RecordedCrashService, + logger: Logger, + recordEvidence?: EvidenceWriter["recordEvidence"], +): void => { + const contract = toolContract("inspect_recorded_crash"); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input, context) => { + const result = await logToolExecution(logger, contract.name, () => + service.inspect(input, { signal: context.mcpReq.signal }), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordEvidence?.(result.value); + return recorded !== undefined && !recorded.ok + ? toCallToolResult(recorded, contract) + : toCallToolResult(result, contract); + }, + ); +}; diff --git a/src/server/sessionAvailabilityPolicy.ts b/src/server/sessionAvailabilityPolicy.ts index 7339296478..829102d648 100644 --- a/src/server/sessionAvailabilityPolicy.ts +++ b/src/server/sessionAvailabilityPolicy.ts @@ -20,6 +20,7 @@ export interface SessionAvailabilityDefaults { | "javascriptRecoveryEnabled" | "webModuleResolutionEnabled" | "binaryLayoutEnabled" + | "recordedCrashEnabled" | "firmwareInspectionEnabled" | "firmwareExtractionEnabled" >; @@ -36,6 +37,8 @@ export const sessionAvailabilityPolicy = ( evmInterfaceEnabled: defaults.optionalFeatures?.evmInterfaceEnabled ?? false, processCaptureEnabled: platform !== "win32", + recordedCrashEnabled: + defaults.optionalFeatures?.recordedCrashEnabled ?? false, binaryLayoutEnabled: defaults.optionalFeatures?.binaryLayoutEnabled ?? false, firmwareInspectionEnabled: diff --git a/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts b/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts new file mode 100644 index 0000000000..5b3ca3b7bb --- /dev/null +++ b/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts @@ -0,0 +1,408 @@ +import type { RecordedCrash } from "../../../src/domain/native/recordedCrash.js"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; +import { PwntoolsRecordedCrashProvider } from "../../../src/native/pwntools/PwntoolsRecordedCrashProvider.js"; +import { PWNTOOLS_PROVIDER_IDENTITY } from "../../../src/native/pwntools/PwntoolsRelease.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; +import { + recordedCrashFixture, + recordedCrashDebuggerFixture, + recordedCrashFixtureBytes, + recordedCrashSignalFixture, +} from "../../fixtures/binaryDiagnostics/recordedCrash.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; + +const requestSchema = z.object({ + snapshot_path: z.string(), + reply_path: z.string(), +}); +const host = process.platform === "linux" && process.arch === "x64"; + +it + .runIf(host) + .each([ + "success", + "wrong-register", + "wrong-register-source", + "missing-thread", + "duplicate-thread", + "missing-signal", + "duplicate-signal", + "missing-register", + "duplicated-physical-thread", + "duplicated-physical-signal", + "omitted-physical-note", + "hidden-segment", + "wrong-header-range", + "wrong-note-kind", + "wrong-owner-display", + "wrong-pid", + "wrong-current-signal", + "wrong-signal-number", + "wrong-signal-code", + "wrong-signal-errno", + "wrong-signal-address", + "wrong-raw-note", + "malformed-range", + ] as const)( + "validates recorded source bytes and cleanup: %s", + async (scenario) => { + const { path, value, bytes } = await fixture( + true, + scenario === "wrong-owner-display", + ); + let owned = ""; + const provider = new PwntoolsRecordedCrashProvider( + { REA_PWNTOOLS_PYTHON: process.execPath }, + async (spawn) => { + owned = spawn.cwd ?? ""; + const request = requestSchema.parse( + JSON.parse(await readFile(spawn.arguments.at(-1) ?? "", "utf8")), + ); + expect(await readFile(request.snapshot_path)).toEqual(bytes); + const payload = structuredClone(value); + alterRecordedCrashReply(payload, scenario); + const { + artifact: _artifact, + diagnostics: _diagnostics, + debugger: _debugger, + decoder_diagnostics: _decoderDiagnostics, + ...raw + } = payload; + await writeFile( + request.reply_path, + JSON.stringify({ + ok: true, + profile: PWNTOOLS_PROVIDER_IDENTITY.version, + value: raw, + }), + ); + return spawnOwnedProviderProcess({ + ...spawn, + command: process.execPath, + arguments: ["-e", "process.stdout.write('recorded warning')"], + }); + }, + ); + const result = await provider.inspect({ + path, + include_debugger_context: false, + }); + if (scenario === "success") { + if (!result.ok) throw result.error; + expect(result.value.threads).toEqual(value.threads); + expect(result.value.diagnostics.stdout).toBe("recorded warning"); + } else + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + capturedOutput: { stdout: "recorded warning" }, + }, + }); + expect(await readFile(path)).toEqual(bytes); + await expect(access(owned)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it + .runIf(host) + .each([ + "unconfigured", + "success", + "process-failure", + "invalid-context", + "cancelled", + "cleanup-failure", + ] as const)( + "owns optional context from the same snapshot: %s", + async (scenario) => { + const { path, value, bytes, root } = await fixture(); + const controller = new AbortController(); + let owned = ""; + let snapshotPath = ""; + const venv = join(root, "venv"); + const plugin = join(root, "gdbinit.py"); + await mkdir(venv); + await writeFile(plugin, ""); + const provider = new PwntoolsRecordedCrashProvider( + { + REA_PWNTOOLS_PYTHON: process.execPath, + ...(scenario === "unconfigured" + ? {} + : { + REA_PWNDBG_GDB: process.execPath, + REA_PWNDBG_GDBINIT: plugin, + REA_PWNDBG_VENV_PATH: venv, + }), + }, + async (spawn) => { + const request = requestSchema.parse( + JSON.parse(await readFile(spawn.arguments.at(-1) ?? "", "utf8")), + ); + expect(await readFile(request.snapshot_path)).toEqual(bytes); + if (snapshotPath === "") { + snapshotPath = request.snapshot_path; + owned = spawn.cwd ?? ""; + const { + artifact: _artifact, + diagnostics: _diagnostics, + debugger: _debugger, + decoder_diagnostics: _decoderDiagnostics, + ...raw + } = value; + await writeFile( + request.reply_path, + JSON.stringify({ + ok: true, + profile: PWNTOOLS_PROVIDER_IDENTITY.version, + value: raw, + }), + ); + return spawnOwnedProviderProcess({ + ...spawn, + command: process.execPath, + arguments: ["-e", "process.stdout.write('basic stage diagnostic')"], + }); + } + expect(request.snapshot_path).toBe(snapshotPath); + const context = recordedCrashDebuggerFixture(); + if (scenario === "invalid-context") + context.maps = context.maps.map((map) => ({ ...map, end: "0x0" })); + await writeFile( + request.reply_path, + JSON.stringify({ ok: true, value: context }), + ); + const launched = await spawnOwnedProviderProcess({ + ...spawn, + command: process.execPath, + arguments: [ + "-e", + scenario === "process-failure" + ? "process.stderr.write('debugger failed',()=>{process.exitCode=9;})" + : scenario === "cancelled" + ? "process.stdout.write('ready\\n'); setInterval(() => {}, 1000)" + : "process.stdout.write('debugger stage diagnostic')", + ], + }); + if (scenario === "cancelled") { + await waitForProviderProcessReady(launched.process); + setImmediate(() => controller.abort()); + } + if (scenario === "cleanup-failure") + return { + ...launched, + cleanup: async () => { + await cleanupOwnedProcessGroup(launched.ownership); + throw new Error("source-owned post-cleanup reporting failure"); + }, + }; + return launched; + }, + ); + const result = await provider.inspect( + { + path, + include_debugger_context: true, + }, + { signal: controller.signal }, + ); + if (scenario === "success") { + if (!result.ok) throw result.error; + expect(result.value.debugger).toMatchObject({ + status: "available", + maps: [{ permissions: null }], + diagnostics: { stdout: "debugger stage diagnostic" }, + }); + } else { + if (result.ok) throw new Error("Expected optional debugger failure"); + const projected = projectAnalysisError(result.error); + if (scenario === "unconfigured") + expect(projected).toMatchObject({ + details: { captured_output: { stdout: "basic stage diagnostic" } }, + }); + else if ( + scenario === "invalid-context" || + scenario === "cancelled" || + scenario === "cleanup-failure" + ) { + const serialized = JSON.stringify(projected); + expect(serialized).toContain("basic stage diagnostic"); + expect(serialized).toContain( + scenario === "cancelled" ? "ready" : "debugger stage diagnostic", + ); + expect(result.error._tag).toBe( + scenario === "invalid-context" + ? "AnalysisOutputError" + : scenario === "cancelled" + ? "AnalysisCancelledError" + : "ProviderAdapterError", + ); + if (scenario === "cleanup-failure") + expect(result.error).toBeInstanceOf(ProviderCleanupError); + } else + expect(projected).toMatchObject({ + category: "execution_failure", + details: { + diagnostics: { + preceding_output: { stdout: "basic stage diagnostic" }, + stderr: "debugger failed", + }, + }, + }); + } + expect(await readFile(path)).toEqual(bytes); + await expect(access(owned)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +const fixture = async (includeSignal = false, opaqueOwner = false) => { + const workspace = await createTestWorkspace("rea-recorded-core-seam-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = join(workspace.root, "recorded.core"); + const value = includeSignal + ? recordedCrashSignalFixture(path) + : recordedCrashFixture(path); + if (opaqueOwner) { + value.notes = value.notes.map((note) => + note.index === 0 + ? { + ...note, + owner_bytes_base64: Buffer.from("EVIL\0").toString("base64"), + owner_display: "EVIL", + } + : note, + ); + value.threads = []; + } + const bytes = recordedCrashFixtureBytes(value); + await writeFile(path, bytes); + return { path, value, bytes, root: workspace.root }; +}; + +const alterRecordedCrashReply = ( + payload: RecordedCrash, + scenario: string, +): void => { + if (scenario === "wrong-owner-display") + payload.notes = payload.notes.map((note) => + note.index === 0 ? { ...note, owner_display: "FORGED" } : note, + ); + if ( + scenario === "duplicated-physical-thread" || + scenario === "duplicated-physical-signal" + ) { + const noteIndex = scenario === "duplicated-physical-thread" ? 0 : 1; + const note = payload.notes[noteIndex]; + if (note === undefined) throw new Error("missing fixture note"); + payload.notes.push({ ...note, index: 2 }); + if (noteIndex === 0) + payload.threads.push( + ...payload.threads.map((thread) => ({ + ...thread, + note_index: 2, + })), + ); + else + payload.signals.push( + ...payload.signals.map((signal) => ({ + ...signal, + note_index: 2, + })), + ); + } + if (scenario === "omitted-physical-note") { + payload.notes = payload.notes.filter((note) => note.index === 0); + payload.signals = []; + } + if (scenario === "hidden-segment") { + payload.segments = []; + payload.notes = []; + payload.threads = []; + payload.signals = []; + } + if (scenario === "wrong-header-range") + payload.segments = payload.segments.map((segment) => ({ + ...segment, + header_location: { offset: "0x0", bytes: "0x38" }, + })); + if (scenario === "wrong-note-kind") { + payload.notes = payload.notes.map((note) => + note.index === 0 ? { ...note, type: "NT_AUXV" } : note, + ); + payload.threads = []; + } + if (scenario === "wrong-register-source") + payload.threads = payload.threads.map((thread) => ({ + ...thread, + registers: thread.registers.map((register) => + register.name === "rdi" + ? { + ...register, + value: "0x0", + location: { offset: "0x104", bytes: "0x8" }, + } + : register, + ), + })); + if (scenario === "missing-thread") payload.threads = []; + if (scenario === "duplicate-thread") + payload.threads = [...payload.threads, ...payload.threads]; + if (scenario === "missing-signal") payload.signals = []; + if (scenario === "duplicate-signal") + payload.signals = [...payload.signals, ...payload.signals]; + if (scenario === "missing-register") + payload.threads = payload.threads.map((thread) => ({ + ...thread, + registers: thread.registers.slice(1), + })); + if (scenario === "wrong-register") + payload.threads = payload.threads.map((t) => ({ + ...t, + registers: t.registers.map((r) => ({ ...r, value: "0x1" })), + })); + if (scenario === "wrong-pid") + payload.threads = payload.threads.map((thread) => ({ + ...thread, + historical_pid: -2, + })); + if (scenario === "wrong-current-signal") + payload.threads = payload.threads.map((thread) => ({ + ...thread, + recorded_current_signal: -1, + })); + if (scenario.startsWith("wrong-signal")) + payload.signals = payload.signals.map((signal) => ({ + ...signal, + ...(scenario === "wrong-signal-number" + ? { + number: 10, + fault_address: null, + fault_address_meaning: "unknown" as const, + } + : scenario === "wrong-signal-code" + ? { code: 2 } + : scenario === "wrong-signal-errno" + ? { errno: 0 } + : { fault_address: "0x1" }), + })); + if (scenario === "wrong-raw-note") + payload.notes = payload.notes.map((n) => ({ + ...n, + descriptor_bytes_base64: Buffer.alloc(336, 1).toString("base64"), + })); + if (scenario === "malformed-range") + payload.notes = payload.notes.map((n) => ({ + ...n, + descriptor_location: { offset: "0x200", bytes: "0x150" }, + })); +}; diff --git a/tests/boundary/mcp/recordedCrashMcp.test.ts b/tests/boundary/mcp/recordedCrashMcp.test.ts new file mode 100644 index 0000000000..364c280d16 --- /dev/null +++ b/tests/boundary/mcp/recordedCrashMcp.test.ts @@ -0,0 +1,76 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it, onTestFinished } from "vitest"; +import { RecordedCrashService } from "../../../src/application/binaryDiagnostics/RecordedCrashService.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { ok } from "../../../src/domain/result.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + recordedCrashFixture, + RECORDED_CRASH_TEST_PROVIDER, +} from "../../fixtures/binaryDiagnostics/recordedCrash.js"; + +it("advertises exact valid schemas and records inline recorded-core evidence without a binary target", async () => { + const value = recordedCrashFixture(); + const service = new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(value)), + }); + const session = createTestBinarySession(() => { + throw new Error("deep provider must not start"); + }); + const server = createServer(session, session, { recordedCrash: service }); + const client = new Client({ name: "recorded-crash-contract", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_recorded_crash", + ); + if (advertised?.outputSchema === undefined) + throw new Error("recorded crash must publish both schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect(ajv.validate(inputSchema, { path: value.artifact.path })).toBe(true); + expect( + ajv.validate(inputSchema, { + path: value.artifact.path, + approval: true, + }), + ).toBe(false); + const response = await client.callTool({ + name: "inspect_recorded_crash", + arguments: { path: value.artifact.path }, + }); + expect(response.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_recorded_crash").outputSchema.parse( + response.structuredContent, + ); + const evidence = parseEvidence(parsed.evidence); + expect(parsed.result).toEqual(value); + expect(parsed.result).toEqual(evidence.normalized_result); + expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); + expect( + ( + await client.callTool({ + name: "inspect_recorded_crash", + arguments: { path: "relative.elf" }, + }) + ).isError, + ).toBe(true); +}); diff --git a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts index 7dad88ab9f..3f14ac5b40 100644 --- a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -7,6 +7,18 @@ import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { reconstructJavaScriptArtifact } from "../../../src/application/javascript/JavaScriptArtifactReconstruction.js"; +import { + analyzeJavaScriptArtifactFiles, + analyzeAndProjectJavaScriptArtifactFiles, +} from "../../../src/application/javascript/JavaScriptArtifactAnalysis.js"; +import { buildJavaScriptArtifactGraph } from "../../../src/application/javascript/JavaScriptArtifactGraphBuilder.js"; +import { + buildJavaScriptSemanticGraph, + createJavaScriptSemanticGraphProjection, +} from "../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; +import { createJavaScriptArtifactReader } from "../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; +import { readJavaScriptArtifactFiles } from "../../../src/artifacts/javascript/JavaScriptArtifactFiles.js"; +import { scanCanonicalArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js"; import { scanArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js"; import { parseJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; @@ -14,6 +26,83 @@ import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/ja import { parseJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraphSerialization.js"; import { writeJavaScriptArtifactFixture } from "../../fixtures/javascriptArtifactApplication.js"; +it("reports file progress and honors cancellation before the next source", async () => { + const root = await fixtureDirectory(); + const controller = new AbortController(); + const progress: string[] = []; + await expect( + reconstructJavaScriptArtifact({ input_path: root }, controller.signal, { + report: async (event) => { + if (event.phase === "parse_javascript_source") { + progress.push(event.message); + if (progress.length === 2) controller.abort(); + } + }, + }), + ).rejects.toMatchObject({ reason: "cancelled" }); + expect(progress).toHaveLength(2); + expect(progress[0]).not.toBe(progress[1]); +}); + +it("preserves graph commitments and export shapes when consuming file-local IR", async () => { + const root = await fixtureDirectory(); + await writeFile( + join(root, "exports.mjs"), + ` + import { readFile } from "node:fs"; + export { readFile as read }; + export function create(value) { return { value, nested: { enabled: true } }; } + function unexported() { return "private"; } + `, + ); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const full = analyzeJavaScriptArtifactFiles(files); + const projection = createJavaScriptSemanticGraphProjection(); + const compact = await analyzeAndProjectJavaScriptArtifactFiles( + files, + projection.projectFile, + ); + const graph = buildJavaScriptArtifactGraph(snapshot, files, full); + expect(buildJavaScriptArtifactGraph(snapshot, files, compact)).toEqual( + graph, + ); + const semantic = projection.finish( + snapshot.manifest.root_sha256, + graph, + compact, + ); + expect(semantic).toEqual( + buildJavaScriptSemanticGraph({ + rootArtifactSha256: snapshot.manifest.root_sha256, + applicationGraph: graph, + analysis: full, + }), + ); + expect( + semantic.nodes.some( + ({ application_node_ids }) => application_node_ids.length > 0, + ), + ).toBe(true); + const exported = compact.files.find( + ({ file }) => file.path === "exports.mjs", + ); + expect(exported?.semantic?.ir.callables.map(({ name }) => name)).toEqual([ + "create", + ]); + expect( + full.files + .find(({ file }) => file.path === "exports.mjs") + ?.semantic?.ir.callables.map(({ name }) => name), + ).toContain("unexported"); + expect(parseJavaScriptSemanticGraph(semantic)).toEqual(semantic); + } finally { + await reader.close(); + } +}); + it("reconstructs package, Electron roles, Webpack/Rspack modules, and cross-layer facts without execution", async () => { const root = await fixtureDirectory(); Reflect.deleteProperty(globalThis, "__rea_bundle_executed"); diff --git a/tests/fixtures/binaryDiagnostics/recordedCrash.ts b/tests/fixtures/binaryDiagnostics/recordedCrash.ts new file mode 100644 index 0000000000..636a20c304 --- /dev/null +++ b/tests/fixtures/binaryDiagnostics/recordedCrash.ts @@ -0,0 +1,220 @@ +import type { RecordedCrash } from "../../../src/domain/native/recordedCrash.js"; + +/** Source-owned observation seam; these bytes do not simulate a verified ELF engine. */ +export const recordedCrashFixture = ( + path = "/artifacts/source-owned.core", +): RecordedCrash => { + const descriptor = Buffer.alloc(336); + descriptor.writeInt32LE(123, 32); + descriptor.writeInt16LE(11, 12); + descriptor.writeBigUInt64LE(0x1122334455667788n, 224); + return { + artifact: { path, sha256: "a".repeat(64), bytes: 512 }, + format: "elf-core", + architecture: "x86_64-little-endian", + target_execution: "not-performed", + live_process_identity: "unknown", + segments: [ + { + index: 0, + type: "PT_NOTE", + header_location: { offset: "0x40", bytes: "0x38" }, + offset: "0x80", + file_size: "0x164", + memory_size: "0x0", + virtual_address: "0x0", + physical_address: "0x0", + alignment: "0x4", + flags: "0x0", + file_backing: "file", + }, + ], + notes: [ + { + index: 0, + segment_index: 0, + type: "NT_PRSTATUS", + location: { offset: "0x80", bytes: "0x164" }, + owner_location: { offset: "0x8c", bytes: "0x5" }, + owner_bytes_base64: Buffer.from("CORE\0").toString("base64"), + owner_display: "CORE", + descriptor_location: { offset: "0x94", bytes: "0x150" }, + descriptor_bytes_base64: descriptor.toString("base64"), + }, + ], + note_padding: [], + threads: [ + { + note_index: 0, + historical_pid: 123, + recorded_current_signal: 11, + registers: [ + "r15", + "r14", + "r13", + "r12", + "rbp", + "rbx", + "r11", + "r10", + "r9", + "r8", + "rax", + "rcx", + "rdx", + "rsi", + "rdi", + "orig_rax", + "rip", + "cs", + "eflags", + "rsp", + "ss", + "fs_base", + "gs_base", + "ds", + "es", + "fs", + "gs", + ].map((name, index) => ({ + name, + value: name === "rdi" ? "0x1122334455667788" : "0x0", + location: { + offset: `0x${(148 + 112 + index * 8).toString(16)}`, + bytes: "0x8", + }, + })), + }, + ], + signals: [], + debugger: { status: "not-requested" }, + note_interpretation_completeness: "unknown", + decoder_diagnostics: { + stdout: "decoder diagnostic", + stderr: "", + truncated: false, + }, + diagnostics: { stdout: "decoder diagnostic", stderr: "", truncated: false }, + limitations: ["Historical PID is recorded metadata."], + }; +}; + +/** Source-owned SIGSEGV descriptor for scalar/source binding regressions. */ +export const recordedCrashSignalFixture = (path?: string): RecordedCrash => { + const value = recordedCrashFixture(path); + const descriptor = Buffer.alloc(32); + descriptor.writeInt32LE(11, 0); + descriptor.writeInt32LE(-2, 4); + descriptor.writeInt32LE(1, 8); + descriptor.writeBigUInt64LE(0x1122334455667788n, 16); + value.artifact.bytes = 640; + value.segments = value.segments.map((segment) => ({ + ...segment, + file_size: "0x198", + })); + value.notes.push({ + index: 1, + segment_index: 0, + type: "NT_SIGINFO", + location: { offset: "0x1e4", bytes: "0x34" }, + owner_location: { offset: "0x1f0", bytes: "0x5" }, + owner_bytes_base64: Buffer.from("CORE\0").toString("base64"), + owner_display: "CORE", + descriptor_location: { offset: "0x1f8", bytes: "0x20" }, + descriptor_bytes_base64: descriptor.toString("base64"), + }); + value.signals.push({ + note_index: 1, + number: 11, + code: 1, + errno: -2, + fault_address: "0x1122334455667788", + fault_address_meaning: "recorded-sigsegv-address", + thread_association: "unknown", + }); + return value; +}; + +/** Construct only the original byte ranges consumed by the provider protocol tests. */ +export const recordedCrashFixtureBytes = (value: RecordedCrash): Buffer => { + const bytes = Buffer.alloc(value.artifact.bytes); + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1, 1]); + bytes.writeUInt16LE(4, 16); + bytes.writeUInt16LE(62, 18); + bytes.writeBigUInt64LE(64n, 32); + bytes.writeUInt16LE(56, 54); + bytes.writeUInt16LE(value.segments.length, 56); + for (const segment of value.segments) { + const start = Number(BigInt(segment.header_location.offset)); + bytes.writeUInt32LE(segment.type === "PT_NOTE" ? 4 : 0, start); + bytes.writeUInt32LE(Number(BigInt(segment.flags)), start + 4); + for (const [field, offset] of [ + [segment.offset, 8], + [segment.virtual_address, 16], + [segment.physical_address, 24], + [segment.file_size, 32], + [segment.memory_size, 40], + [segment.alignment, 48], + ] as const) + bytes.writeBigUInt64LE(BigInt(field), start + offset); + } + for (const note of value.notes) { + const start = Number(BigInt(note.location.offset)); + bytes.writeUInt32LE(Number(BigInt(note.owner_location.bytes)), start); + bytes.writeUInt32LE( + Number(BigInt(note.descriptor_location.bytes)), + start + 4, + ); + bytes.writeUInt32LE( + note.type === "NT_PRSTATUS" ? 1 : 0x53494749, + start + 8, + ); + Buffer.from(note.owner_bytes_base64, "base64").copy( + bytes, + Number(BigInt(note.owner_location.offset)), + ); + Buffer.from(note.descriptor_bytes_base64, "base64").copy( + bytes, + Number(BigInt(note.descriptor_location.offset)), + ); + } + return bytes; +}; + +export const RECORDED_CRASH_TEST_PROVIDER = { + id: "fixture-recorded-core", + name: "Source-owned recorded-core seam", + version: "1", +} as const; + +/** Source-owned debugger representation with deliberately unreported permission flags. */ +export const recordedCrashDebuggerFixture = (): Extract< + RecordedCrash["debugger"], + { status: "available" } +> => ({ + status: "available", + gdb_version: "source seam", + pwndbg_version: "source seam", + connection: "core", + executable: null, + historical_pid: 123, + reported_limits: { + address_space_bytes: 3221225472, + cpu_seconds: 30, + file_size_bytes: 67108864, + }, + maps: [ + { + start: "0x1000", + end: "0x2000", + offset: "0x0", + reported_flags: 0, + permissions: null, + pathname_display: "recorded/path", + current_file_identity: "unknown", + }, + ], + confidence: "derived", + completeness: "unknown", + diagnostics: { stdout: "", stderr: "", truncated: false }, +}); diff --git a/third_party/pwndbg/LICENSE.md b/third_party/pwndbg/LICENSE.md new file mode 100644 index 0000000000..bbc1e5d145 --- /dev/null +++ b/third_party/pwndbg/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2026 Dominik 'Disconnect3d' Czarnota + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/pwndbg/README.md b/third_party/pwndbg/README.md new file mode 100644 index 0000000000..e0dff52bc2 --- /dev/null +++ b/third_party/pwndbg/README.md @@ -0,0 +1,23 @@ +# pwndbg core context adapter + +REA uses caller-supplied unchanged [pwndbg](https://github.com/pwndbg/pwndbg) +2026.09.15, commit `55596aa676a2756efb8d67034f31675170eabe48`, and GDB. +The original MIT license is retained in `LICENSE.md`. Engine source and its +dependencies are not vendored or acquired by REA. + +The adapter executes the selected upstream `gdbinit.py` with its original +`__file__`, an explicitly selected existing virtual environment, and +`PWNDBG_NO_AUTOUPDATE=1`. The upstream entrypoint and dependencies are trusted +caller-supplied executable code. Reported versions establish observed behavior, +not source authenticity. The real verification lane checks out the exact commit +and uses its frozen `uv.lock` in isolated runner storage. + +Only `pwndbg.aglib.vmmap.get()` is queried, after establishing a core-only GDB +connection without an executable. All original core notes and recorded register +values come from the independent pwntools/pyelftools decoder. Mapping candidates +are derived; original pathnames do not identify current host files, and reported +flags zero leave permissions unknown. REA does not expose upstream command +execution, live attach, target launch, implicit executable pairing or updates. + +Local actual verification uses Linux x64, GDB 13.1 and the exact upstream tag/build. +Other GDB versions require their corresponding real workflow before being claimed.