fix(firmware): preserve admission and extraction coverage outcomes

This commit is contained in:
morluto committed 2026-10-24 03:00:09 +08:00
1 parent 0a0f98ca4c
commit 15450643f7
18 files changed
+766 -85

No files matched your search

+27 -5
View File
@@ -27,7 +27,10 @@ checks the tool's `--version` banner when executing a request. Binwalk must
print `binwalk <version>` and Unblob must print the version alone. Inspection and
extraction have independent prerequisites; one does not require the other.
Both require util-linux `prlimit`, normally `/usr/bin/prlimit`; override its
absolute path with `REA_FIRMWARE_PRLIMIT_COMMAND` if needed.
absolute path with `REA_FIRMWARE_PRLIMIT_COMMAND` if needed. `rea doctor` does
not currently inspect firmware readiness. Engine selection/version checks run
when the firmware operation is requested; successful general doctor checks do
not establish firmware tool or external-extractor availability.
REA retains at most 8 MiB of combined stdout and stderr per command. Exceeding
that diagnostic budget stops analysis with an explicit truncated-output error;
@@ -71,6 +74,15 @@ Unblob submodule. In Git Bash, prefix the `sparse-checkout` command with
`MSYS_NO_PATHCONV=1`; otherwise `!/tests/integration/` is rewritten as a
Windows path.
The extraction destination must be absent with an existing writable parent.
REA checks that selection before starting an engine and still acquires it
exclusively during publication. Invalid sources and destinations identify the
caller argument; failures inside an acquired workspace keep their own cause.
Published file `relative_path` preserves producer spelling for lineage; `path`
uses the writer's actual normalized filename. An extracted filename that cannot
be represented losslessly as UTF-8 fails publication explicitly with its raw
name bytes in hexadecimal instead of silently omitting the entry.
## CLI and MCP
```sh
@@ -123,14 +135,19 @@ build's source revision, so that field remains unknown.
selected input have a known `root_file_range`, adjusted by the selected offset.
- Decompressed children have unknown original byte ranges and runtime addresses.
Task lineage does not establish a byte-for-byte mapping through decompression.
- `partial` marks unknown/encrypted chunks, reported extraction problems,
- `partial` marks unknown/encrypted chunks, selected input bytes outside all
reported chunk intervals, reported extraction problems,
unpublished links/special files or reached depth limits. `complete` means the
engine finished its configured extraction without those reported conditions;
selected input is covered by reported chunk intervals and the engine finished
its configured extraction without those reported conditions;
default skip rules and format coverage still apply. It does not establish
complete firmware reconstruction.
- `extraction_gaps` makes those conditions recoverable: a row identifies an
unextracted region or a failed task/stage. A chunk retains its identity and
byte range, plus its original-file range when it belongs to the selected input.
`unclassified-range` rows identify selected input intervals with no reported
chunk; their provider identity and handler stay null. An empty chunk inventory
therefore cannot establish complete extraction of a nonempty selected input.
A multi-file stage retains its reported ID and handler with unknown byte ranges;
a standalone task failure keeps unavailable stage identity null. Rows retain exact dependency names from
the producer's missing-dependency reports; an accepted report with no names
@@ -215,10 +232,15 @@ tree semantics, vulnerability scanning, emulation or hardware runtime capture.
Diagnostic output is bounded with exact captured byte counts.
- Default staging budget: **256 MiB** and **10,000 entries**, configurable using
`max_output_bytes` / `max_output_files` (CLI `--max-output-bytes` /
`--max-output-files`). Depth defaults to 3, allowed range 1–10.
`--max-output-files`). Exceeding either selected staging budget returns a
nonretryable resource constraint naming the selected limit. Depth defaults to
3, allowed range 1–10.
Staging budgets are polled and rechecked after exit; short-lived overshoot is
possible. The inherited file-size limit bounds each file, not total storage;
possible. The per-file process limit is the larger of the selected payload budget
and the fixed report/diagnostic file allowances, so a small payload selection
still permits the mandatory report. A tighter inherited host limit still applies.
The inherited file-size limit bounds each file, not total storage;
address-space limits are per process, not an aggregate memory quota. Worker
count is not CPU affinity for every external extractor. These limits do not
claim cgroup/container containment. Cancellation cleans the owned process group
@@ -9,11 +9,12 @@ import {
} from "../../domain/firmware/firmwareAnalysis.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { Evidence } from "../../domain/evidence.js";
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
import { projectInputIssues } from "../../domain/inputIssueProjection.js";
import { err, type Result } from "../../domain/result.js";
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
@@ -38,11 +39,17 @@ export class FirmwareAnalysisService {
const parsed = firmwareRequestSchema.safeParse({ operation, input });
if (!parsed.success)
return err(
analysisInputErrorFromIssues(
new AnalysisInputError(
operation,
parsed.error.issues,
{ operation, input },
{ cause: parsed.error },
projectInputIssues(parsed.error.issues, { operation, input }).map(
(issue) => ({
...issue,
// `input` is the internal discriminated request envelope.
path:
issue.path[0] === "input" ? issue.path.slice(1) : issue.path,
}),
),
),
);
const executed = await this.provider.execute(parsed.data, options);
+76 -21
View File
@@ -1,6 +1,8 @@
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import {
access,
lstat,
chmod,
mkdir,
open,
@@ -96,27 +98,8 @@ export class SafeOutputTree {
platform: NodeJS.Platform = process.platform,
options: { readonly acceptExistingEmpty?: boolean } = {},
): Promise<SafeOutputTree> {
if (!isAbsolute(outputRoot))
throw new SafeOutputTreeDestinationRefusal(
"Extraction output root must be absolute",
);
// Path syntax follows the host; `platform` selects only the name policy.
const requested = absoluteCallerPath(outputRoot);
const name = basename(requested);
if (name === "." || name === "..")
throw new SafeOutputTreeDestinationRefusal(
"Invalid extraction output root",
);
const parent = await realpath(dirname(requested)).catch(
(cause: unknown) => {
throw new ArtifactReaderFailure(
"unavailable",
"Extraction output parent is unavailable",
{ cause },
);
},
);
const canonicalOutput = join(parent, name);
const { requested, canonicalOutput } =
await this.#resolveDestination(outputRoot);
let created = true;
await mkdir(canonicalOutput, { mode: 0o700 }).catch((cause: unknown) => {
if (!isAlreadyExists(cause)) throw cause;
@@ -161,6 +144,78 @@ export class SafeOutputTree {
}
}
/** Read-only early admission; create() still rechecks and acquires exclusively. */
static async checkDestination(outputRoot: string): Promise<void> {
const { canonicalOutput } = await this.#resolveDestination(outputRoot);
const existing = await lstat(canonicalOutput).catch((cause: unknown) => {
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
return undefined;
if (
cause instanceof Error &&
"code" in cause &&
["ENOTDIR", "EACCES", "EPERM"].includes(String(cause.code))
)
throw new SafeOutputTreeDestinationRefusal(
"Extraction output path is unavailable",
{ cause },
);
throw cause;
});
if (existing !== undefined)
throw new SafeOutputTreeDestinationRefusal(
"Extraction output root already exists",
);
await access(
dirname(canonicalOutput),
constants.W_OK | constants.X_OK,
).catch((cause: unknown) => {
if (
cause instanceof Error &&
"code" in cause &&
["EACCES", "EPERM", "ENOENT", "ENOTDIR"].includes(String(cause.code))
)
throw new SafeOutputTreeDestinationRefusal(
"Extraction output parent is not writable",
{ cause },
);
throw cause;
});
}
static async #resolveDestination(outputRoot: string) {
if (!isAbsolute(outputRoot))
throw new SafeOutputTreeDestinationRefusal(
"Extraction output root must be absolute",
);
// Path syntax follows the host; `platform` selects only the name policy.
const requested = absoluteCallerPath(outputRoot);
const name = basename(requested);
if (name === "." || name === "..")
throw new SafeOutputTreeDestinationRefusal(
"Invalid extraction output root",
);
const parent = await realpath(dirname(requested)).catch(
(cause: unknown) => {
if (
cause instanceof Error &&
"code" in cause &&
["ENOENT", "ENOTDIR", "EACCES", "EPERM"].includes(String(cause.code))
)
throw new SafeOutputTreeDestinationRefusal(
"Extraction output parent is unavailable",
{ cause },
);
throw new ArtifactReaderFailure(
"unavailable",
"Extraction output parent is unavailable",
{ cause },
);
},
);
const canonicalOutput = join(parent, name);
return { requested, canonicalOutput };
}
get outputRoot(): string {
return this.#outputRoot;
}
+2 -1
View File
@@ -4,7 +4,8 @@ import { ArtifactReaderFailure } from "./ArtifactReader.js";
/**
* `SafeOutputTree.create` refused the caller's selected destination itself:
* a relative or reserved root, an existing root, or an accepted existing root
* that is not an empty directory.
* that is not an empty directory, or an unavailable parent observed during
* destination admission.
* Setup failures after a tree is acquired are not refusals of the selection.
*/
export class SafeOutputTreeDestinationRefusal extends ArtifactReaderFailure {
@@ -13,7 +13,7 @@ export const FIRMWARE_TOOL_CONTRACTS = [
name: "inspect_firmware_regions",
...toolContractMetadata("inspect_firmware_regions"),
kind: "firmware-provider",
description: `Inspect signature candidates in a local firmware file with ${FIRMWARE_TOOL_REQUIREMENTS.binwalk.requirement}. Returns offsets, reported lengths, descriptions and raw confidence values. Length validation and runtime addresses remain unknown. No extraction or target execution. Linux, caller-supplied tool and util-linux prlimit required; 128 MiB input limit, one worker. Adjust analysis_timeout_ms for the analysis wall-clock budget (default 120000); fixed per-process limits are unchanged.`,
description: `Inspect signature candidates in a local firmware file with ${FIRMWARE_TOOL_REQUIREMENTS.binwalk.requirement}. Returns offsets, reported lengths, descriptions and raw confidence values. Length validation and runtime addresses remain unknown. No extraction or target execution. Linux, caller-supplied tool, util-linux prlimit and procps ps required; 128 MiB input limit, one worker. Adjust analysis_timeout_ms for the analysis wall-clock budget (default 120000); fixed per-process limits are unchanged.`,
inputSchema: firmwareInputSchemas.inspect_firmware_regions,
outputSchema: evidenceResultOf(
firmwareResultSchemas.inspect_firmware_regions,
@@ -29,7 +29,7 @@ export const FIRMWARE_TOOL_CONTRACTS = [
name: "extract_firmware",
...toolContractMetadata("extract_firmware"),
kind: "firmware-provider",
description: `Extract caller-selected firmware bytes with ${FIRMWARE_TOOL_REQUIREMENTS.unblob.requirement} into an absent output directory. Returns verified child paths/digests, task derivations, unknown chunks, extraction diagnostics and explicit extraction_gaps rows naming unextracted regions, missing extractor dependencies and a continuation_id referencing shared extraction_continuations recovery instructions. Preserves the default sandbox; publishes regular files only. Linux, caller-supplied tool, format-specific extractors and util-linux prlimit required. No mounts or extracted-code execution. Select a returned native file with open_binary for Ghidra analysis; decompressed offsets are not original-file addresses.`,
description: `Extract caller-selected firmware bytes with ${FIRMWARE_TOOL_REQUIREMENTS.unblob.requirement} into an absent output directory. Returns verified child paths/digests, task derivations, unknown chunks and uncovered selected input ranges, extraction diagnostics and explicit extraction_gaps rows naming unextracted regions, missing extractor dependencies and a continuation_id referencing shared extraction_continuations recovery instructions. Preserves the default sandbox; publishes regular files only. Linux, caller-supplied tool, format-specific extractors, util-linux prlimit and procps ps required. No mounts or extracted-code execution. Select a returned native file with open_binary for Ghidra analysis; decompressed offsets are not original-file addresses.`,
inputSchema: firmwareInputSchemas.extract_firmware,
outputSchema: evidenceResultOf(firmwareResultSchemas.extract_firmware),
examples: [
+3
View File
@@ -117,6 +117,7 @@ export const extractionGap = z.strictObject({
kind: z.enum([
"missing-extractor-dependency",
"unknown-chunk-handler",
"unclassified-range",
"encrypted-chunk",
"depth-limited",
"extractor-failure",
@@ -171,6 +172,8 @@ export const firmwareResultSchemas = {
coverage: z.enum(["complete", "partial"]),
files: z.array(
z.strictObject({
// Producer-relative identity retains source spelling; `path` identifies
// the actual published file after destination name normalization.
relative_path: z.string(),
path: z.string(),
size: bytes,
+57
View File
@@ -8,6 +8,7 @@ import { expect, it as test, vi } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { FIRMWARE_LIMITS } from "./FirmwareRelease.js";
import { FirmwareFileAccess } from "./FirmwareFiles.js";
import {
resolveFirmwareCommand,
@@ -16,6 +17,7 @@ import {
import {
spawnOwnedProviderProcess,
type ProviderProcessHandle,
type OwnedProviderProcessSpawnOptions,
} from "../process/ProviderProcess.js";
import { AnalysisError } from "../domain/analysisErrorBase.js";
import { ExecFileOutputCancelledError } from "../process/ExecFileOutput.js";
@@ -38,6 +40,7 @@ it.each([
: { caller: "opaque admission cancellation" };
const retainCleanup = vi.fn();
const failure: unknown = await runFirmwareCommand({
fileSizeCeiling: Number.MAX_SAFE_INTEGER,
engine: "binwalk",
operation: "inspect_firmware_regions",
command: "/usr/bin/true",
@@ -88,6 +91,7 @@ it("does not relabel an unrelated launch failure when a deadline also fires", as
const failure = new Error("distinct launch refusal");
await expect(
runFirmwareCommand({
fileSizeCeiling: Number.MAX_SAFE_INTEGER,
engine: "binwalk",
operation: "inspect_firmware_regions",
command: "/usr/bin/true",
@@ -226,6 +230,7 @@ it("keeps captured tool output when firmware stops on SIGXCPU", async () => {
}, 15);
const analysis = new AbortController();
const failure: unknown = await runFirmwareCommand({
fileSizeCeiling: Number.MAX_SAFE_INTEGER,
engine: "binwalk",
operation: "inspect_firmware_regions",
command: "/opt/binwalk",
@@ -278,3 +283,55 @@ const executableFixture = async (root: string, name: string) => {
await chmod(path, 0o700);
return path;
};
test.each([
{
selected: 1,
inherited: Number.MAX_SAFE_INTEGER,
expected: 8 * 1024 * 1024,
},
{
selected: 16 * 1024 * 1024,
inherited: Number.MAX_SAFE_INTEGER,
expected: 16 * 1024 * 1024,
},
{ selected: 16 * 1024 * 1024, inherited: 1024, expected: 1024 },
])(
"separates payload $selected from metadata file headroom and inherited ceiling $inherited",
async ({ selected, inherited, expected }) => {
const root = await createTestTempDirectory("rea-firmware-file-policy-");
const refused = new Error("stop after inspecting launch policy");
const launcher = vi.fn(
async (options: OwnedProviderProcessSpawnOptions) => {
expect(options.arguments).toContain(`--fsize=${expected}:${expected}`);
expect(options.arguments).toContain(
`--as=${FIRMWARE_LIMITS.addressSpaceBytes}:${FIRMWARE_LIMITS.addressSpaceBytes}`,
);
throw refused;
},
);
await expect(
runFirmwareCommand({
engine: "unblob",
operation: "extract_firmware",
command: "/synthetic/unblob",
limiter: "/synthetic/prlimit",
fileSizeCeiling: inherited,
args: [],
cwd: root,
environment: {},
signal: new AbortController().signal,
deadline: {
phase: "analysis",
signal: new AbortController().signal,
timeoutMs: 120_000,
expiresAtMs: performance.now() + 120_000,
},
launcher,
outputBudget: { root, bytes: selected, entries: 10 },
retainCleanup: vi.fn(),
}),
).rejects.toBe(refused);
expect(launcher).toHaveBeenCalledOnce();
},
);
+24 -6
View File
@@ -134,7 +134,12 @@ export const resolveFirmwareCommand = async (
`Resource limiter is not an executable regular file: ${limiter}: ${cause instanceof Error ? cause.message : String(cause)}; provide util-linux prlimit`,
);
}
return { command, limiter, sha256 };
return {
command,
limiter,
sha256,
fileSizeCeiling: await files.fileSizeCeiling(signal),
};
};
/** Execute one bounded provider command and verify its owned descendants have stopped. */
@@ -143,6 +148,8 @@ export const runFirmwareCommand = async (context: {
operation: string;
command: string;
limiter: string;
/** Observed inherited soft/hard file policy; never raised by metadata headroom. */
fileSizeCeiling: number;
args: readonly string[];
cwd: string;
environment: Readonly<Record<string, string | undefined>>;
@@ -154,6 +161,21 @@ export const runFirmwareCommand = async (context: {
retainCleanup(supervisor: ProviderProcessSupervisor): void;
}) => {
const { engine, operation, signal } = context;
// RLIMIT_FSIZE covers every producer file, including mandatory reports/logs.
// Aggregate extracted payload remains governed by outputBudget inventory.
const fileSizeBytes = Math.min(
context.fileSizeCeiling,
Math.max(
context.outputBudget?.bytes ?? 0,
FIRMWARE_LIMITS.reportBytes,
FIRMWARE_LIMITS.diagnosticBytes,
),
);
const fileSizeAdjustable =
context.outputBudget !== undefined &&
context.fileSizeCeiling > context.outputBudget.bytes &&
context.outputBudget.bytes >=
Math.max(FIRMWARE_LIMITS.reportBytes, FIRMWARE_LIMITS.diagnosticBytes);
const checkInterruption = () => {
const failure = firmwarePhaseInterruption(
operation,
@@ -170,8 +192,7 @@ export const runFirmwareCommand = async (context: {
...prlimitArgumentVector(
{
address_space_bytes: FIRMWARE_LIMITS.addressSpaceBytes,
file_bytes:
context.outputBudget?.bytes ?? FIRMWARE_LIMITS.reportBytes,
file_bytes: fileSizeBytes,
cpu_seconds: FIRMWARE_LIMITS.cpuSeconds,
},
context.command,
@@ -295,9 +316,6 @@ export const runFirmwareCommand = async (context: {
snapshot.exitCode === 1
)
) {
const fileSizeBytes =
context.outputBudget?.bytes ?? FIRMWARE_LIMITS.reportBytes;
const fileSizeAdjustable = context.outputBudget !== undefined;
const signalLimit = firmwareSignalLimitError(
operation,
snapshot.signal,
+102 -21
View File
@@ -2,15 +2,18 @@ import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { chmod, lstat, opendir, readlink, realpath } from "node:fs/promises";
import { isAbsolute, join, relative, sep } from "node:path";
import { Transform } from "node:stream";
import { Readable, Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import {
AnalysisCancelledError,
AnalysisAccessDeniedError,
AnalysisResourceConstraintError,
AnalysisInputError,
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import type { FirmwareRequest } from "../domain/firmware/firmwareAnalysis.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import { readLinuxResourceLimits } from "../process/LinuxResourceLimits.js";
import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
import { FIRMWARE_LIMITS } from "./FirmwareRelease.js";
import { firmwareLimitRecovery } from "./FirmwareFailures.js";
@@ -21,9 +24,11 @@ import {
withArtifactRegularFile,
assertArtifactReadActive,
mapArtifactFileFailure,
artifactSelectedPathErrno,
} from "../artifacts/ArtifactFileOperation.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { readArtifactFileChunks } from "../artifacts/readArtifactFileChunks.js";
import { decodeDirectoryEntryName } from "../filesystem/decodeDirectoryEntryName.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
/** One regular file or unpublished filesystem entry in a private extraction. */
@@ -57,6 +62,23 @@ export class FirmwareFileAccess {
return this.#resources.close();
}
/** Read inherited file policy through this same retryable descriptor owner. */
async fileSizeCeiling(signal?: AbortSignal): Promise<number> {
return this.#resources.run(async () => {
assertArtifactReadActive(signal);
const limits = await readLinuxResourceLimits(
{
address_space_bytes: FIRMWARE_LIMITS.addressSpaceBytes,
cpu_seconds: FIRMWARE_LIMITS.cpuSeconds,
file_bytes: Number.MAX_SAFE_INTEGER,
},
this.#resources,
);
assertArtifactReadActive(signal);
return limits.file_bytes;
});
}
/** Hash one admitted regular file without retaining its bytes in memory. */
hash(path: string, signal?: AbortSignal): Promise<string> {
return this.#resources.run(() => this.#hash(path, signal));
@@ -86,7 +108,14 @@ export class FirmwareFileAccess {
async #snapshot(request: FirmwareRequest, root: string, signal: AbortSignal) {
const sourcePath = absoluteCallerPath(request.input.path);
const source = await realpath(sourcePath);
const source = await realpath(sourcePath).catch((cause: unknown) => {
throw firmwareSourceAdmissionFailure(
request.operation,
sourcePath,
cause,
);
});
let sourceAdmitted = false;
const fullPath = join(root, "firmware.bin");
const copied = await withArtifactRegularFile(
source,
@@ -94,6 +123,7 @@ export class FirmwareFileAccess {
signal,
async (handle) => {
const stat = await handle.stat();
sourceAdmitted = true;
if (stat.size === 0 || stat.size > FIRMWARE_LIMITS.inputBytes)
throw firmwareInputFailure(request.operation);
const hash = createHash("sha256");
@@ -117,9 +147,10 @@ export class FirmwareFileAccess {
return { size, sha256: hash.digest("hex") };
},
).catch((cause: unknown) => {
throw firmwareAdmissionFailure(cause, (primary) =>
firmwareInputFailure(request.operation, primary),
);
// Destination writes and reads after admission retain their own failure.
throw sourceAdmitted
? cause
: firmwareSourceAdmissionFailure(request.operation, sourcePath, cause);
});
const { size, sha256 } = copied;
await chmod(fullPath, 0o400);
@@ -181,7 +212,14 @@ export class FirmwareFileAccess {
withArtifactRegularFile(entry.path, this.#resources, signal, (handle) =>
tree.write(
entry.relativePath,
readArtifactFileChunks(handle, { signal }),
// SafeOutputTree awaits destination setup before consuming. Defer
// the abort-bound descriptor stream until it has a consumer.
Readable.from(
(async function* () {
assertArtifactReadActive(signal);
yield* readArtifactFileChunks(handle, { signal });
})(),
),
{ sha256, bytes: entry.size },
signal,
),
@@ -273,15 +311,33 @@ const firmwareReportFailure = (
diagnostics: { report_state: state },
});
/** Preserve admission semantics inside the same exact owner's physical outcome. */
const firmwareAdmissionFailure = (
/** Attribute only failures observed while admitting the caller-selected source. */
const firmwareSourceAdmissionFailure = (
operation: string,
path: string,
cause: unknown,
classify: (cause: NonRegularFileReadError) => Error,
): unknown => {
return mapArtifactFileFailure(cause, (primary) =>
primary instanceof NonRegularFileReadError ? classify(primary) : primary,
);
};
): unknown =>
mapArtifactFileFailure(cause, (primary) => {
if (primary instanceof NonRegularFileReadError)
return firmwareInputFailure(operation, primary);
if (
primary instanceof Error &&
"code" in primary &&
(primary.code === "EACCES" || primary.code === "EPERM")
)
return new AnalysisAccessDeniedError(operation, path, primary.code, {
cause: primary,
});
if (artifactSelectedPathErrno(primary) !== undefined)
return new AnalysisInputError(operation, { cause: primary }, [
{
path: ["path"],
reason: "invalid_value",
message: "Firmware source must resolve to an accessible regular file",
},
]);
return primary;
});
const firmwareInputFailure = (operation: string, cause?: unknown) => {
const message = `Firmware must be a nonempty regular file no larger than ${String(FIRMWARE_LIMITS.inputBytes)} bytes. Select a stable copy within this allowance. ${firmwareLimitRecovery(`The firmware input allowance is ${String(FIRMWARE_LIMITS.inputBytes)} bytes`)}`;
@@ -321,7 +377,9 @@ export const inventoryFirmwareOutput = async (
directoryStat.isSymbolicLink()
)
continue;
const handle = await opendir(directory).catch((cause: unknown) => {
const handle = await opendir(directory, {
encoding: "buffer" as BufferEncoding,
}).catch((cause: unknown) => {
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
return undefined;
throw cause;
@@ -332,11 +390,20 @@ export const inventoryFirmwareOutput = async (
throwIfFirmwareInventoryCancelled(signal, operation);
count++;
if (count > maxEntries)
throw firmwareStagingLimit(operation, "max_output_files", maxEntries);
const decoded = decodeDirectoryEntryName(entry.name);
if (decoded.kind === "unrepresentable")
throw new AnalysisOutputError(
operation,
`Staging entry budget exceeded (${maxEntries})`,
"Extracted filename cannot be represented losslessly as UTF-8",
{
diagnostics: {
directory: relative(root, directory),
raw_name_hex: decoded.rawHex,
},
},
);
names.push(entry.name);
names.push(decoded.name);
}
for (const name of names.sort()) {
const path = join(directory, name);
@@ -356,10 +423,7 @@ export const inventoryFirmwareOutput = async (
}
if (stat.isFile()) bytes += stat.size;
if (bytes > maxBytes)
throw new AnalysisOutputError(
operation,
`Staging byte budget exceeded (${maxBytes})`,
);
throw firmwareStagingLimit(operation, "max_output_bytes", maxBytes);
entries.push({
path,
relativePath: relative(root, path).split(sep).join("/"),
@@ -378,6 +442,23 @@ export const inventoryFirmwareOutput = async (
);
};
/** The same selected aggregate budget is enforced during and after extraction. */
const firmwareStagingLimit = (
operation: string,
field: "max_output_files" | "max_output_bytes",
maximum: number,
) =>
new AnalysisResourceConstraintError(
operation,
"capacity",
`Firmware staging exceeded ${field} (${maximum})`,
{ [field]: maximum },
{
retryable: false,
remediationAction: `Increase ${field} within its schema maximum if the larger extraction is required, or select a materially narrower range. analysis_timeout_ms does not raise this staging allowance. Repeating the unchanged request fails the same way.`,
},
);
/** Bind producer paths to the selected input or private extraction tree. */
export const firmwareLogicalPath = (
path: string,
+13 -11
View File
@@ -1,4 +1,4 @@
import { join, isAbsolute } from "node:path";
import { join } from "node:path";
import type { FirmwareAnalysisPort } from "../application/firmware/FirmwareAnalysisPort.js";
import {
createAnalysisExecution,
@@ -15,7 +15,6 @@ import { AnalysisError } from "../domain/analysisErrorBase.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
@@ -24,7 +23,8 @@ import { jsonValueSchema } from "../domain/jsonValue.js";
import { err, ok } from "../domain/result.js";
import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js";
import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js";
import type { SafeOutputTree } from "../artifacts/SafeOutputTree.js";
import { SafeOutputTree } from "../artifacts/SafeOutputTree.js";
import { outputDirectoryAdmissionRefusal } from "../artifacts/SafeOutputTreeAdmission.js";
import { publishFirmwareExtraction } from "./FirmwarePublication.js";
import {
BINWALK_PROVIDER_IDENTITY,
@@ -346,6 +346,16 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
throw new AnalysisCancelledError(request.operation, {
cause: signal.reason,
});
if (request.operation === "extract_firmware")
await SafeOutputTree.checkDestination(
request.input.output_directory,
).catch((cause: unknown) => {
throw outputDirectoryAdmissionRefusal(
request.operation,
"output_directory",
cause,
);
});
const command = await resolveFirmwareCommand(
this.environment,
engineName,
@@ -442,14 +452,6 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
});
raw = { report, execution: processRun, version_execution: versionRun };
} else {
if (!isAbsolute(request.input.output_directory))
throw new AnalysisInputError(request.operation, undefined, [
{
path: ["output_directory"],
reason: "invalid_format",
message: "Extraction output directory must be absolute",
},
]);
const outputRoot = join(root.path, "extracted");
const processRun = await run(
[
+32 -9
View File
@@ -9,6 +9,7 @@ import { createHash } from "node:crypto";
import { join } from "node:path";
import type { z } from "zod";
import { SafeOutputTree } from "../artifacts/SafeOutputTree.js";
import { outputDirectoryAdmissionRefusal } from "../artifacts/SafeOutputTreeAdmission.js";
import { SafeOutputTreeCreationFailure } from "../artifacts/SafeOutputTreeCreationFailure.js";
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
@@ -26,6 +27,8 @@ const GAP_CONTINUATIONS: Record<ExtractionGap["kind"], string> = {
"REA installs nothing. Add the named dependency to your own environment and rerun extract_firmware on the same immutable input with a fresh output_directory to re-evaluate the affected region; otherwise it stays unextracted.",
"unknown-chunk-handler":
"If Binwalk is configured with REA_BINWALK_COMMAND, run inspect_firmware_regions on the same input for signature hypotheses. Without configured Binwalk that inspection is unavailable. Alternatively, select a materially narrower range with extract_firmware and a fresh output_directory; the region is not decoded content.",
"unclassified-range":
"The producer reported no chunk covering these bytes. Their format and extraction status remain unknown. If Binwalk is configured, inspect_firmware_regions can supply signature hypotheses; otherwise retain this range as unexamined.",
"encrypted-chunk":
"No static extraction continues past encryption; treat the region as unexamined rather than empty or corrupt.",
"extractor-failure":
@@ -156,10 +159,15 @@ export const publishFirmwareExtraction = async (context: {
"extract_firmware",
`Extracted bytes disagree with provider report: ${entry.relativePath}`,
);
await context.files.publish(entry, sha256, tree, context.signal);
const written = await context.files.publish(
entry,
sha256,
tree,
context.signal,
);
files.push({
relative_path: entry.relativePath,
path: join(tree.outputRoot, entry.relativePath),
path: join(tree.outputRoot, written.relativePath),
size: entry.size,
sha256,
reported_sha256: reported?.sha256 ?? null,
@@ -213,13 +221,20 @@ export const publishFirmwareExtraction = async (context: {
: new AnalysisCleanupLease(async () => {
await ownedTree.rollback();
});
const primary = firmwareFailure(cause, {
engine: "unblob",
operation: "extract_firmware",
deadline: context.deadline,
treeCleanupLease: treeLease,
publication: progress(),
});
const primary = firmwareFailure(
outputDirectoryAdmissionRefusal(
"extract_firmware",
"output_directory",
cause,
),
{
engine: "unblob",
operation: "extract_firmware",
deadline: context.deadline,
treeCleanupLease: treeLease,
publication: progress(),
},
);
try {
await treeLease?.release();
} catch (cleanupCause: unknown) {
@@ -434,6 +449,14 @@ const extractionGaps = (
};
const gaps: ExtractionGap[] = [
...dependencyGaps,
...normalized.unclassified_ranges.map((unclassified): ExtractionGap => ({
...emptyGap,
...unclassified,
kind: "unclassified-range",
continuation_id: unclassified.input_path.startsWith("$output/")
? nestedContinuationId(unclassified.input_path)
: continuation(GAP_CONTINUATIONS["unclassified-range"]),
})),
...normalized.failed_extraction_stages
.filter(({ kind }) => kind === "extractor-failure")
.map(({ input_path, provider_id, handler }): ExtractionGap => ({
+78
View File
@@ -1133,3 +1133,81 @@ describe("firmware producer boundaries", () => {
).toThrow();
});
});
describe("selected firmware byte coverage", () => {
const known = (id: string, start: number, end: number) => ({
__typename__: "ChunkReport",
id,
start_offset: start,
end_offset: end,
size: end - start,
handler_name: "gzip",
is_encrypted: false,
extraction_reports: [],
});
it.each([
{ name: "empty chunk inventory", spans: [], gaps: [[0, 20]] },
{
name: "internal and trailing holes",
spans: [
[2, 5],
[7, 10],
],
gaps: [
[0, 2],
[5, 2],
[10, 10],
],
},
{
name: "overlapping unordered coverage",
spans: [
[8, 20],
[0, 10],
[2, 4],
],
gaps: [],
},
{
name: "adjacent coverage",
spans: [
[10, 20],
[0, 10],
],
gaps: [],
},
] as const)(
"derives $name from byte intervals without invented chunk identities",
({ spans, gaps }) => {
const record = root();
const reports = record.reports.filter(
(report) => report.__typename__ !== "UnknownChunkReport",
);
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...reports,
...spans.map(([start, end], index) =>
known(String(index), start, end),
),
],
},
],
context,
);
expect(normalized.coverage).toBe(
gaps.length > 0 ? "partial" : "complete",
);
expect(normalized.unclassified_ranges).toEqual(
gaps.map(([offset, length]) => ({
input_path: "$input",
range: { offset, length },
root_file_range: { offset: 64 + offset, length },
})),
);
expect(normalized.chunks).toHaveLength(spans.length);
},
);
});
+24
View File
@@ -546,10 +546,33 @@ export const normalizeUnblobReport = (
"extract_firmware",
"Report contains tasks disconnected from the selected input",
);
// A producer can emit no UnknownChunkReport for an opaque file. Compute
// examined byte coverage from reported intervals rather than from empty errors.
const unclassifiedRanges: {
input_path: string;
range: { offset: number; length: number };
root_file_range: { offset: number; length: number };
}[] = [];
let coveredEnd = 0;
const gap = (offset: number, length: number) =>
unclassifiedRanges.push({
input_path: "$input",
range: { offset, length },
root_file_range: { offset: context.offset + offset, length },
});
for (const chunk of chunks
.filter((chunk) => chunk.input_path === "$input")
.sort((left, right) => left.range.offset - right.range.offset)) {
if (chunk.range.offset > coveredEnd)
gap(coveredEnd, chunk.range.offset - coveredEnd);
coveredEnd = Math.max(coveredEnd, chunk.range.offset + chunk.range.length);
}
if (coveredEnd < context.length) gap(coveredEnd, context.length - coveredEnd);
return {
files,
sizes,
chunks,
unclassified_ranges: unclassifiedRanges,
derivations,
depth_limited_paths: depths.sort(),
failed_extraction_stages: failedExtractionStages.sort((left, right) =>
@@ -595,6 +618,7 @@ export const normalizeUnblobReport = (
: 0,
),
coverage:
unclassifiedRanges.length > 0 ||
depths.length > 0 ||
diagnostics.length > 0 ||
chunks.some((c) => c.handler === null || c.encrypted === true)
@@ -45,7 +45,7 @@ cliTest(
operation: "extract_firmware",
issues: [
{
path: ["input", "range", missing],
path: ["range", missing],
reason: "missing_argument",
},
],
@@ -787,13 +787,17 @@ it("preserves existing output and rejects invalid intervals before invoking extr
const previous = fixture.launches.length;
const invalid = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
output_directory: `${fixture.output}-invalid-range`,
range: { offset: 100, length: 1 },
});
expect(invalid).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError" },
});
if (invalid.ok) throw new Error("Expected out-of-file-range rejection");
expect(projectAnalysisError(invalid.error)).toMatchObject({
details: { issues: [{ path: ["range"], reason: "out_of_range" }] },
});
expect(fixture.launches).toHaveLength(previous);
await assertFirmwareCleanup(fixture.launches);
});
@@ -0,0 +1,159 @@
import { createHash } from "node:crypto";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { FirmwareProvider } from "../../../src/firmware/FirmwareProvider.js";
import { FirmwareAnalysisService } from "../../../src/application/firmware/FirmwareAnalysisService.js";
import { FirmwareFileAccess } from "../../../src/firmware/FirmwareFiles.js";
import { publishFirmwareExtraction } from "../../../src/firmware/FirmwarePublication.js";
import { normalizeUnblobReport } from "../../../src/firmware/FirmwareReports.js";
import { startFirmwareDeadline } from "../../../src/firmware/FirmwareDeadline.js";
import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
it.each(["existing", "missing-parent", "relative"] as const)(
"refuses %s destinations before selecting or launching a firmware engine",
async (kind) => {
const root = await createTestTempDirectory("rea-firmware-destination-");
const output =
kind === "existing"
? join(root, "exists")
: kind === "missing-parent"
? join(root, "missing", "output")
: "relative-output";
if (kind === "existing") await mkdir(output);
const launcher = vi.fn(async () => {
throw new Error("Must not launch");
});
const provider = new FirmwareProvider({}, launcher);
try {
const result = await new FirmwareAnalysisService(provider).execute(
"extract_firmware",
{ path: join(root, "absent-input"), output_directory: output },
);
expect(result).toMatchObject({
ok: false,
error: {
_tag: "AnalysisInputError",
issues: [{ path: ["output_directory"], reason: "invalid_value" }],
},
});
expect(launcher).not.toHaveBeenCalled();
} finally {
await provider.close();
}
},
);
it("early destination admission creates nothing and publication still acquires exclusively", async () => {
const root = await createTestTempDirectory("rea-firmware-preflight-");
const output = join(root, "output");
await SafeOutputTree.checkDestination(output);
await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" });
await mkdir(output);
await expect(SafeOutputTree.create(output)).rejects.toMatchObject({
name: "SafeOutputTreeDestinationRefusal",
});
expect(await access(output)).toBeUndefined();
});
it("publishes the writer's normalized path while retaining source spelling and unclassified input coverage", async () => {
const root = await createTestTempDirectory("rea-firmware-published-name-");
const staging = join(root, "staging");
await mkdir(staging);
const name = "cafe\u0301.bin";
const bytes = Buffer.from("child");
const path = join(staging, name);
await writeFile(path, bytes);
const sha256 = createHash("sha256").update(bytes).digest("hex");
const inputPath = join(root, "input");
const inputHash = "a".repeat(64);
const normalized = normalizeUnblobReport(
[
{
task: { path: inputPath, depth: 0, blob_id: "", is_multi_file: false },
reports: [
{
__typename__: "StatReport",
path: inputPath,
size: 1,
is_file: true,
is_dir: false,
is_link: false,
link_target: null,
},
{ __typename__: "HashReport", sha256: inputHash },
],
subtasks: [],
},
],
{
inputPath,
outputRoot: staging,
sha256: inputHash,
length: 1,
offset: 7,
maxDepth: 3,
},
);
const files = new FirmwareFileAccess();
const output = join(root, "published");
const retainCleanup = vi.fn();
try {
const result = await publishFirmwareExtraction({
files,
workspacePath: root,
deadline: startFirmwareDeadline("publication", 120_000),
analysisTimeoutMs: 120_000,
outputDirectory: output,
entries: [
{
path,
relativePath: name,
size: bytes.length,
kind: "file",
linkTarget: null,
},
],
normalized,
selection: { offset: 7, length: 1, sha256: inputHash },
maxDepth: 3,
engine: {
name: "unblob",
version: "26.6.4",
executable_path: "/synthetic/unblob",
executable_sha256: inputHash,
source_revision: null,
worker_count: 1,
},
signal: new AbortController().signal,
retainCleanup,
});
expect(result.files).toMatchObject([
{
relative_path: name,
path: join(output, name.normalize("NFC")),
sha256,
},
]);
const published = result.files[0];
if (published === undefined) throw new Error("Expected published child");
expect(await readFile(published.path)).toEqual(bytes);
expect(result).toMatchObject({
coverage: "partial",
extraction_gaps: [
{
kind: "unclassified-range",
input_path: "$input",
provider_id: null,
handler: null,
range: { offset: 0, length: 1 },
root_file_range: { offset: 7, length: 1 },
},
],
});
expect(retainCleanup).not.toHaveBeenCalled();
} finally {
await files.close();
}
});
@@ -1,6 +1,6 @@
import { mkdir, writeFile } from "node:fs/promises";
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { expect, it, vi } from "vitest";
import {
inventoryFirmwareOutput,
@@ -10,6 +10,9 @@ import { readFirmwareCommandReport } from "../../../src/firmware/FirmwareCommand
import { FIRMWARE_LIMITS } from "../../../src/firmware/FirmwareRelease.js";
import { AnalysisError } from "../../../src/domain/analysisErrorBase.js";
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js";
import { firmwareRequestSchema } from "../../../src/domain/firmware/firmwareAnalysis.js";
import { createHash } from "node:crypto";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
it.each([
@@ -161,3 +164,147 @@ it("orders extracted firmware paths by code point independently of host locale",
"\u00e4.bin",
]);
});
it.skipIf(process.platform !== "linux")(
"refuses raw filenames that cannot round-trip through UTF-8 instead of omitting their bytes",
async () => {
const root = await createTestTempDirectory("rea-firmware-raw-name-");
await writeFile(
Buffer.concat([
Buffer.from(`${root}/bad`),
Buffer.from([0xff]),
Buffer.from(".bin"),
]),
"123",
);
await writeFile(join(root, "good.bin"), "1234");
await expect(
inventoryFirmwareOutput(root, 4, 10, "extract_firmware"),
).rejects.toMatchObject({
_tag: "AnalysisOutputError",
diagnostics: { raw_name_hex: "626164ff2e62696e" },
});
},
);
it.each([
{ bytes: 1, files: 10, expected: "max_output_bytes" },
{ bytes: 10, files: 1, expected: "max_output_files" },
])(
"reports the selected $expected staging budget as a deterministic resource refusal",
async ({ bytes, files, expected }) => {
const root = await createTestTempDirectory("rea-firmware-staging-");
await writeFile(join(root, "a"), "1");
await writeFile(join(root, "b"), "2");
await expect(
inventoryFirmwareOutput(root, bytes, files, "extract_firmware"),
).rejects.toMatchObject({
_tag: "AnalysisResourceConstraintError",
resource: "capacity",
reportedLimits: { [expected]: 1 },
retryable: false,
remediationAction: expect.stringContaining(expected),
});
},
);
it("attributes missing sources to the selected path without relabeling snapshot destination failure", async () => {
const root = await createTestTempDirectory("rea-firmware-source-");
const files = new FirmwareFileAccess();
const request = (path: string) =>
firmwareRequestSchema.parse({
operation: "inspect_firmware_regions",
input: { path },
});
await expect(
files.snapshot(
request(join(root, "absent")),
root,
new AbortController().signal,
),
).rejects.toMatchObject({
_tag: "AnalysisInputError",
issues: [{ path: ["path"], reason: "invalid_value" }],
});
const source = join(root, "source");
await writeFile(source, "firmware");
await expect(
files.snapshot(
request(source),
join(root, "missing-workspace"),
new AbortController().signal,
),
).rejects.toMatchObject({ code: "ENOENT" });
await files.close();
});
it("does not activate an abort-bound source during awaited destination setup", async () => {
const root = await createTestTempDirectory("rea-firmware-publish-abort-");
const source = join(root, "source");
const bytes = Buffer.from("firmware");
await writeFile(source, bytes);
const files = new FirmwareFileAccess();
const tree = await SafeOutputTree.create(join(root, "published"));
const controller = new AbortController();
const reason = { stop: "during destination setup" };
const write = tree.write.bind(tree);
const spy = vi.spyOn(tree, "write").mockImplementation(async (...args) => {
controller.abort(reason);
// Yield a turn while the source has no consumer, reproducing the handoff.
await new Promise<void>((resolve) => setImmediate(resolve));
return write(...args);
});
try {
await expect(
files.publish(
{
path: source,
relativePath: "nested/file",
kind: "file",
size: bytes.length,
linkTarget: null,
},
createHash("sha256").update(bytes).digest("hex"),
tree,
controller.signal,
),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
} finally {
spy.mockRestore();
await tree.rollback();
await files.close();
}
await expect(access(join(root, "published"))).rejects.toMatchObject({
code: "ENOENT",
});
expect(await readFile(source)).toEqual(bytes);
});
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"keeps denied source access distinct from invalid paths and producer failures",
async () => {
const root = await createTestTempDirectory("rea-firmware-source-denial-");
const source = join(root, "input");
await writeFile(source, "firmware", { mode: 0o000 });
const files = new FirmwareFileAccess();
try {
await expect(
files.snapshot(
firmwareRequestSchema.parse({
operation: "inspect_firmware_regions",
input: { path: source },
}),
root,
new AbortController().signal,
),
).rejects.toMatchObject({
_tag: "AnalysisAccessDeniedError",
path: source,
systemCode: "EACCES",
});
} finally {
await chmod(source, 0o600);
await files.close();
}
},
);
@@ -45,7 +45,7 @@ describe("firmware request validation projection", () => {
operation: "extract_firmware",
issues: [
{
path: ["input", "range", missing],
path: ["range", missing],
reason: "missing_argument",
},
],