diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 3b8f40b01f..0cd7ceef26 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -105,6 +105,14 @@ overall native heap and total resident memory are not capped. Row limits do not override these byte and resource limits. Resource-limit failures do not return an incomplete schema as successful complete evidence. +If the combined database and WAL exceed the snapshot budget, the refusal is a +`resource_constraint` and reports both the configured byte limit and the exact +observed combined size as a decimal string. The remediation directs callers to +create a smaller consistent offline snapshot containing only relevant data +through the application's supported export workflow. A generic backup or copy +does not guarantee a smaller result; REA does not truncate or modify the +selected source. + MCP also applies its configured response budget to the complete Evidence envelope. If delivery exceeds that budget, it reports a transport constraint with guidance for exporting retained Evidence, requesting a matching larger diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs index f284625cc4..2be464ec47 100644 --- a/scripts/lib/test-projects.mjs +++ b/scripts/lib/test-projects.mjs @@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [ group: "source", include: [ "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts", ], }, { diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts index 09a1c08677..2e67b94067 100644 --- a/src/sqlite/SqliteDatabaseFailures.ts +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -79,7 +79,7 @@ export const sqliteDatabaseFailure = ( return new AnalysisInputError(operation, { cause }, [ { path: ["path"], - reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + reason: "invalid_format", message: cause.message, }, ]); @@ -110,7 +110,12 @@ export const sqliteDatabaseFailure = ( "The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.", }, ); - if (cause.code === "ENOENT" || cause.code === "ENOTDIR") + if ( + cause.code === "ENOENT" || + cause.code === "ENOTDIR" || + cause.code === "ELOOP" || + cause.code === "ENAMETOOLONG" + ) return new AnalysisInputError(operation, { cause }, [ { path: ["path"], diff --git a/src/sqlite/SqliteDatabaseSnapshot.ts b/src/sqlite/SqliteDatabaseSnapshot.ts index 1384fbdb6e..e3c35197f9 100644 --- a/src/sqlite/SqliteDatabaseSnapshot.ts +++ b/src/sqlite/SqliteDatabaseSnapshot.ts @@ -3,7 +3,10 @@ import { constants, type BigIntStats } from "node:fs"; import { join } from "node:path"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; -import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readStableArtifact, @@ -50,13 +53,35 @@ const duringAcquisition = async ( if ( cause instanceof Error && "code" in cause && - ["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code)) + ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes( + String(cause.code), + ) ) throw new ArtifactReaderFailure( "integrity", `SQLite snapshot file set changed during acquisition: ${path}`, { cause }, ); + if (cause instanceof ArtifactReaderFailure && cause.reason === "limit") { + const changed = new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set exceeded its observed size during acquisition: ${path}`, + { + cause, + ...(cause.partialObservation === undefined + ? {} + : { partialObservation: cause.partialObservation }), + }, + cause.details, + ); + if (cause.cleanup !== undefined) + throw ArtifactReaderFailure.withCleanup( + changed, + cause.cleanup, + cause.partialObservation, + ); + throw changed; + } throw cause; } }; @@ -195,6 +220,21 @@ export const captureSqliteDatabaseSnapshot = async ( if (database === undefined || wal === undefined || before[2] === undefined) throw new Error("Incomplete database file-set observation"); if (database === null) throw missingDatabase(path); + const selectedSize = database.size + (wal?.size ?? 0n); + if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) + throw new AnalysisResourceConstraintError( + "inspect_sqlite_database", + "file-size", + `Selected SQLite database and WAL total ${selectedSize.toString()} bytes, exceeding the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte offline snapshot budget: ${path}`, + { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: selectedSize.toString(), + }, + { + remediationAction: + "Use the application's supported export workflow to create a consistent, smaller offline SQLite snapshot containing only the relevant data, then inspect that snapshot. REA will not truncate or modify the selected source.", + }, + ); if (before[2] !== null) await verifyInactiveJournal( `${path}-journal`, @@ -204,12 +244,6 @@ export const captureSqliteDatabaseSnapshot = async ( "initial", signal, ); - const selectedSize = database.size + (wal?.size ?? 0n); - if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) - throw new ArtifactReaderFailure( - "limit", - `Selected SQLite database and WAL exceed the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte snapshot budget: ${path}`, - ); const snapshotPath = join(root, "database.snapshot"); const captured = await readObservedArtifact( path, diff --git a/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts b/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts new file mode 100644 index 0000000000..52c9d02ceb --- /dev/null +++ b/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts @@ -0,0 +1,194 @@ +import { lstat, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import type { StableArtifactFileSystem } from "../artifacts/readStableArtifact.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import type { AnalysisPartialObservation } from "../domain/analysisErrorBase.js"; +import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; +import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js"; + +it("reports filesystem sizes above the safe-integer range exactly before opening files", async () => { + const databaseBytes = BigInt(Number.MAX_SAFE_INTEGER) + 1n; + const walBytes = 17n; + const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-budget-")); + const databasePath = join(workspace, "selected.db"); + const walPath = `${databasePath}-wal`; + const journalPath = `${databasePath}-journal`; + const privateRoot = join(workspace, "private"); + await writeFile(databasePath, "tiny fixture"); + const stat = await lstat(databasePath, { bigint: true }); + const open = async (): Promise => { + throw new Error("An over-budget source must be refused before open"); + }; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === journalPath) + throw Object.assign(new Error("journal absent"), { code: "ENOENT" }); + const size = + path === databasePath + ? databaseBytes + : path === walPath + ? walBytes + : stat.size; + return Object.assign( + Object.create(Object.getPrototypeOf(stat)) as typeof stat, + stat, + { size }, + ); + }, + open, + }; + const resourceScope = new ArtifactResourceScope(); + + try { + const failure = await captureSqliteDatabaseSnapshot( + databasePath, + privateRoot, + resourceScope, + undefined, + fileSystem, + ).then( + () => { + throw new Error("Expected the oversized snapshot to be refused"); + }, + (cause: unknown) => cause, + ); + const projected = projectAnalysisError( + sqliteDatabaseFailure(failure, databasePath, "artifact-read"), + ); + expect(failure).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_sqlite_database", + resource: "file-size", + reportedLimits: { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: ( + databaseBytes + walBytes + ).toString(), + }, + remediationAction: expect.stringContaining( + "smaller offline SQLite snapshot", + ), + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { + reported_limits: { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: ( + databaseBytes + walBytes + ).toString(), + }, + }, + remediation: { + action: expect.stringContaining("supported export workflow"), + }, + }); + await expect(readdir(privateRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await resourceScope.close(); + await rm(workspace, { force: true, recursive: true }); + } +}); + +it.each([ + { label: "incomplete cleanup", retainCleanup: true }, + { label: "partial observation without cleanup", retainCleanup: false }, +])( + "preserves $label when a late read-limit failure becomes changed evidence", + async ({ retainCleanup }) => { + const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-read-cleanup-")); + const databasePath = join(workspace, "selected.db"); + await writeFile(databasePath, "tiny fixture"); + const stat = await lstat(databasePath, { bigint: true }); + const cleanup = { + reason: "fixture file-handle close failed", + resources: [databasePath], + }; + const partialObservation = { + operation: "address_to_file_offset", + address: "0x1000", + provider_file_offset: 16, + provider_source_path: null, + provider_image_header_hex: null, + original_file_offset: { + available: false, + reason: "fixture observation", + }, + } satisfies AnalysisPartialObservation; + const details = { + logicalPath: "selected.db", + declaredSha256: null, + calculatedSha256: null, + unpacked: false, + }; + const primary = new ArtifactReaderFailure( + "limit", + "Selected artifact grew beyond its byte limit", + { partialObservation }, + details, + ); + const limitFailure = retainCleanup + ? ArtifactReaderFailure.withCleanup(primary, cleanup) + : primary; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === `${databasePath}-journal`) + throw Object.assign(new Error("journal absent"), { code: "ENOENT" }); + return stat; + }, + open: async () => { + throw limitFailure; + }, + }; + const resourceScope = new ArtifactResourceScope(); + + try { + const failure = await captureSqliteDatabaseSnapshot( + databasePath, + join(workspace, "private"), + resourceScope, + undefined, + fileSystem, + ).then( + () => { + throw new Error("Expected the changed snapshot to be rejected"); + }, + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ reason: "integrity" }); + if (retainCleanup) + expect(failure).toMatchObject({ + cleanup, + cause: expect.any(ArtifactReaderFailure), + }); + expect(failure).toMatchObject({ partialObservation, details }); + const projectedFailure = sqliteDatabaseFailure( + failure, + databasePath, + "artifact-read", + ); + if (retainCleanup) { + expect(projectedFailure).toBeInstanceOf(ProviderCleanupError); + expect(projectedFailure).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: cleanup.resources, + }); + } + } finally { + await resourceScope.close(); + await rm(workspace, { force: true, recursive: true }); + } + }, +); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index d23c032a6e..1767129c61 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -1,5 +1,12 @@ import { createHash } from "node:crypto"; -import { readFile, stat, writeFile } from "node:fs/promises"; +import { + lstat, + readFile, + stat, + symlink, + truncate, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { expect, onTestFinished } from "vitest"; @@ -20,6 +27,107 @@ import { cliTest } from "../support/cli/cliFixture.js"; const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); +for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) { + cliTest.skipIf(expectedErrno === "ELOOP" && process.platform === "win32")( + `reports an initial ${expectedErrno} path failure through CLI and MCP`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-path-failure-"); + const path = + expectedErrno === "ELOOP" + ? join(root, "cycle-a", "selected.db") + : join(root, "a".repeat(300), "selected.db"); + if (expectedErrno === "ELOOP") { + await symlink("cycle-b", join(root, "cycle-a")); + await symlink("cycle-a", join(root, "cycle-b")); + } + const result = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + const expectedError = { + code: "invalid_request", + details: { + issues: [ + { + path: ["path"], + reason: "invalid_value", + message: expect.stringContaining(expectedErrno), + }, + ], + }, + }; + expect(result.json).toMatchObject(expectedError); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { path }); + expect(parseMcpToolError(response).error).toMatchObject(expectedError); + }, + ); +} + +cliTest( + "reports the combined database and WAL byte budget through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-size-budget-"); + const path = join(root, "selected.db"); + const walPath = `${path}-wal`; + const maximum = 256 * 1024 * 1024; + const databaseBytes = maximum / 2; + const walBytes = databaseBytes + 1; + await writeFile(path, ""); + await writeFile(walPath, ""); + await truncate(path, databaseBytes); + await truncate(walPath, walBytes); + const before = await Promise.all( + [path, walPath].map(async (filePath) => ({ + path: filePath, + state: await lstat(filePath, { bigint: true }), + })), + ); + const cliResult = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const { call } = await connectLocalToolsMcp(); + const mcpResponse = await call("inspect_sqlite_database", { path }); + const mcpError = parseMcpToolError(mcpResponse).error; + const expectedError = { + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { + reported_limits: { + sqlite_snapshot_input_bytes: maximum, + observed_sqlite_snapshot_input_bytes: String(maximum + 1), + }, + }, + remediation: { + action: expect.stringContaining("supported export workflow"), + }, + }; + expect(cliResult.exitCode).toBe(1); + expect(cliResult.json).toMatchObject(expectedError); + expect(mcpError).toMatchObject(expectedError); + const after = await Promise.all( + before.map(async ({ path: filePath }) => ({ + path: filePath, + state: await lstat(filePath, { bigint: true }), + })), + ); + for (const [index, original] of before.entries()) { + const current = after[index]; + expect(current?.state).toMatchObject({ + dev: original.state.dev, + ino: original.state.ino, + mode: original.state.mode, + size: original.state.size, + mtimeNs: original.state.mtimeNs, + ctimeNs: original.state.ctimeNs, + }); + } + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "bounds default context and projects exact columns without evaluating unrelated cells", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts index 4fce636ceb..44b52ceb17 100644 --- a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -121,6 +121,29 @@ it.runIf(process.platform !== "win32").each([ }, ); +it.each(["ELOOP", "ENAMETOOLONG"] as const)( + "treats %s during file acquisition as changed evidence, not an invalid input path", + async (code) => { + const { database, capture } = await fixture(); + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) + throw Object.assign( + new Error(`${code}: path changed after observation`), + { code }, + ); + return open(path, flags); + }, + }; + + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining(database.path), + }); + }, +); + it("refuses an active journal even before SQLite writes its magic header", async () => { const { database, capture } = await fixture(); const writer = new DatabaseSync(database.path);