diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index 9afce7cba3..97054ce256 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -735,13 +735,28 @@ export class LldbCallTracer implements NativeCallTracer { }), ); } - return ok({ + const trace: NativeCallTrace = { debugger: { ...lldb, version: output.version }, run: output, stdout: stdout.value, stderr: stderr.value, terminated, - }); + }; + if (exited.lateTimeout === true) + return err( + new AnalysisTimeoutError( + OPERATION, + input.duration_ms + DEADLINE_GRACE_MS, + { + partialObservation: partialObservationFromTrace( + request, + trace, + "timeout", + ), + }, + ), + ); + return ok(trace); } } @@ -1056,6 +1071,8 @@ type LldbExit = LldbOutcome & { readonly targetLaunched: boolean; readonly targetIdentity: string | undefined; readonly cleanupFailure: string | undefined; + /** Internal deadline elapsed during cleanup after a physical LLDB exit. */ + readonly lateTimeout?: true; readonly cleanupOwner?: ProviderProcessSupervisor; }; @@ -1095,17 +1112,34 @@ const runLldb = async ( let targetIdentity: string | undefined; let supervisor: ProviderProcessSupervisor | undefined; let retained = false; + let processFailureBeforeInterruption: + | { + readonly exitCode: number | null; + readonly output: string; + } + | undefined; const finish = async (outcome: LldbOutcome): Promise => { + // Select the primary result once at the owner boundary, before cleanup. + // An exit observed before interruption outranks an interrupted wait, while + // an already selected exit keeps its fuller snapshot and diagnostics. + const selectedOutcome = + outcome.kind !== "exited" && + processFailureBeforeInterruption !== undefined + ? { kind: "exited" as const, ...processFailureBeforeInterruption } + : outcome; const stopped = await supervisor?.stop(); retained = stopped?.status === "incomplete"; + const lateTimeout = + selectedOutcome.kind === "exited" && deadline.interruption === "timeout" + ? true + : undefined; return { - ...(deadline.interruption === undefined - ? outcome - : { kind: deadline.interruption }), + ...selectedOutcome, targetLaunched: supervisor !== undefined, targetIdentity, cleanupFailure: stopped?.status === "incomplete" ? stopped.reason : undefined, + ...(lateTimeout === undefined ? {} : { lateTimeout }), ...(retained && supervisor !== undefined ? { cleanupOwner: supervisor } : {}), @@ -1126,7 +1160,24 @@ const runLldb = async ( ownsProcessLifetime: true, cleanup: () => cleanupOwnedProcessGroup(launched.ownership), }, - { maxDiagnosticBytes: DIAGNOSTIC_BYTES }, + { + maxDiagnosticBytes: DIAGNOSTIC_BYTES, + onDiagnostic: (event) => { + // Latch failure at the process event; a later snapshot cannot order it against abort. + if ( + event.type === "exit" && + deadline.interruption === undefined && + (event.signal !== null || (event.code !== null && event.code !== 0)) + ) + processFailureBeforeInterruption ??= { + exitCode: event.code, + output: + `${event.snapshot.stdout.text}${event.snapshot.stderr.text}`.slice( + -DIAGNOSTIC_BYTES, + ), + }; + }, + }, ); let identityAckWritten = false; while (!(await supervisor.waitForOutputClose(20))) { @@ -1148,8 +1199,9 @@ const runLldb = async ( identityAckWritten = true; } const interruption = deadline.interruption; - if (interruption !== undefined) + if (interruption !== undefined) { return await finish({ kind: interruption }); + } await waitForAbortableDelay(20, deadline.signal); } const snapshot = supervisor.snapshot(); diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts index 8e124f5093..a2b653cd68 100644 --- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts +++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts @@ -1,5 +1,6 @@ import { access, + mkdtemp, readFile, rm, writeFile, @@ -8,11 +9,17 @@ import { type FileHandle, } from "node:fs/promises"; import { dirname } from "node:path"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { describe, expect, it, onTestFinished, vi } from "vitest"; import { z } from "zod"; -import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisTimeoutError, +} from "../../../../src/domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js"; import { ArtifactFileCleanupFailure } from "../../../../src/artifacts/ArtifactFileOperation.js"; import { resolvedAnalysisError } from "../../../../src/domain/analysisErrorBase.js"; @@ -26,6 +33,12 @@ import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; import { createDeferred } from "../../../fixtures/binarySession.js"; import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; +import { + ProviderProcessSupervisor, + type ProviderProcessStopOptions, + type ProviderProcessStopResult, +} from "../../../../src/process/ProviderProcess.js"; +import { ProviderStartupDeadline } from "../../../../src/process/ProviderDeadline.js"; import { NativeFixtureRunner, nativeMachoTarget, @@ -420,6 +433,309 @@ describe("LLDB failure retention through the production tracer", () => { }); }); +describe("LLDB primary outcome during owned process cleanup", () => { + it + .skipIf(process.platform === "win32") + .each(["caller cancellation", "deadline timeout"] as const)( + "preserves a physical bridge failure and no-target proof when %s arrives during successful process cleanup", + async (interruption) => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "rea-lldb-primary-outcome-"), + ); + const fakeLldb = join(fixtureRoot, "fake-lldb"); + await writeFile( + fakeLldb, + `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture physical bridge failure", target_process_created: false }));\nprocess.exit(17);\n`, + { mode: 0o700 }, + ); + + const controller = new AbortController(); + const stopEntered = createDeferred(); + const stopReleased = createDeferred(); + let stopping = false; + const deadlineSpy = + interruption === "deadline timeout" + ? vi + .spyOn(ProviderStartupDeadline.prototype, "interruption", "get") + .mockImplementation(() => (stopping ? "timeout" : undefined)) + : undefined; + let observedExit: + | ReturnType + | undefined; + let observedStop: ProviderProcessStopResult | undefined; + const originalStop = ProviderProcessSupervisor.prototype.stop; + const stopSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "stop") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + options?: ProviderProcessStopOptions, + ): Promise { + observedExit = this.observedExit(); + stopping = true; + stopEntered.resolve(undefined); + await stopReleased.promise; + observedStop = await originalStop.call(this, options); + return observedStop; + }); + const tracer = new LldbCallTracer({}, undefined, async () => + ok({ path: fakeLldb, sha256: "c".repeat(64) }), + ); + + try { + const pending = tracer.trace( + request, + interruption === "caller cancellation" + ? controller.signal + : undefined, + ); + await stopEntered.promise; + expect(observedExit).toEqual({ code: 17, signal: null }); + if (interruption === "caller cancellation") + controller.abort(new Error("abort during final cleanup")); + stopReleased.resolve(undefined); + const result = await pending; + + expect(observedStop?.status).toMatch( + /^(already-exited|verified-cleanup|terminated|killed)$/u, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the bridge target error"); + expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); + if (!(result.error instanceof AnalysisCapabilityUnavailableError)) + return; + expect(result.error.reason).toBe( + "lldb-target-unloadable: fixture physical bridge failure", + ); + expect(result.error.cleanup).toBeUndefined(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + expect(result.error.partialObservation).toMatchObject({ + coverage: { status: "partial", reason: "tracer-failure" }, + process: { pid: null }, + }); + } finally { + stopReleased.resolve(undefined); + stopSpy.mockRestore(); + deadlineSpy?.mockRestore(); + await tracer.close(); + await rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); +}); + +describe("LLDB process exit while output is still open", () => { + it.skipIf(process.platform === "win32")( + "keeps a nonzero leader exit observed before cancellation while inherited output is still open", + async () => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "rea-lldb-exit-before-cancel-"), + ); + const fakeLldb = join(fixtureRoot, "fake-lldb"); + await writeFile( + fakeLldb, + `#!${process.execPath}\nconst fs = require("node:fs");\nconst { spawn } = require("node:child_process");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nconst child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: ["ignore", "inherit", "inherit"] });\nchild.once("error", () => process.exit(92));\nchild.once("spawn", () => { fs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture leader failure before drain", target_process_created: false })); process.exit(17); });\n`, + { mode: 0o700 }, + ); + + const controller = new AbortController(); + const drainObserved = createDeferred(); + const drainReleased = createDeferred(); + const originalWait = + ProviderProcessSupervisor.prototype.waitForOutputClose; + const drainSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + timeoutMs?: number, + signal?: AbortSignal, + ): Promise { + const closed = await originalWait.call(this, timeoutMs, signal); + if (!closed && !stopping && this.observedExit()?.code === 17) { + drainObserved.resolve(undefined); + await drainReleased.promise; + } + return closed; + }); + let stopping = false; + let stopReceipt: ProviderProcessStopResult | undefined; + const originalStop = ProviderProcessSupervisor.prototype.stop; + const stopSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "stop") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + options?: ProviderProcessStopOptions, + ): Promise { + stopping = true; + stopReceipt = await originalStop.call(this, options); + return stopReceipt; + }); + const tracer = new LldbCallTracer({}, undefined, async () => + ok({ path: fakeLldb, sha256: "d".repeat(64) }), + ); + + try { + const pending = tracer.trace(request, controller.signal); + await drainObserved.promise; + controller.abort(new Error("abort after leader exit")); + drainReleased.resolve(undefined); + const result = await pending; + + expect(stopReceipt?.status).toMatch( + /^(already-exited|verified-cleanup|terminated|killed)$/u, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the bridge target error"); + expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); + if (!(result.error instanceof AnalysisCapabilityUnavailableError)) + return; + expect(result.error.reason).toBe( + "lldb-target-unloadable: fixture leader failure before drain", + ); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + } finally { + drainReleased.resolve(undefined); + drainSpy.mockRestore(); + stopSpy.mockRestore(); + await tracer.close(); + await rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); +}); + +describe("LLDB process exit at output-close loop completion", () => { + it.skipIf(process.platform === "win32")( + "keeps a nonzero exit when output closure and cancellation meet at loop exit", + async () => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "rea-lldb-closed-output-cancel-"), + ); + const fakeLldb = join(fixtureRoot, "fake-lldb"); + await writeFile( + fakeLldb, + `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture exit before closed-output cancellation", target_process_created: false }));\nprocess.exit(17);\n`, + { mode: 0o700 }, + ); + + const controller = new AbortController(); + let interruptionApplied = false; + let stopReceipt: ProviderProcessStopResult | undefined; + const originalWait = + ProviderProcessSupervisor.prototype.waitForOutputClose; + const waitSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + timeoutMs?: number, + signal?: AbortSignal, + ): Promise { + const closed = await originalWait.call(this, timeoutMs, signal); + if ( + !interruptionApplied && + closed && + this.observedExit()?.code === 17 + ) { + interruptionApplied = true; + controller.abort(new Error("abort at output-close loop exit")); + } + return closed; + }); + const originalStop = ProviderProcessSupervisor.prototype.stop; + const stopSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "stop") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + options?: ProviderProcessStopOptions, + ): Promise { + stopReceipt = await originalStop.call(this, options); + return stopReceipt; + }); + const tracer = new LldbCallTracer({}, undefined, async () => + ok({ path: fakeLldb, sha256: "f".repeat(64) }), + ); + + try { + const result = await tracer.trace(request, controller.signal); + + expect(interruptionApplied).toBe(true); + expect(stopReceipt?.status).toMatch( + /^(already-exited|verified-cleanup|terminated|killed)$/u, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the bridge target error"); + expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); + if (!(result.error instanceof AnalysisCapabilityUnavailableError)) + return; + expect(result.error.reason).toBe( + "lldb-target-unloadable: fixture exit before closed-output cancellation", + ); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + } finally { + waitSpy.mockRestore(); + stopSpy.mockRestore(); + await tracer.close(); + await rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); +}); + +describe("LLDB late timeout publication", () => { + it("converts a late deadline to timeout at successful-result publication", async () => { + let runtime: string | undefined; + const tracer = new LldbCallTracer( + {}, + async (_executable, arguments_) => { + const { config } = await configFromArguments(arguments_); + runtime = dirname(config.result_path); + await writeFile( + config.observation_path, + `${JSON.stringify({ kind: "event", event: EVENT })}\n`, + ); + await writeFile(config.stdout_capture_path, ""); + await writeFile(config.stderr_capture_path, ""); + await writeFile( + config.result_path, + JSON.stringify(successfulTracerOutput), + ); + return { + kind: "exited", + targetLaunched: false, + exitCode: 0, + output: "", + targetIdentity: undefined, + cleanupFailure: undefined, + lateTimeout: true, + }; + }, + async () => ok(fakeTool), + ); + + const result = await tracer.trace(request); + if (runtime === undefined) throw new Error("LLDB fixture did not run"); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the late deadline to be retained"); + expect(result.error).toBeInstanceOf(AnalysisTimeoutError); + if (!(result.error instanceof AnalysisTimeoutError)) return; + expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms); + expect(result.error.cleanup).toBeUndefined(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.partialObservation).toMatchObject({ + coverage: { status: "partial", reason: "timeout" }, + process: { + outcome: "exited", + exit_status: 0, + terminated: true, + }, + events: [EVENT], + }); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + describe("LLDB pre-target phase proof", () => { it("releases a proven pre-target integrity rejection and allows tracer reuse", async () => { let launches = 0;