fix(lldb): preserve observed failures through interrupted cleanup

src/native/LldbCallTracer.ts

@@ -735,13 +735,28 @@ export class LldbCallTracer implements NativeCallTracer {
         }),
       );
     }
-    return ok({
+    const trace: NativeCallTrace = {
       debugger: { ...lldb, version: output.version },
       run: output,
       stdout: stdout.value,
       stderr: stderr.value,
       terminated,
-    });
+    };
+    if (exited.lateTimeout === true)
+      return err(
+        new AnalysisTimeoutError(
+          OPERATION,
+          input.duration_ms + DEADLINE_GRACE_MS,
+          {
+            partialObservation: partialObservationFromTrace(
+              request,
+              trace,
+              "timeout",
+            ),
+          },
+        ),
+      );
+    return ok(trace);
   }
 }
 
@@ -1056,6 +1071,8 @@ type LldbExit = LldbOutcome & {
   readonly targetLaunched: boolean;
   readonly targetIdentity: string | undefined;
   readonly cleanupFailure: string | undefined;
+  /** Internal deadline elapsed during cleanup after a physical LLDB exit. */
+  readonly lateTimeout?: true;
   readonly cleanupOwner?: ProviderProcessSupervisor;
 };
 
@@ -1095,17 +1112,34 @@ const runLldb = async (
   let targetIdentity: string | undefined;
   let supervisor: ProviderProcessSupervisor | undefined;
   let retained = false;
+  let processFailureBeforeInterruption:
+    | {
+        readonly exitCode: number | null;
+        readonly output: string;
+      }
+    | undefined;
   const finish = async (outcome: LldbOutcome): Promise<LldbExit> => {
+    // Select the primary result once at the owner boundary, before cleanup.
+    // An exit observed before interruption outranks an interrupted wait, while
+    // an already selected exit keeps its fuller snapshot and diagnostics.
+    const selectedOutcome =
+      outcome.kind !== "exited" &&
+      processFailureBeforeInterruption !== undefined
+        ? { kind: "exited" as const, ...processFailureBeforeInterruption }
+        : outcome;
     const stopped = await supervisor?.stop();
     retained = stopped?.status === "incomplete";
+    const lateTimeout =
+      selectedOutcome.kind === "exited" && deadline.interruption === "timeout"
+        ? true
+        : undefined;
     return {
-      ...(deadline.interruption === undefined
-        ? outcome
-        : { kind: deadline.interruption }),
+      ...selectedOutcome,
       targetLaunched: supervisor !== undefined,
       targetIdentity,
       cleanupFailure:
         stopped?.status === "incomplete" ? stopped.reason : undefined,
+      ...(lateTimeout === undefined ? {} : { lateTimeout }),
       ...(retained && supervisor !== undefined
         ? { cleanupOwner: supervisor }
         : {}),
@@ -1126,7 +1160,24 @@ const runLldb = async (
         ownsProcessLifetime: true,
         cleanup: () => cleanupOwnedProcessGroup(launched.ownership),
       },
-      { maxDiagnosticBytes: DIAGNOSTIC_BYTES },
+      {
+        maxDiagnosticBytes: DIAGNOSTIC_BYTES,
+        onDiagnostic: (event) => {
+          // Latch failure at the process event; a later snapshot cannot order it against abort.
+          if (
+            event.type === "exit" &&
+            deadline.interruption === undefined &&
+            (event.signal !== null || (event.code !== null && event.code !== 0))
+          )
+            processFailureBeforeInterruption ??= {
+              exitCode: event.code,
+              output:
+                `${event.snapshot.stdout.text}${event.snapshot.stderr.text}`.slice(
+                  -DIAGNOSTIC_BYTES,
+                ),
+            };
+        },
+      },
     );
     let identityAckWritten = false;
     while (!(await supervisor.waitForOutputClose(20))) {
@@ -1148,8 +1199,9 @@ const runLldb = async (
         identityAckWritten = true;
       }
       const interruption = deadline.interruption;
-      if (interruption !== undefined)
+      if (interruption !== undefined) {
         return await finish({ kind: interruption });
+      }
       await waitForAbortableDelay(20, deadline.signal);
     }
     const snapshot = supervisor.snapshot();

tests/boundary/providers/native/lldbCallTracerFailures.test.ts

@@ -1,5 +1,6 @@
 import {
   access,
+  mkdtemp,
   readFile,
   rm,
   writeFile,
@@ -8,11 +9,17 @@ import {
   type FileHandle,
 } from "node:fs/promises";
 import { dirname } from "node:path";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 
 import { describe, expect, it, onTestFinished, vi } from "vitest";
 import { z } from "zod";
 
-import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js";
+import {
+  AnalysisCancelledError,
+  AnalysisCapabilityUnavailableError,
+  AnalysisTimeoutError,
+} from "../../../../src/domain/analysisErrorCore.js";
 import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js";
 import { ArtifactFileCleanupFailure } from "../../../../src/artifacts/ArtifactFileOperation.js";
 import { resolvedAnalysisError } from "../../../../src/domain/analysisErrorBase.js";
@@ -26,6 +33,12 @@ import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js";
 import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js";
 import { createDeferred } from "../../../fixtures/binarySession.js";
 import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js";
+import {
+  ProviderProcessSupervisor,
+  type ProviderProcessStopOptions,
+  type ProviderProcessStopResult,
+} from "../../../../src/process/ProviderProcess.js";
+import { ProviderStartupDeadline } from "../../../../src/process/ProviderDeadline.js";
 import {
   NativeFixtureRunner,
   nativeMachoTarget,
@@ -420,6 +433,309 @@ describe("LLDB failure retention through the production tracer", () => {
   });
 });
 
+describe("LLDB primary outcome during owned process cleanup", () => {
+  it
+    .skipIf(process.platform === "win32")
+    .each(["caller cancellation", "deadline timeout"] as const)(
+    "preserves a physical bridge failure and no-target proof when %s arrives during successful process cleanup",
+    async (interruption) => {
+      const fixtureRoot = await mkdtemp(
+        join(tmpdir(), "rea-lldb-primary-outcome-"),
+      );
+      const fakeLldb = join(fixtureRoot, "fake-lldb");
+      await writeFile(
+        fakeLldb,
+        `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture physical bridge failure", target_process_created: false }));\nprocess.exit(17);\n`,
+        { mode: 0o700 },
+      );
+
+      const controller = new AbortController();
+      const stopEntered = createDeferred<void>();
+      const stopReleased = createDeferred<void>();
+      let stopping = false;
+      const deadlineSpy =
+        interruption === "deadline timeout"
+          ? vi
+              .spyOn(ProviderStartupDeadline.prototype, "interruption", "get")
+              .mockImplementation(() => (stopping ? "timeout" : undefined))
+          : undefined;
+      let observedExit:
+        | ReturnType<ProviderProcessSupervisor["observedExit"]>
+        | undefined;
+      let observedStop: ProviderProcessStopResult | undefined;
+      const originalStop = ProviderProcessSupervisor.prototype.stop;
+      const stopSpy = vi
+        .spyOn(ProviderProcessSupervisor.prototype, "stop")
+        .mockImplementation(async function (
+          this: ProviderProcessSupervisor,
+          options?: ProviderProcessStopOptions,
+        ): Promise<ProviderProcessStopResult> {
+          observedExit = this.observedExit();
+          stopping = true;
+          stopEntered.resolve(undefined);
+          await stopReleased.promise;
+          observedStop = await originalStop.call(this, options);
+          return observedStop;
+        });
+      const tracer = new LldbCallTracer({}, undefined, async () =>
+        ok({ path: fakeLldb, sha256: "c".repeat(64) }),
+      );
+
+      try {
+        const pending = tracer.trace(
+          request,
+          interruption === "caller cancellation"
+            ? controller.signal
+            : undefined,
+        );
+        await stopEntered.promise;
+        expect(observedExit).toEqual({ code: 17, signal: null });
+        if (interruption === "caller cancellation")
+          controller.abort(new Error("abort during final cleanup"));
+        stopReleased.resolve(undefined);
+        const result = await pending;
+
+        expect(observedStop?.status).toMatch(
+          /^(already-exited|verified-cleanup|terminated|killed)$/u,
+        );
+        expect(result.ok).toBe(false);
+        if (result.ok) throw new Error("Expected the bridge target error");
+        expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+        if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+          return;
+        expect(result.error.reason).toBe(
+          "lldb-target-unloadable: fixture physical bridge failure",
+        );
+        expect(result.error.cleanup).toBeUndefined();
+        expect(result.error.cleanupIncomplete).toBe(false);
+        expect(result.error.cleanupResources).toEqual([]);
+        expect(result.error.partialObservation).toMatchObject({
+          coverage: { status: "partial", reason: "tracer-failure" },
+          process: { pid: null },
+        });
+      } finally {
+        stopReleased.resolve(undefined);
+        stopSpy.mockRestore();
+        deadlineSpy?.mockRestore();
+        await tracer.close();
+        await rm(fixtureRoot, { recursive: true, force: true });
+      }
+    },
+  );
+});
+
+describe("LLDB process exit while output is still open", () => {
+  it.skipIf(process.platform === "win32")(
+    "keeps a nonzero leader exit observed before cancellation while inherited output is still open",
+    async () => {
+      const fixtureRoot = await mkdtemp(
+        join(tmpdir(), "rea-lldb-exit-before-cancel-"),
+      );
+      const fakeLldb = join(fixtureRoot, "fake-lldb");
+      await writeFile(
+        fakeLldb,
+        `#!${process.execPath}\nconst fs = require("node:fs");\nconst { spawn } = require("node:child_process");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nconst child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: ["ignore", "inherit", "inherit"] });\nchild.once("error", () => process.exit(92));\nchild.once("spawn", () => { fs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture leader failure before drain", target_process_created: false })); process.exit(17); });\n`,
+        { mode: 0o700 },
+      );
+
+      const controller = new AbortController();
+      const drainObserved = createDeferred<void>();
+      const drainReleased = createDeferred<void>();
+      const originalWait =
+        ProviderProcessSupervisor.prototype.waitForOutputClose;
+      const drainSpy = vi
+        .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose")
+        .mockImplementation(async function (
+          this: ProviderProcessSupervisor,
+          timeoutMs?: number,
+          signal?: AbortSignal,
+        ): Promise<boolean> {
+          const closed = await originalWait.call(this, timeoutMs, signal);
+          if (!closed && !stopping && this.observedExit()?.code === 17) {
+            drainObserved.resolve(undefined);
+            await drainReleased.promise;
+          }
+          return closed;
+        });
+      let stopping = false;
+      let stopReceipt: ProviderProcessStopResult | undefined;
+      const originalStop = ProviderProcessSupervisor.prototype.stop;
+      const stopSpy = vi
+        .spyOn(ProviderProcessSupervisor.prototype, "stop")
+        .mockImplementation(async function (
+          this: ProviderProcessSupervisor,
+          options?: ProviderProcessStopOptions,
+        ): Promise<ProviderProcessStopResult> {
+          stopping = true;
+          stopReceipt = await originalStop.call(this, options);
+          return stopReceipt;
+        });
+      const tracer = new LldbCallTracer({}, undefined, async () =>
+        ok({ path: fakeLldb, sha256: "d".repeat(64) }),
+      );
+
+      try {
+        const pending = tracer.trace(request, controller.signal);
+        await drainObserved.promise;
+        controller.abort(new Error("abort after leader exit"));
+        drainReleased.resolve(undefined);
+        const result = await pending;
+
+        expect(stopReceipt?.status).toMatch(
+          /^(already-exited|verified-cleanup|terminated|killed)$/u,
+        );
+        expect(result.ok).toBe(false);
+        if (result.ok) throw new Error("Expected the bridge target error");
+        expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+        if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+          return;
+        expect(result.error.reason).toBe(
+          "lldb-target-unloadable: fixture leader failure before drain",
+        );
+        expect(result.error.cleanupIncomplete).toBe(false);
+        expect(result.error.cleanupResources).toEqual([]);
+      } finally {
+        drainReleased.resolve(undefined);
+        drainSpy.mockRestore();
+        stopSpy.mockRestore();
+        await tracer.close();
+        await rm(fixtureRoot, { recursive: true, force: true });
+      }
+    },
+  );
+});
+
+describe("LLDB process exit at output-close loop completion", () => {
+  it.skipIf(process.platform === "win32")(
+    "keeps a nonzero exit when output closure and cancellation meet at loop exit",
+    async () => {
+      const fixtureRoot = await mkdtemp(
+        join(tmpdir(), "rea-lldb-closed-output-cancel-"),
+      );
+      const fakeLldb = join(fixtureRoot, "fake-lldb");
+      await writeFile(
+        fakeLldb,
+        `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture exit before closed-output cancellation", target_process_created: false }));\nprocess.exit(17);\n`,
+        { mode: 0o700 },
+      );
+
+      const controller = new AbortController();
+      let interruptionApplied = false;
+      let stopReceipt: ProviderProcessStopResult | undefined;
+      const originalWait =
+        ProviderProcessSupervisor.prototype.waitForOutputClose;
+      const waitSpy = vi
+        .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose")
+        .mockImplementation(async function (
+          this: ProviderProcessSupervisor,
+          timeoutMs?: number,
+          signal?: AbortSignal,
+        ): Promise<boolean> {
+          const closed = await originalWait.call(this, timeoutMs, signal);
+          if (
+            !interruptionApplied &&
+            closed &&
+            this.observedExit()?.code === 17
+          ) {
+            interruptionApplied = true;
+            controller.abort(new Error("abort at output-close loop exit"));
+          }
+          return closed;
+        });
+      const originalStop = ProviderProcessSupervisor.prototype.stop;
+      const stopSpy = vi
+        .spyOn(ProviderProcessSupervisor.prototype, "stop")
+        .mockImplementation(async function (
+          this: ProviderProcessSupervisor,
+          options?: ProviderProcessStopOptions,
+        ): Promise<ProviderProcessStopResult> {
+          stopReceipt = await originalStop.call(this, options);
+          return stopReceipt;
+        });
+      const tracer = new LldbCallTracer({}, undefined, async () =>
+        ok({ path: fakeLldb, sha256: "f".repeat(64) }),
+      );
+
+      try {
+        const result = await tracer.trace(request, controller.signal);
+
+        expect(interruptionApplied).toBe(true);
+        expect(stopReceipt?.status).toMatch(
+          /^(already-exited|verified-cleanup|terminated|killed)$/u,
+        );
+        expect(result.ok).toBe(false);
+        if (result.ok) throw new Error("Expected the bridge target error");
+        expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+        if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+          return;
+        expect(result.error.reason).toBe(
+          "lldb-target-unloadable: fixture exit before closed-output cancellation",
+        );
+        expect(result.error.cleanupIncomplete).toBe(false);
+        expect(result.error.cleanupResources).toEqual([]);
+      } finally {
+        waitSpy.mockRestore();
+        stopSpy.mockRestore();
+        await tracer.close();
+        await rm(fixtureRoot, { recursive: true, force: true });
+      }
+    },
+  );
+});
+
+describe("LLDB late timeout publication", () => {
+  it("converts a late deadline to timeout at successful-result publication", async () => {
+    let runtime: string | undefined;
+    const tracer = new LldbCallTracer(
+      {},
+      async (_executable, arguments_) => {
+        const { config } = await configFromArguments(arguments_);
+        runtime = dirname(config.result_path);
+        await writeFile(
+          config.observation_path,
+          `${JSON.stringify({ kind: "event", event: EVENT })}\n`,
+        );
+        await writeFile(config.stdout_capture_path, "");
+        await writeFile(config.stderr_capture_path, "");
+        await writeFile(
+          config.result_path,
+          JSON.stringify(successfulTracerOutput),
+        );
+        return {
+          kind: "exited",
+          targetLaunched: false,
+          exitCode: 0,
+          output: "",
+          targetIdentity: undefined,
+          cleanupFailure: undefined,
+          lateTimeout: true,
+        };
+      },
+      async () => ok(fakeTool),
+    );
+
+    const result = await tracer.trace(request);
+    if (runtime === undefined) throw new Error("LLDB fixture did not run");
+    expect(result.ok).toBe(false);
+    if (result.ok) throw new Error("Expected the late deadline to be retained");
+    expect(result.error).toBeInstanceOf(AnalysisTimeoutError);
+    if (!(result.error instanceof AnalysisTimeoutError)) return;
+    expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms);
+    expect(result.error.cleanup).toBeUndefined();
+    expect(result.error.cleanupIncomplete).toBe(false);
+    expect(result.error.partialObservation).toMatchObject({
+      coverage: { status: "partial", reason: "timeout" },
+      process: {
+        outcome: "exited",
+        exit_status: 0,
+        terminated: true,
+      },
+      events: [EVENT],
+    });
+    await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" });
+  });
+});
+
 describe("LLDB pre-target phase proof", () => {
   it("releases a proven pre-target integrity rejection and allows tracer reuse", async () => {
     let launches = 0;