diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 8239426e5a..c2b958555f 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -3819,7 +3819,7 @@ const failureOutput = (stdout = "", stderr = "") => ({ }), }); const projectFailure = async ( - kind: "cancelled" | "process" | "protocol" | "remote" | "timeout", + kind: "cancelled" | "process" | "protocol" | "remote" | "start" | "timeout", diagnostics: Readonly>, ) => { const ghidra = provider(installationHost(), () => ({ @@ -4161,6 +4161,19 @@ describe("Ghidra SIGKILL projection", () => { }, ); + it("keeps start-failure SIGKILL cause unknown despite a cgroup OOM delta", async () => { + const diagnostics = { ...killed, memory_cgroup: cgroup(1, 3_221_225_472) }; + const projected = await projectFailure("start", diagnostics); + expect(projected.code).toBe("execution_failure"); + expect(projected.message).toContain("SIGKILL"); + expect(projected.message).toContain("does not establish its cause"); + expect(projected.details?.diagnostics).toMatchObject(diagnostics); + expect(projected.remediation.action).toContain( + "host or container termination records", + ); + }); + + it.each([ ["without cgroup evidence", {}], [ @@ -4175,9 +4188,18 @@ describe("Ghidra SIGKILL projection", () => { }); expect(projected.code).toBe("execution_failure"); expect(projected.message).toContain("SIGKILL"); - expect(projected.message).toContain("out of memory"); + expect(projected.message).toContain("does not establish its cause"); expect(projected.message).not.toContain("rea doctor"); - expect(projected.remediation.action).toContain("decompiler"); + expect(projected.remediation.action).toContain( + "host or container termination records", + ); + expect(projected.remediation.action).not.toContain( + "GHIDRA_HEADLESS_MAXMEM", + ); + expect(projected.details?.diagnostics).toMatchObject({ + ...killed, + ...extra, + }); }); it("keeps an observed OOME primary over the kill", async () => { diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index eb5ea3b1bf..f0cdbbf392 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -1025,7 +1025,7 @@ const projectPrimarySessionError = ( ? "Ghidra's output for this operation is incomplete. No OutOfMemoryError was confirmed; a memory cause cannot be ruled out." : undefined; if (memoryFailure !== undefined) return memoryFailure; - const killed = projectExternalSigkill(operation, failure, context, { + const killed = projectSigkill(operation, failure, context, { restart: exitedProviderMessage(operation, failure, health), uncertainty, }); @@ -1049,18 +1049,6 @@ const projectPrimarySessionError = ( }), }); } - if ( - (failure.kind === "start" || - failure.kind === "process" || - failure.kind === "protocol") && - failure.diagnostics.exit_signal === "SIGKILL" - ) - return new ProviderAdapterError("ghidra", operation, { - ...context, - userMessage: `Ghidra process diagnostics report termination by SIGKILL. The signal alone does not establish its cause or an out-of-memory failure.${uncertainty === undefined ? "" : ` ${uncertainty}`}`, - remediationAction: - "Review the retained process output and host or container termination records before choosing a resource-specific remedy. Export useful Evidence, close the binary and resolve any cleanup failure, then call open_binary for the same target to start a new Ghidra session. Session-only annotations are not retained.", - }); const exited = exitedProviderMessage(operation, failure, health); if (exited !== undefined) return new ProviderAdapterError("ghidra", operation, { @@ -1196,13 +1184,13 @@ const GHIDRA_PROCESS_TREE_MEMORY_ACTION = "A memory limit covers Ghidra's native decompiler processes as well as its Java heap, and their number follows the available CPUs, so a larger GHIDRA_HEADLESS_MAXMEM does not help. Raise the container or host memory limit, or allow fewer CPUs with a longer REA_GHIDRA_STARTUP_TIMEOUT_MS."; /** - * A SIGKILL observed before the process supervisor began stopping Ghidra did - * not come from REA. Kernel and container OOM killers send it without any - * OutOfMemoryError text, so a rise in the oom_kill count of REA's cgroup is - * the memory evidence when available. That count covers every process in the - * cgroup and its child groups, not only REA's process tree. + * Own SIGKILL interpretation for startup and running-provider failures. + * The signal alone leaves its cause unknown. For process/protocol failures + * observed before REA requested a stop, a rising cgroup oom_kill count supplies + * separate memory evidence. That count covers the cgroup and its child groups, + * not only REA's process tree. */ -const projectExternalSigkill = ( +const projectSigkill = ( operation: AnalysisOperation, failure: GhidraSessionError, context: { @@ -1215,9 +1203,10 @@ const projectExternalSigkill = ( }, ): AnalysisError | undefined => { if ( - (failure.kind !== "process" && failure.kind !== "protocol") || - failure.diagnostics.exit_signal !== "SIGKILL" || - failure.diagnostics.stop_requested !== false + (failure.kind !== "start" && + failure.kind !== "process" && + failure.kind !== "protocol") || + failure.diagnostics.exit_signal !== "SIGKILL" ) return undefined; const retry = guidance.restart ?? "Then retry."; @@ -1225,6 +1214,8 @@ const projectExternalSigkill = ( failure.diagnostics.memory_cgroup, ); if ( + (failure.kind === "process" || failure.kind === "protocol") && + failure.diagnostics.stop_requested === false && cgroup.success && cgroup.data.oom_kill_after > cgroup.data.oom_kill_before ) { @@ -1246,8 +1237,9 @@ const projectExternalSigkill = ( } return new ProviderAdapterError("ghidra", operation, { ...context, - userMessage: `Ghidra was killed by SIGKILL before REA stopped it. REA could not confirm the sender; a kernel or container out of memory killer is a common one and leaves no Java OutOfMemoryError in the output.${guidance.uncertainty === undefined ? "" : ` ${guidance.uncertainty}`}`, - remediationAction: `Check the host or container for an out-of-memory kill, for example in the kernel log or as OOMKilled in docker inspect. ${GHIDRA_PROCESS_TREE_MEMORY_ACTION} ${retry}`, + userMessage: `Ghidra process diagnostics report termination by SIGKILL. The signal alone does not establish its cause or an out-of-memory failure.${guidance.uncertainty === undefined ? "" : ` ${guidance.uncertainty}`}`, + remediationAction: + "Review the retained process output and host or container termination records before choosing a resource-specific remedy. Export useful Evidence, close the binary and resolve any cleanup failure, then call open_binary for the same target to start a new Ghidra session. Session-only annotations are not retained.", }); };