fix(python): align pyc and PYZ marshal decoding

Fixes #6560
Fixes #6559

docs/python-packaged-applications.md

@@ -96,8 +96,13 @@ returns a resource failure.
 
 Supported final Python bytecode magic values cover Python 3.7 through 3.14.
 Supported marshal directory values are lists, tuples, strings, integers and
-bytes. An unsupported magic or directory stays an opaque member with a
-limitation and `pyz_status: "opaque"`; its module contents remain unknown.
+bytes. Directory values follow the identified CPython marshal semantics; for
+example, a long integer with a zero most-significant digit is malformed and is
+not normalized into a valid table value. An unsupported magic or malformed or
+unsupported directory stays an opaque member with a limitation and
+`pyz_status: "opaque"`; no child entries from that table are projected, and
+their module contents remain unknown. The original PYZ member and its digest
+remain available independently. Healthy CArchive siblings are still inventoried.
 Any opaque PYZ makes producer traversal coverage unknown, including retained
 summaries and exhausted queries. Per-archive reasons remain on the opaque
 occurrences; the coverage observation summarizes their count without repeating

docs/python-pyc.md

@@ -1,10 +1,10 @@
 # CPython pyc inspection
 
-`inspect_python_pyc` / `inspect-python-pyc` answers which CPython version wrote
-one selected local `.pyc` file, how its 16-byte header records source identity,
-and which nested code objects marshal stored. It reads the file without running
-bytecode and requires no Python installation, active binary target, Hopper,
-Ghidra, or IDA.
+`inspect_python_pyc` / `inspect-python-pyc` reports the supported CPython format
+and version declared by one selected local `.pyc` file's magic, how its 16-byte
+header records source identity, and which nested code objects its decoded
+marshal graph contains. It reads the file without running bytecode and requires
+no Python installation, active binary target, Hopper, Ghidra, or IDA.
 
 ```bash
 rea inspect-python-pyc /absolute/path/to/module.pyc --json
@@ -33,10 +33,12 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length.
 - `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size`
   for timestamp pycs.
 
-`code_objects` lists each distinct marshal code object once, the top-level
-module code first. `nested_code` holds the indexes of the distinct code objects
-reachable from that object's constants, including code nested in tuples and
-other containers and the owner itself when a constant container reaches it.
+`code_objects` lists each code object reachable in the decoded marshal graph
+once, the top-level module code first. This follows CPython container behavior;
+it is not an inventory of raw serialized code-object records. `nested_code`
+holds the indexes of the distinct code objects reachable from that object's
+constants, including code nested in tuples and other containers and the owner
+itself when a constant container reaches it.
 A function shared through a marshal reference is named by index once per owner,
 even when that owner reaches it again through the same container. Relationships
 are collected after the marshal graph is complete, so a container that a nested
@@ -46,6 +48,13 @@ filename, the signed 32-bit `co_firstlineno` value, argument counts, flags,
 names, locals (or `localsplusnames` on 3.11+), const
 summaries, and bytecode length plus SHA-256.
 
+Code-object equality used by sets and dictionaries follows the selected layout:
+3.7–3.10 ignore line tables, while 3.11+ include line and exception tables. The
+artifact SHA-256 identifies the exact selected `.pyc` bytes independently of
+this decoded object graph. Reference slots follow each tag's CPython behavior:
+singleton values and `TYPE_REF` do not reserve new slots even when their tag
+has `FLAG_REF` set.
+
 Bytecode is not decompiled. Const strings may be truncated in the summary while
 the marshal walk still consumed the original bytes; a string summary's `bytes`
 is that marshal length. Integer consts report `bits`; their decimal `value` is

src/artifacts/pyinstaller/MarshalSubset.test.ts

@@ -34,6 +34,13 @@ it("resolves a shared string through a reference reserved before the tuple", ()
   ]);
 });
 
+it("does not allocate a new reference slot for a flagged TYPE_REF", () => {
+  const parsed = readMarshalSubset(
+    bytes("2904fa0161f200000000fa01627201000000"),
+  );
+  expect(parsed).toEqual({ ok: true, value: ["a", "a", "b", "b"] });
+});
+
 it("reads a long integer that fits in a safe integer", () => {
   const parsed = readMarshalSubset(
     bytes(
@@ -45,6 +52,33 @@ it("reads a long integer that fits in a safe integer", () => {
   expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]);
 });
 
+it.each(["6c0200000003000000", "6cfeffffff03000000"])(
+  "rejects marshal long %s with a zero most-significant digit as malformed",
+  (hex) => {
+    expect(readMarshalSubset(bytes(hex))).toMatchObject({
+      ok: false,
+      error: {
+        reason: "format",
+        message: "PYZ marshal long is not normalized",
+      },
+    });
+  },
+);
+
+it("reads the zero-digit marshal long encoding as zero", () => {
+  expect(readMarshalSubset(bytes("6c00000000"))).toEqual({
+    ok: true,
+    value: 0,
+  });
+});
+
+it("keeps positive over-budget marshal long digit counts as resource refusals", () => {
+  expect(readMarshalSubset(bytes("6c09000000"))).toMatchObject({
+    ok: false,
+    error: { reason: "limit" },
+  });
+});
+
 it("reads a negative 32-bit integer", () => {
   const parsed = readMarshalSubset(
     bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"),

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -67,6 +67,7 @@ const readObject = (
   if (!code.ok) return code;
   const flag = (code.value & FLAG_REF) !== 0;
   const type = code.value & ~FLAG_REF;
+  if (type === 0x72) return readReference(cursor);
   if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor));
   if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor));
   if (type === 0x7a || type === 0x5a)
@@ -78,7 +79,6 @@ const readObject = (
   if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor));
   if (type === 0x29) return readSequence(cursor, flag, readByte(cursor));
   if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor));
-  if (type === 0x72) return readReference(cursor, flag);
   return err(
     pyInstallerFailure(
       "format",
@@ -130,7 +130,6 @@ const readSequence = (
 
 const readReference = (
   cursor: MarshalCursor,
-  flag: boolean,
 ): Result<MarshalValue, PyInstallerFailure> => {
   const index = readInt32(cursor);
   if (!index.ok) return index;
@@ -142,7 +141,6 @@ const readReference = (
         "PYZ marshal reference does not point at a completed value",
       ),
     );
-  if (flag) cursor.refs.push(value);
   return ok(value);
 };
 
@@ -219,6 +217,7 @@ const readLong = (
       pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),
     );
   let magnitude = 0n;
+  let mostSignificantDigit = 0;
   for (let index = 0; index < count; index += 1) {
     const digit = readUint16(cursor);
     if (!digit.ok) return digit;
@@ -226,8 +225,13 @@ const readLong = (
       return err(
         pyInstallerFailure("format", "PYZ marshal long digit is out of range"),
       );
+    mostSignificantDigit = digit.value;
     magnitude += BigInt(digit.value) << BigInt(15 * index);
   }
+  if (count > 0 && mostSignificantDigit === 0)
+    return err(
+      pyInstallerFailure("format", "PYZ marshal long is not normalized"),
+    );
   if (magnitude > BigInt(Number.MAX_SAFE_INTEGER))
     return err(
       pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"),

src/python/PythonPycParser.ts

@@ -45,6 +45,14 @@ const TYPE_ASCII_INTERNED = 0x41;
 const TYPE_SMALL_TUPLE = 0x29;
 const TYPE_SHORT_ASCII = 0x7a;
 const TYPE_SHORT_ASCII_INTERNED = 0x5a;
+const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([
+  TYPE_NULL,
+  TYPE_NONE,
+  TYPE_FALSE,
+  TYPE_TRUE,
+  TYPE_STOPITER,
+  TYPE_ELLIPSIS,
+]);
 /** CPython `CO_VARARGS` / `CO_VARKEYWORDS`. Each consumes one local name. */
 const CO_VARARGS = 0x04;
 const CO_VARKEYWORDS = 0x08;
@@ -309,11 +317,13 @@ class MarshalReader {
     const raw = this.u8();
     const flagged = (raw & FLAG_REF) !== 0;
     const type = raw & ~FLAG_REF;
-    if (type === TYPE_REF) {
-      if (flagged) malformed("TYPE_REF must not also set FLAG_REF.");
-      return this.readRef();
-    }
-    const slot = flagged ? this.#intern.length : undefined;
+    // CPython masks FLAG_REF before dispatch; TYPE_REF returns its existing
+    // slot directly, and singleton tags bypass reference registration.
+    if (type === TYPE_REF) return this.readRef();
+    const slot =
+      flagged && !TYPES_WITHOUT_REFERENCE_SLOTS.has(type)
+        ? this.#intern.length
+        : undefined;
     if (slot !== undefined) this.#intern.push(undefined);
     const value = this.readTyped(type, depth, slot);
     if (slot !== undefined && this.#intern[slot] === undefined)

tests/acceptance/pyinstallerArchive.test.ts

@@ -88,6 +88,38 @@ const onefile = async (
   return path;
 };
 
+const int32 = (value: number): Buffer => {
+  const bytes = Buffer.alloc(4);
+  bytes.writeInt32LE(value);
+  return bytes;
+};
+
+const marshalInt = (value: number): Buffer =>
+  Buffer.concat([Buffer.from([0x69]), int32(value)]);
+
+const marshalLong = (digits: readonly number[]): Buffer =>
+  Buffer.concat([
+    Buffer.from([0x6c]),
+    int32(digits.length),
+    ...digits.map((digit) => {
+      const bytes = Buffer.alloc(2);
+      bytes.writeUInt16LE(digit);
+      return bytes;
+    }),
+  ]);
+
+const namespaceToc = (typeCode: Buffer): Buffer =>
+  Buffer.concat([
+    Buffer.from([0x5b]),
+    int32(1),
+    Buffer.from([0x29, 2, 0x7a, 3]),
+    Buffer.from("pkg"),
+    Buffer.from([0x29, 3]),
+    typeCode,
+    marshalInt(12),
+    marshalInt(0),
+  ]);
+
 const namespacePathCollision = async (): Promise<string> => {
   const directory = await createTestTempDirectory("rea-pyinstaller-namespace-");
   const path = join(directory, "collision");
@@ -345,6 +377,67 @@ it("reports unknown traversal through the CLI for an opaque PYZ", async () => {
   });
 });
 
+it.each([
+  ["integer", marshalInt(3), true],
+  ["normalized long", marshalLong([3]), true],
+  ["unnormalized long", marshalLong([3, 0]), false],
+])(
+  "keeps PYZ namespace projection consistent with CPython for %s type codes",
+  async (_encoding, typeCode, decoded) => {
+    const path = await onefile(
+      PYTHON_39_MAGIC,
+      false,
+      false,
+      namespaceToc(typeCode),
+    );
+    const result = await execute(process.execPath, [
+      cli,
+      "inspect-artifact",
+      path,
+      "--json",
+    ]);
+    const inspection = artifactInspectionResultSchema.parse(
+      parseEvidence(JSON.parse(result.stdout)).normalized_result,
+    );
+    const inventory = inspection.substeps[0]?.evidence.normalized_result;
+    const pyz = inventory?.occurrences.find(
+      (row) => row.logical_path === "PYZ.pyz",
+    );
+    const healthySibling = inventory?.occurrences.find(
+      (row) => row.logical_path === "healthy.txt",
+    );
+    expect(healthySibling).toMatchObject({ hash_status: "verified" });
+    expect(
+      inventory?.nodes.find(
+        (node) => node.artifact_id === healthySibling?.artifact_id,
+      ),
+    ).toMatchObject({
+      sha256: createHash("sha256")
+        .update(Buffer.from("healthy sibling"))
+        .digest("hex"),
+      size: Buffer.byteLength("healthy sibling"),
+    });
+    if (decoded) {
+      expect(inventory?.coverage.status).toBe("complete");
+      expect(pyz?.format_metadata).toMatchObject({ pyz_status: "decoded" });
+      expect(
+        inventory?.occurrences.some(
+          (row) => row.logical_path === "PYZ.pyz/pkg",
+        ),
+      ).toBe(true);
+    } else {
+      expect(inventory?.coverage.status).toBe("unknown");
+      expect(pyz?.format_metadata).toMatchObject({ pyz_status: "opaque" });
+      expect(
+        inventory?.occurrences.some(
+          (row) => row.logical_path === "PYZ.pyz/pkg",
+        ),
+      ).toBe(false);
+      expect(pyz?.limitations.join(" ")).toContain("long is not normalized");
+    }
+  },
+);
+
 it.each([
   ["negative string length", "5b0100000073ffffffff"],
   ["negative list count", "5bffffffff"],

tests/acceptance/pythonPycCli.test.ts

@@ -61,6 +61,56 @@ cliTest(
   },
 );
 
+cliTest(
+  "matches CPython reference slots for singleton and reference tags",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-python-pyc-ref-slots-");
+    const cases = [
+      {
+        name: "flagged-none",
+        consts: "2903cefa01787200000000",
+        expected: [
+          { kind: "none" },
+          { kind: "str", value: "x" },
+          { kind: "str", value: "x" },
+        ],
+      },
+      {
+        name: "flagged-ref",
+        consts: "2904fa0161f200000000fa01627201000000",
+        expected: [
+          { kind: "str", value: "a" },
+          { kind: "str", value: "a" },
+          { kind: "str", value: "b" },
+          { kind: "str", value: "b" },
+        ],
+      },
+    ];
+    for (const testCase of cases) {
+      const path = join(root, `${testCase.name}.pyc`);
+      await writeFile(
+        path,
+        Buffer.concat([
+          pythonPycHeader(3531),
+          marshalCode("py311", {
+            constsObject: Buffer.from(testCase.consts, "hex"),
+          }),
+        ]),
+      );
+      const result = await cli.run({
+        arguments: ["inspect-python-pyc", path, "--json"],
+      });
+      expect(result.exitCode, result.stderr).toBe(0);
+      const evidence = toolContract("inspect_python_pyc").outputSchema.parse(
+        result.json,
+      );
+      expect(evidence.normalized_result.code_objects[0]?.consts).toMatchObject(
+        testCase.expected,
+      );
+    }
+  },
+);
+
 cliTest(
   "counts code keys using the selected CPython equality rules",
   async ({ cli }) => {