fix(python): preserve source boundaries in pyc previews (#6492)

Preserve source code-point prefix endpoints in the shared marshal UTF-8 decoder, including separately encoded surrogate values. Keep native TextDecoder spans, full validation, byte identity and the existing 80-unit budget. Update the PyInstaller consumer without changing its text contract.

Validation: 207 relevant source tests;24 affected compiled CLI/MCP/filesystem CI tests; independent CPython3.11.2 compiler/marshal oracle and real argparse;18 equivalent performance tasks; build/check passed. Codex reviewed7c0f7abc6b without major findings.

CI bypass authorized for demonstrated baseline-only failures:101 source and179 built cases reproduced on actual tested main ba9c5bc68. Cancelled shard remainder is unverified. See PR body and discussion for scope and evidence.

Fixes #6476

src/artifacts/pyinstaller/MarshalSubset.ts

@@ -190,7 +190,7 @@ const readUtf8 = (
     return err(
       pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"),
     );
-  return value;
+  return ok(value.value.text);
 };
 
 const readByteString = (

src/domain/python/marshalUtf8.test.ts

@@ -13,11 +13,55 @@ it.each([
   (hex, value) => {
     expect(decodeMarshalUtf8(Buffer.from(hex, "hex"))).toEqual({
       ok: true,
-      value,
+      value: { text: value, prefixEnd: value.length },
     });
   },
 );
 
+it("keeps source-point boundaries before UTF-16 merges adjacent surrogate encodings", () => {
+  const prefix = "a".repeat(79);
+  const scalar = Buffer.from(prefix + "\u{10000}z");
+  const separate = Buffer.concat([
+    Buffer.from(prefix),
+    Buffer.from("eda080edb080", "hex"),
+    Buffer.from("z"),
+  ]);
+  const a = decodeMarshalUtf8(scalar, 80);
+  const b = decodeMarshalUtf8(separate, 80);
+  expect(a).toEqual({
+    ok: true,
+    value: { text: prefix + "\u{10000}z", prefixEnd: 79 },
+  });
+  expect(b).toEqual({
+    ok: true,
+    value: { text: prefix + "\ud800\udc00z", prefixEnd: 80 },
+  });
+  if (a.ok && b.ok) {
+    expect(a.value.text.slice(0, a.value.prefixEnd)).toBe(prefix);
+    expect(b.value.text.slice(0, b.value.prefixEnd)).toBe(prefix + "\ud800");
+  }
+});
+
+it("validates bytes after a closed preview and preserves whole boundary controls", () => {
+  expect(decodeMarshalUtf8(Buffer.from("\u{1f600}".repeat(40)), 80)).toEqual({
+    ok: true,
+    value: { text: "\u{1f600}".repeat(40), prefixEnd: 80 },
+  });
+  expect(decodeMarshalUtf8(Buffer.from("abc"), 0)).toEqual({
+    ok: true,
+    value: { text: "abc", prefixEnd: 0 },
+  });
+  expect(
+    decodeMarshalUtf8(
+      Buffer.concat([
+        Buffer.from("a".repeat(100)),
+        Buffer.from("eda08080", "hex"),
+      ]),
+      80,
+    ),
+  ).toEqual({ ok: false, error: { kind: "invalid-marshal-utf8" } });
+});
+
 it.each([
   "c2eda080",
   "e282eda080",

src/domain/python/marshalUtf8.ts

@@ -2,12 +2,48 @@ import { err, ok, type Result } from "../result.js";
 
 const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
+export interface DecodedMarshalUtf8 {
+  readonly text: string;
+  /** UTF-16 prefix endpoint between source code points, before JS merges surrogates. */
+  readonly prefixEnd: number;
+}
+
+/** This span contains valid scalars, or one separately encoded surrogate. */
+const spanPrefixEnd = (text: string, limit: number): number => {
+  const end = Math.min(text.length, limit);
+  return end > 0 &&
+    end < text.length &&
+    text.charCodeAt(end - 1) >= 0xd800 &&
+    text.charCodeAt(end - 1) <= 0xdbff &&
+    text.charCodeAt(end) >= 0xdc00 &&
+    text.charCodeAt(end) <= 0xdfff
+    ? end - 1
+    : end;
+};
+
 /** CPython marshal uses UTF-8 surrogatepass; all other malformed bytes fail. */
 export const decodeMarshalUtf8 = (
   bytes: Buffer,
-): Result<string, { readonly kind: "invalid-marshal-utf8" }> => {
+  maximumPrefixUnits = Infinity,
+): Result<DecodedMarshalUtf8, { readonly kind: "invalid-marshal-utf8" }> => {
+  if (
+    maximumPrefixUnits !== Infinity &&
+    (!Number.isSafeInteger(maximumPrefixUnits) || maximumPrefixUnits < 0)
+  )
+    throw new RangeError(
+      "Marshal prefix limit must be a nonnegative safe integer.",
+    );
   const chunks: string[] = [];
   let start = 0;
+  let prefixEnd = 0;
+  let prefixOpen = true;
+  const append = (text: string): void => {
+    if (text.length !== 0) chunks.push(text);
+    if (!prefixOpen) return;
+    const end = spanPrefixEnd(text, maximumPrefixUnits - prefixEnd);
+    prefixEnd += end;
+    prefixOpen = end === text.length && prefixEnd < maximumPrefixUnits;
+  };
   try {
     for (
       let offset = bytes.indexOf(0xed);
@@ -27,16 +63,19 @@ export const decodeMarshalUtf8 = (
         continue;
       // Only ED A0..BF 80..BF encodes a surrogate. The fatal decoder owns
       // every unchanged span, including incomplete scalars on either side.
-      chunks.push(
-        UTF8.decode(bytes.subarray(start, offset)),
+      append(UTF8.decode(bytes.subarray(start, offset)));
+      append(
         String.fromCharCode(0xd000 | ((second & 0x3f) << 6) | (third & 0x3f)),
       );
       start = offset + 3;
       offset += 2;
     }
-    if (start === 0) return ok(UTF8.decode(bytes));
-    chunks.push(UTF8.decode(bytes.subarray(start)));
-    return ok(chunks.join(""));
+    if (start === 0) {
+      const text = UTF8.decode(bytes);
+      return ok({ text, prefixEnd: spanPrefixEnd(text, maximumPrefixUnits) });
+    }
+    append(UTF8.decode(bytes.subarray(start)));
+    return ok({ text: chunks.join(""), prefixEnd });
   } catch (cause: unknown) {
     if (cause instanceof TypeError)
       return err({ kind: "invalid-marshal-utf8" });

src/python/PythonPycParser.test.ts

@@ -1625,3 +1625,21 @@ it("leaves 3.11 alpha magics with other code layouts unsupported", () => {
     parse(pythonPycFixture("py311", { magic: 3495 })).header,
   ).toMatchObject({ python_version: "3.11", code_layout: "py311" });
 });
+
+it.each([false, true])(
+  "preserves a source-encoded lone surrogate at the preview boundary (truncated=%s)",
+  (truncated) => {
+    const text = Buffer.concat([
+      Buffer.from("a".repeat(79)),
+      Buffer.from([0xed, 0xa0, 0x80]),
+      ...(truncated ? [Buffer.from("z")] : []),
+    ]);
+    const report = parse(withConsts([marshalUnicode(text)]));
+    expect(root(report).consts[0]).toEqual({
+      kind: "str",
+      value: "a".repeat(79) + "\ud800",
+      bytes: truncated ? 83 : 82,
+      truncated,
+    });
+  },
+);

src/python/PythonPycParser.ts

@@ -80,6 +80,7 @@ type MarshalValue =
       readonly bytes: number;
       /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */
       readonly raw: Buffer;
+      readonly previewEnd: number;
     }
   | { readonly kind: "tuple"; readonly items: MarshalValue[] }
   | { readonly kind: "list"; readonly items: MarshalValue[] }
@@ -1483,15 +1484,17 @@ class MarshalReader {
       return {
         kind: "str",
         value: bytes.toString("latin1"),
+        previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
         raw: bytes,
       };
     }
-    const decoded = decodeMarshalUtf8(bytes);
+    const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);
     if (!decoded.ok) return malformed("Marshal string is not valid UTF-8.");
     return {
       kind: "str",
-      value: decoded.value,
+      value: decoded.value.text,
+      previewEnd: decoded.value.prefixEnd,
       bytes: length,
       raw: bytes,
     };
@@ -1665,21 +1668,11 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => {
       return { kind: "float", value: jsonFloat(value.value) };
     case "str": {
       const truncated = value.value.length > PYTHON_PYC_CONST_PREVIEW_LIMIT;
-      let end = PYTHON_PYC_CONST_PREVIEW_LIMIT;
-      // Preserve source lone surrogates, but do not split a source scalar.
-      if (
-        truncated &&
-        value.value.charCodeAt(end - 1) >= 0xd800 &&
-        value.value.charCodeAt(end - 1) <= 0xdbff &&
-        value.value.charCodeAt(end) >= 0xdc00 &&
-        value.value.charCodeAt(end) <= 0xdfff
-      )
-        end--;
       return {
         kind: "str",
         bytes: value.bytes,
         truncated,
-        value: truncated ? value.value.slice(0, end) : value.value,
+        value: truncated ? value.value.slice(0, value.previewEnd) : value.value,
       };
     }
     case "bytes":

tests/acceptance/pythonPycCli.test.ts

@@ -717,3 +717,123 @@ cliTest("reports a truncated pyc as invalid input", async ({ cli }) => {
     details: { operation: "inspect_python_pyc" },
   });
 });
+
+cliTest(
+  "preserves whole Unicode scalars in bounded pyc previews through CLI and MCP",
+  async ({ cli }) => {
+    const cases = [
+      {
+        input: "a" + "😀".repeat(40),
+        preview: "a" + "😀".repeat(39),
+        bytes: 161,
+        truncated: true,
+      },
+      {
+        input: "a" + "😀".repeat(40) + "z",
+        preview: "a" + "😀".repeat(39),
+        bytes: 162,
+        truncated: true,
+      },
+      {
+        input: "a" + "😀".repeat(39),
+        preview: "a" + "😀".repeat(39),
+        bytes: 157,
+        truncated: false,
+      },
+      {
+        input: "😀".repeat(40),
+        preview: "😀".repeat(40),
+        bytes: 160,
+        truncated: false,
+      },
+      {
+        input: "z".repeat(81),
+        preview: "z".repeat(80),
+        bytes: 81,
+        truncated: true,
+      },
+    ];
+    const consts = cases.map(({ input }) => {
+      const text = Buffer.from(input, "utf8");
+      const header = Buffer.alloc(5);
+      header[0] = 0x75;
+      header.writeUInt32LE(text.length, 1);
+      return Buffer.concat([header, text]);
+    });
+    const bytes = Buffer.concat([
+      pythonPycHeader(3531),
+      marshalCode("py311", { consts }),
+    ]);
+    const root = await createTestTempDirectory("rea-python-pyc-preview-");
+    const path = join(root, "unicode.pyc");
+    await writeFile(path, bytes);
+    const response = await cli.run({
+      arguments: ["inspect-python-pyc", path, "--json"],
+    });
+    expect(response.exitCode, response.stderr).toBe(0);
+    const contract = toolContract("inspect_python_pyc");
+    const report = contract.outputSchema.parse(response.json).normalized_result;
+    expect(report.coverage.status).toBe("complete");
+    expect(report.code_objects[0]?.consts).toEqual(
+      cases.map((entry) => ({
+        kind: "str",
+        value: entry.preview,
+        bytes: entry.bytes,
+        truncated: entry.truncated,
+      })),
+    );
+    const call = await cli.sharedMcpCall?.();
+    if (call === undefined) throw new Error("Expected file MCP session");
+    const mcp = await call("inspect_python_pyc", { path });
+    expect(mcp.isError).not.toBe(true);
+    expect(
+      contract.outputSchema.parse(mcp.structuredContent).normalized_result,
+    ).toEqual(report);
+    const toon = await cli.run({
+      arguments: ["inspect-python-pyc", path, "--format", "toon"],
+    });
+    expect(toon.exitCode, toon.stderr).toBe(0);
+    expect(toon.stdout).not.toContain("�");
+  },
+);
+
+cliTest(
+  "distinguishes source surrogate code points from astral scalars at the preview cut",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-python-source-boundary-");
+    const path = join(root, "source-boundaries.pyc");
+    const prefix = "a".repeat(79);
+    const scalar = Buffer.from(prefix + "𐀀z");
+    const separate = Buffer.concat([
+      Buffer.from(prefix),
+      Buffer.from([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]),
+      Buffer.from("z"),
+    ]);
+    await writeFile(
+      path,
+      Buffer.concat([
+        pythonPycHeader(3531),
+        marshalCode("py311", {
+          consts: [marshalUtf8(scalar), marshalUtf8(separate)],
+        }),
+      ]),
+    );
+    const response = await cli.run({
+      arguments: ["inspect-python-pyc", path, "--json"],
+    });
+    expect(response.exitCode, response.stderr).toBe(0);
+    const schema = toolContract("inspect_python_pyc").outputSchema;
+    const report = schema.parse(response.json).normalized_result;
+    expect(report.code_objects[0]?.consts).toEqual([
+      { kind: "str", value: prefix, bytes: 84, truncated: true },
+      { kind: "str", value: prefix + "\ud800", bytes: 86, truncated: true },
+    ]);
+    const call = await cli.sharedMcpCall?.();
+    if (call === undefined) throw new Error("Expected file MCP session");
+    const mcp = await call("inspect_python_pyc", { path });
+    expect(mcp.isError).not.toBe(true);
+    expect(schema.parse(mcp.structuredContent).normalized_result).toEqual(
+      report,
+    );
+  },
+);