fix(managed): honor metadata ExtraData before table decoding (#1893)

Co-authored-by: LouisChen <3361008+chqcjq@users.noreply.github.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>

docs/managed-code-analysis.md

@@ -126,6 +126,13 @@ Member inspection binds build-local tokens, signatures, row offsets, method
 bodies, and decoded relationships to that artifact and MVID. Missing or malformed
 facts remain explicit in coverage and per-member status.
 
+Metadata table streams may carry the CLR-compatible `ExtraData` extension
+(`HeapSizes` flag `0x40`), which Microsoft readers honor although it is not
+specified by ECMA-335. REA skips its fixed four-byte word after the row counts
+before decoding table rows; the word's value is not a length. An incomplete
+word is reported as a metadata coverage failure at its file offset, without
+reading beyond the declared table stream.
+
 For unoptimized `#-` metadata, nonempty `FieldPtr` and `MethodPtr` tables
 translate TypeDef list positions to physical member definitions. REA resolves
 these indirections for member ownership and local attribute-constructor types;

src/dotnet/ManagedMetadataExtraData.test.ts

@@ -0,0 +1,208 @@
+import { describe, expect, it } from "vitest";
+
+import { inspectManagedArtifactBytes } from "./ManagedArtifactInspector.js";
+import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js";
+import { readManagedPeLayout } from "./ManagedPeReader.js";
+import {
+  buildManagedPeFixture,
+  managedPeFixtureTarget,
+} from "./ManagedPe.fixture.js";
+
+const fixtureTablesStream = (bytes: Buffer) => {
+  const pe = readManagedPeLayout(bytes);
+  if (pe.cli === null) throw new Error("Fixture omitted CLI metadata");
+  const root = pe.rvaToOffset(
+    pe.cli.metadata.rva,
+    pe.cli.metadata.size,
+    "cli.metadata",
+  );
+  const versionLength = bytes.readUInt32LE(root + 12);
+  const streamHeader = ((root + 16 + versionLength + 3) & ~3) + 4;
+  expect(
+    bytes.toString(
+      "ascii",
+      streamHeader + 8,
+      bytes.indexOf(0, streamHeader + 8),
+    ),
+  ).toBe("#~");
+  const tablesOffset = root + bytes.readUInt32LE(streamHeader);
+  const valid = bytes.readBigUInt64LE(tablesOffset + 8);
+  let count = 0;
+  for (let table = 0; table < 64; table += 1)
+    if ((valid & (1n << BigInt(table))) !== 0n) count += 1;
+  return { streamHeader, tablesOffset, extraDataOffset: 24 + count * 4 };
+};
+
+describe("CLR metadata tables ExtraData extension", () => {
+  it.each([
+    { label: "absent extension", metadataExtraData: undefined },
+    { label: "zero word", metadataExtraData: 0 },
+    { label: "nonzero word", metadataExtraData: 0x1234_5678 },
+    { label: "maximum unsigned word", metadataExtraData: 0xffff_ffff },
+  ])(
+    "preserves metadata and member identities with $label",
+    ({ metadataExtraData }) => {
+      const plain = buildManagedPeFixture();
+      const plainTarget = managedPeFixtureTarget(plain);
+      const baseline = inspectManagedArtifactBytes(plain, plainTarget);
+      const baselineMembers = inspectManagedMembersBytes(plain, plainTarget);
+      const bytes = buildManagedPeFixture(
+        metadataExtraData === undefined ? {} : { metadataExtraData },
+      );
+      const target = managedPeFixtureTarget(bytes);
+      const artifact = inspectManagedArtifactBytes(bytes, target);
+      const members = inspectManagedMembersBytes(bytes, target);
+      const shift = metadataExtraData === undefined ? 0 : 4;
+
+      expect(artifact.coverage).toEqual({ state: "complete", issues: [] });
+      expect(members.coverage).toEqual({ state: "complete", issues: [] });
+      if (baseline.module === null || baseline.assembly === null)
+        throw new Error("Fixture omitted module or assembly");
+      expect(artifact.module).toEqual({
+        ...baseline.module,
+        row_offset: baseline.module.row_offset + shift,
+      });
+      expect(artifact.assembly).toEqual({
+        ...baseline.assembly,
+        row_offset: baseline.assembly.row_offset + shift,
+      });
+      expect(artifact.metadata.table_row_counts).toEqual(
+        baseline.metadata.table_row_counts,
+      );
+      expect(artifact.target_frameworks).toEqual(baseline.target_frameworks);
+      expect(members.types).toEqual(
+        baselineMembers.types.map((type) => ({
+          ...type,
+          row_offset: type.row_offset + shift,
+        })),
+      );
+      expect(members.fields).toEqual(
+        baselineMembers.fields.map((field) => ({
+          ...field,
+          row_offset: field.row_offset + shift,
+        })),
+      );
+      expect(members.methods).toEqual(
+        baselineMembers.methods.map((method) => ({
+          ...method,
+          row_offset: method.row_offset + shift,
+        })),
+      );
+      expect(members.call_edges).toEqual(baselineMembers.call_edges);
+      expect(members.field_accesses).toEqual(baselineMembers.field_accesses);
+    },
+  );
+});
+
+describe("CLR ExtraData index widths and boundaries", () => {
+  it("combines ExtraData with actual four-byte string heap indexes", () => {
+    const fieldName = "f".repeat(65_536);
+    const plain = buildManagedPeFixture({ fieldName });
+    const baseline = inspectManagedMembersBytes(
+      plain,
+      managedPeFixtureTarget(plain),
+    );
+    const bytes = buildManagedPeFixture({
+      fieldName,
+      metadataExtraData: 0x1234_5678,
+    });
+    const { tablesOffset } = fixtureTablesStream(bytes);
+    expect(bytes[tablesOffset + 6]).toBe(0x41);
+    const result = inspectManagedMembersBytes(
+      bytes,
+      managedPeFixtureTarget(bytes),
+    );
+
+    expect(baseline.coverage).toEqual({ state: "complete", issues: [] });
+    expect(result.coverage).toEqual({ state: "complete", issues: [] });
+    expect(result.module?.name).toBe("Fixture.dll");
+    expect(result.module?.mvid).toBe(baseline.module?.mvid);
+    expect(result.types.map((type) => type.full_name)).toEqual([
+      "Fixture.Program",
+    ]);
+    expect(result.fields.map(({ token, name }) => ({ token, name }))).toEqual([
+      { token: "0x04000001", name: fieldName },
+    ]);
+    // Main is placed after the large field name, so its index exceeds 0xffff.
+    expect(result.methods.map(({ token, name }) => ({ token, name }))).toEqual([
+      { token: "0x06000001", name: "Main" },
+    ]);
+  });
+
+  it("honors ExtraData in an unoptimized table stream without pointer tables", () => {
+    const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 });
+    const baselineTarget = managedPeFixtureTarget(bytes);
+    const baseline = inspectManagedArtifactBytes(bytes, baselineTarget);
+    const baselineMembers = inspectManagedMembersBytes(bytes, baselineTarget);
+    const { streamHeader } = fixtureTablesStream(bytes);
+    bytes.write("#-", streamHeader + 8, "ascii");
+    const target = managedPeFixtureTarget(bytes);
+    const result = inspectManagedArtifactBytes(bytes, target);
+    const members = inspectManagedMembersBytes(bytes, target);
+
+    expect(result.coverage).toEqual({ state: "complete", issues: [] });
+    expect(members.coverage).toEqual({ state: "complete", issues: [] });
+    expect(result.metadata.stream_names).toContain("#-");
+    expect(result.metadata.stream_names).not.toContain("#~");
+    expect(result.module).toEqual(baseline.module);
+    expect(result.assembly).toEqual(baseline.assembly);
+    expect(members.types).toEqual(baselineMembers.types);
+    expect(members.fields).toEqual(baselineMembers.fields);
+    expect(members.methods).toEqual(baselineMembers.methods);
+  });
+
+  it("reports a truncated row count before interpreting ExtraData", () => {
+    const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 });
+    const { streamHeader, extraDataOffset } = fixtureTablesStream(bytes);
+    bytes.writeUInt32LE(extraDataOffset - 1, streamHeader + 4);
+    const result = inspectManagedMembersBytes(
+      bytes,
+      managedPeFixtureTarget(bytes),
+    );
+
+    expect(result.coverage).toMatchObject({
+      state: "unavailable",
+      issues: [
+        expect.objectContaining({
+          code: "invalid-metadata-root",
+          scope: "metadata.table-row-count",
+        }),
+      ],
+    });
+    expect(result.types).toEqual([]);
+    expect(result.methods).toEqual([]);
+  });
+
+  it.each([0, 1, 3])(
+    "rejects an ExtraData word with only %i declared bytes",
+    (availableBytes) => {
+      const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 });
+      const { streamHeader, tablesOffset, extraDataOffset } =
+        fixtureTablesStream(bytes);
+      expect(bytes[tablesOffset + 6]).toBe(0x40);
+      expect(bytes.readUInt32LE(tablesOffset + extraDataOffset)).toBe(
+        0x1234_5678,
+      );
+      // Only the declared tables-stream extent is shortened; the PE and heaps stay intact.
+      bytes.writeUInt32LE(extraDataOffset + availableBytes, streamHeader + 4);
+
+      const result = inspectManagedMembersBytes(
+        bytes,
+        managedPeFixtureTarget(bytes),
+      );
+      expect(result.coverage).toMatchObject({
+        state: "unavailable",
+        issues: [
+          expect.objectContaining({
+            code: "invalid-metadata-root",
+            scope: "metadata.tables-extra-data",
+            offset: tablesOffset + extraDataOffset,
+          }),
+        ],
+      });
+      expect(result.module).toBeNull();
+      expect(result.types).toEqual([]);
+      expect(result.methods).toEqual([]);
+    },
+  );
+});

src/dotnet/ManagedMetadataLayout.ts

@@ -358,6 +358,18 @@ const buildMetadataTables = ({
     if ((valid & (1n << BigInt(index))) === 0n) continue;
     tableCursor += 4;
   }
+  // CLR readers honor an optional extra-data word after the row counts.
+  // Its value is opaque; it is not a length or part of the first table row.
+  if (((tableBytes[6] ?? 0) & 0x40) !== 0) {
+    if (tableCursor > tableBytes.length - 4)
+      throw managedFailure(
+        "invalid-metadata-root",
+        "metadata.tables-extra-data",
+        "Metadata tables extra-data word is truncated",
+        tablesOffset + tableCursor,
+      );
+    tableCursor += 4;
+  }
   for (let index = 0; index < METADATA_TABLE_NAMES.length; index += 1) {
     const count = rowCounts[index] ?? 0;
     if ((valid & (1n << BigInt(index))) === 0n) continue;

src/dotnet/ManagedPe.fixture.ts

@@ -20,6 +20,7 @@ interface ManagedPeFixtureOptions {
   readonly ilBody?: Buffer;
   readonly malformedAssemblyReferenceRows?: readonly number[];
   readonly malformedCustomAttributeRows?: readonly number[];
+  readonly metadataExtraData?: number;
   readonly metadataValidMaskExtra?: bigint;
   readonly methodName?: string;
   readonly methodFlags?: number;
@@ -466,12 +467,13 @@ const tablesStream = (
   rows: ReadonlyMap<number, readonly Buffer[]>,
   extraValidMask: bigint,
   heapSizes: number,
+  extraData?: number,
 ): Buffer => {
   let valid = extraValidMask;
   for (const table of rows.keys()) valid |= 1n << BigInt(table);
   const header = Buffer.alloc(24);
   header.writeUInt8(2, 4);
-  header.writeUInt8(heapSizes, 6);
+  header.writeUInt8(heapSizes | (extraData === undefined ? 0 : 0x40), 6);
   header.writeUInt8(1, 7);
   header.writeBigUInt64LE(valid, 8);
   const counts: Buffer[] = [];
@@ -486,7 +488,12 @@ const tablesStream = (
     if (tableRows === undefined) continue;
     data.push(...tableRows);
   }
-  return Buffer.concat([header, ...counts, ...data]);
+  return Buffer.concat([
+    header,
+    ...counts,
+    ...(extraData === undefined ? [] : [u32(extraData)]),
+    ...data,
+  ]);
 };
 
 interface FixtureMetadataRowContext {
@@ -875,7 +882,12 @@ const buildManagedFixtureMetadata = (
     (guidSize === 4 ? 0x02 : 0) |
     (blobSize === 4 ? 0x04 : 0);
   const metadata = metadataRoot(
-    tablesStream(rows, options.metadataValidMaskExtra ?? 0n, heapFlags),
+    tablesStream(
+      rows,
+      options.metadataValidMaskExtra ?? 0n,
+      heapFlags,
+      options.metadataExtraData,
+    ),
     strings.toBuffer(),
     guidHeap(options.mvid),
     blobs.toBuffer(),

tests/acceptance/managedMetadataExtraData.test.ts

@@ -0,0 +1,84 @@
+import { createHash } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { Client } from "@modelcontextprotocol/client";
+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
+import { expect } from "vitest";
+
+import { parseEvidence } from "../../src/domain/evidence.js";
+import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js";
+import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
+import { cliTest } from "../support/cli/cliFixture.js";
+
+cliTest(
+  "CLI and stdio MCP inspect CLR ExtraData metadata without shifting table identities",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-managed-extra-data-");
+    const path = join(root, "extra-data.dll");
+    const bytes = buildManagedPeFixture({ metadataExtraData: 0x1234_5678 });
+    const sha256 = createHash("sha256").update(bytes).digest("hex");
+    await writeFile(path, bytes);
+    const transport = new StdioClientTransport({
+      command: process.execPath,
+      args: [resolve("scripts/rea.mjs"), "mcp"],
+      cwd: process.cwd(),
+      env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
+      stderr: "pipe",
+    });
+    transport.stderr?.on("data", () => undefined);
+    const client = new Client({
+      name: "managed-extra-data-parity",
+      version: "1",
+    });
+    try {
+      await client.connect(transport);
+      for (const name of [
+        "inspect_managed_artifact",
+        "inspect_managed_members",
+      ] as const) {
+        const cliResult = await cli.run({
+          arguments: [name.replaceAll("_", "-"), path, "--json"],
+        });
+        expect(cliResult.exitCode, cliResult.stderr).toBe(0);
+        const cliEvidence = parseEvidence(cliResult.json);
+        const response = await client.callTool({ name, arguments: { path } });
+        expect(response.isError, JSON.stringify(response)).not.toBe(true);
+        const mcpEvidence = parseEvidence(response.structuredContent);
+        expect(mcpEvidence.normalized_result).toEqual(
+          cliEvidence.normalized_result,
+        );
+        expect(mcpEvidence.subject?.digest.sha256).toBe(sha256);
+        expect(cliEvidence.subject?.digest.sha256).toBe(sha256);
+        expect(mcpEvidence.normalized_result).toMatchObject({
+          module: {
+            name: "Fixture.dll",
+            mvid: "00112233-4455-6677-8899-aabbccddeeff",
+          },
+          coverage: { state: "complete", issues: [] },
+        });
+        if (name === "inspect_managed_artifact") {
+          expect(mcpEvidence.normalized_result).toMatchObject({
+            assembly: { name: "Fixture.Managed" },
+          });
+        } else {
+          expect(mcpEvidence.normalized_result).toMatchObject({
+            types: [expect.objectContaining({ full_name: "Fixture.Program" })],
+            methods: [
+              expect.objectContaining({
+                token: "0x06000001",
+                name: "Main",
+                declaring_type: "Fixture.Program",
+              }),
+            ],
+          });
+        }
+      }
+    } finally {
+      try {
+        await client.close();
+      } finally {
+        await transport.close();
+      }
+    }
+  },
+);