fix(model): preserve optional metadata and tensor alignment

Fixes #6656
Fixes #6637
Fixes #6636

docs/model-artifacts.md

@@ -30,6 +30,9 @@ result records the exact selected path, file size, header length, header SHA-256
 bytes read and read scope. GGUF alignment padding is outside this prefix.
 `artifact.sha256` and the Evidence subject are null: a header digest is not a
 commitment to the weights. Different files can have identical headers and sizes.
+The reported GGUF tensor-data offset is aligned after the tensor directory,
+including when the directory contains no tensors; `header_length` still ends at
+the directory, before that padding.
 
 `hash: "full"` additionally streams the remaining file bytes through SHA-256.
 It returns a full-file digest and Evidence subject, while retaining the same
@@ -59,7 +62,10 @@ declared float type; negative zero is `"-0"`. Unknown tensor types retain their
 lengths; affected totals and storage validation remain unknown.
 
 Safetensors reports the header extent, exact string `__metadata__` values, dtype,
-shape and declared data ranges. JSON duplicate keys, invalid UTF-8, out-of-file
+shape and declared data ranges. A null `__metadata__` is treated as omitted;
+unknown fields on tensor entries are ignored within the reader's 32-level JSON
+nesting limit, while the required dtype, shape and data offsets are still
+validated. JSON duplicate keys, invalid UTF-8, out-of-file
 ranges, overlaps, gaps, trailing unindexed data and known dtype/shape byte-length
 contradictions are rejected with a typed `file_format_rejected` constraint (public code `target_unavailable`, category `unsupported_target`).
 A missing, non-regular or otherwise unusable selected path is also `target_unavailable`;

src/model/GgufHeader.ts

@@ -384,9 +384,9 @@ export class GgufHeader {
       progress?.onTensorDeclaration(tensor);
     }
     const tensor_data_offset =
-      tensorCount === 0
-        ? this.reader.position
-        : Math.ceil(this.reader.position / alignment) * alignment;
+      Math.ceil(this.reader.position / alignment) * alignment;
+    if (tensor_data_offset > this.reader.size)
+      rejectModel("GGUF tensor data section begins outside the file");
     for (const tensor of tensors) {
       const offset = BigInt(tensor_data_offset) + BigInt(tensor.data_offset);
       if (

src/model/ModelFormat.ts

@@ -5,6 +5,7 @@ export const MODEL_CAPACITY_LIMITS = [
   "representation-bytes",
   "exact-offset",
   "metadata-nesting",
+  "json-nesting",
 ] as const;
 
 /** Which fixed model allowance refused the request. */
@@ -49,6 +50,7 @@ export const MODEL_HEADER_BYTES = 100_000_000;
 export const MODEL_REPORT_BYTES = 32 * 1024 * 1024;
 export const MODEL_ARRAY_PREVIEW = 8;
 export const MODEL_METADATA_NESTING = 32;
+export const MODEL_JSON_NESTING = 32;
 
 const fixedModelLimit = (allowance: string): string =>
   `${allowance}. This release has no runtime override. Additional host memory does not raise it. Repeating the unchanged request fails the same way.`;
@@ -74,6 +76,10 @@ export const modelCapacityRemediation = (limit: ModelCapacityLimit): string => {
       return fixedModelLimit(
         `GGUF metadata nesting is limited to ${String(MODEL_METADATA_NESTING)} array frames`,
       );
+    case "json-nesting":
+      return fixedModelLimit(
+        `Safetensors JSON nesting is limited to ${String(MODEL_JSON_NESTING)} levels`,
+      );
   }
 };
 
@@ -92,6 +98,8 @@ export const modelCapacityReportedLimits = (
       return { maximum_exact_offset: Number.MAX_SAFE_INTEGER };
     case "metadata-nesting":
       return { maximum_array_frames: MODEL_METADATA_NESTING };
+    case "json-nesting":
+      return { maximum_json_depth: MODEL_JSON_NESTING };
   }
 };
 

src/model/SafetensorsHeader.ts

@@ -8,6 +8,7 @@ import type {
 import { ModelHeaderReader } from "./ModelHeaderReader.js";
 import {
   MODEL_HEADER_BYTES,
+  MODEL_JSON_NESTING,
   ModelFormatFailure,
   ModelReportBudget,
   rejectModel,
@@ -15,9 +16,15 @@ import {
 } from "./ModelFormat.js";
 import { safetensorsBits } from "./ModelTensorTypes.js";
 
+class JsonNumber {
+  constructor(readonly raw: string) {}
+}
 type HeaderValue =
   | string
   | bigint
+  | boolean
+  | null
+  | JsonNumber
   | HeaderValue[]
   | { [key: string]: HeaderValue };
 
@@ -64,10 +71,26 @@ class HeaderJson {
     if (++this.#nodes % 4096 === 0) yield;
     this.#space();
     this.budget.charge(64);
-    if (depth > 4)
-      rejectModel("Safetensors header has an invalid nested structure");
+    if (depth > MODEL_JSON_NESTING)
+      throw new ModelFormatFailure(
+        "capacity",
+        `Safetensors JSON nesting exceeds ${String(MODEL_JSON_NESTING)} levels`,
+        "json-nesting",
+      );
     const c = this.text[this.#position];
     if (c === '"') return this.#string();
+    if (this.text.startsWith("null", this.#position)) {
+      this.#position += 4;
+      return null;
+    }
+    if (this.text.startsWith("true", this.#position)) {
+      this.#position += 4;
+      return true;
+    }
+    if (this.text.startsWith("false", this.#position)) {
+      this.#position += 5;
+      return false;
+    }
     if (c === "{" || c === "[") {
       this.#position++;
       const object: { [key: string]: HeaderValue } = Object.create(null) as {
@@ -100,20 +123,37 @@ class HeaderJson {
       }
     }
     const start = this.#position;
-    while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) {
+    if (this.text[this.#position] === "-") this.#position++;
+    const integerStart = this.#position;
+    if (this.text[this.#position] === "0") this.#position++;
+    else if (/^[1-9]$/u.test(this.text[this.#position] ?? "")) {
       this.#position++;
-      if (this.#position - start > 20)
-        rejectModel("Safetensors unsigned integer exceeds uint64");
+      while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
     }
-    const raw = this.text.slice(start, this.#position);
-    if (!/^(?:0|[1-9][0-9]*)$/u.test(raw))
-      return rejectModel(
-        "Safetensors header requires unsigned integer declarations",
-      );
-    const integer = BigInt(raw);
-    if (integer > 0xffffffffffffffffn)
-      rejectModel("Safetensors unsigned integer exceeds uint64");
-    return integer;
+    if (this.#position === integerStart)
+      return rejectModel("Invalid safetensors JSON value");
+    if (this.text[this.#position] === ".") {
+      this.#position++;
+      const fractionStart = this.#position;
+      while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
+      if (this.#position === fractionStart)
+        return rejectModel("Invalid safetensors JSON value");
+    }
+    if (/[eE]/u.test(this.text[this.#position] ?? "")) {
+      this.#position++;
+      if (/[+-]/u.test(this.text[this.#position] ?? "")) this.#position++;
+      const exponentStart = this.#position;
+      while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
+      if (this.#position === exponentStart)
+        return rejectModel("Invalid safetensors JSON value");
+    }
+    const numeric = this.text.slice(start, this.#position);
+    if (/^(?:0|[1-9][0-9]*)$/u.test(numeric)) {
+      if (numeric.length > 20) return new JsonNumber(numeric);
+      const integer = BigInt(numeric);
+      if (integer <= 0xffffffffffffffffn) return integer;
+    }
+    return new JsonNumber(numeric);
   }
   *read(): Generator<void, HeaderValue> {
     if (this.text[0] !== "{")
@@ -129,7 +169,13 @@ class HeaderJson {
 const objectOf = (
   value: HeaderValue | undefined,
 ): { [key: string]: HeaderValue } => {
-  if (value === undefined || typeof value !== "object" || Array.isArray(value))
+  if (
+    value === undefined ||
+    value === null ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    value instanceof JsonNumber
+  )
     return rejectModel("Expected safetensors header object");
   return value;
 };
@@ -211,7 +257,7 @@ export const readSafetensorsHeader = async (
   const root = objectOf(step.value);
   const source = { source_offset: 8, source_length: size };
   const metadata: ModelMetadata[] = [];
-  if (Object.hasOwn(root, "__metadata__")) {
+  if (Object.hasOwn(root, "__metadata__") && root.__metadata__ !== null) {
     for (const [key, value] of Object.entries(objectOf(root.__metadata__))) {
       if (typeof value !== "string")
         rejectModel("Safetensors __metadata__ values must be strings");
@@ -237,7 +283,6 @@ export const readSafetensorsHeader = async (
   for (const name of names) {
     const entry = objectOf(root[name]);
     if (
-      Object.keys(entry).length !== 3 ||
       !Object.hasOwn(entry, "dtype") ||
       !Object.hasOwn(entry, "shape") ||
       !Object.hasOwn(entry, "data_offsets")

tests/acceptance/modelArtifactCli.test.ts

@@ -549,6 +549,134 @@ cliTest(
   },
 );
 
+cliTest(
+  "preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-model-boundary-compat-");
+    const { call } = await connectLocalToolsMcp();
+    const safetensors = safetensorsFixture(
+      '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
+      Buffer.alloc(4),
+    );
+    const emptyGgufHeader = ggufFixture({ tensors: [], metadata: [] });
+    const alignedOffset = Math.ceil(emptyGgufHeader.header.length / 32) * 32;
+    const emptyGguf = Buffer.concat([
+      emptyGgufHeader.header,
+      Buffer.alloc(alignedOffset - emptyGgufHeader.header.length),
+    ]);
+
+    for (const [name, bytes] of [
+      ["optional.safetensors", safetensors.bytes],
+      ["empty.gguf", emptyGguf],
+    ] as const) {
+      const path = join(root, name);
+      await writeFile(path, bytes);
+      const result = await cli.run({
+        arguments: [
+          "inspect-model-artifact",
+          path,
+          "--detail",
+          "complete",
+          "--json",
+        ],
+      });
+      expect(result.exitCode, result.stderr).toBe(0);
+      const report = modelArtifactSchema.parse(
+        toolContract("inspect_model_artifact").outputSchema.parse(result.json)
+          .normalized_result,
+      );
+      if (name.endsWith("safetensors")) {
+        expect(report.metadata).toEqual([]);
+        expect(report.tensors[0]?.name).toBe("weight");
+        expect(report.artifact.header_sha256).toBe(
+          createHash("sha256").update(safetensors.header).digest("hex"),
+        );
+      } else {
+        expect(report.tensors).toEqual([]);
+        expect(report.tensor_data_offset).toBe(alignedOffset);
+        expect(report.artifact.header_length).toBe(
+          emptyGgufHeader.header.length,
+        );
+        expect(report.artifact.header_sha256).toBe(
+          createHash("sha256").update(emptyGgufHeader.header).digest("hex"),
+        );
+      }
+      const response = await call("inspect_model_artifact", {
+        path,
+        detail: "complete",
+      });
+      expect(response.isError, JSON.stringify(response)).not.toBe(true);
+      expect(
+        parseEvidence(response.structuredContent).normalized_result,
+      ).toEqual(report);
+    }
+
+    const mapPath = join(root, "same-key-map.safetensors");
+    await writeFile(
+      mapPath,
+      safetensorsFixture(
+        '{"__metadata__":{"json_number":"1.25"}}',
+        Buffer.alloc(0),
+      ).bytes,
+    );
+    const mapCli = await cli.run({
+      arguments: [
+        "inspect-model-artifact",
+        mapPath,
+        "--detail",
+        "complete",
+        "--json",
+      ],
+    });
+    expect(mapCli.exitCode, mapCli.stderr).toBe(0);
+    const mapReport = modelArtifactSchema.parse(
+      toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json)
+        .normalized_result,
+    );
+    expect(
+      mapReport.metadata.map(({ key, value }) => ({ key, value })),
+    ).toEqual([{ key: "json_number", value: "1.25" }]);
+    const mapMcp = await call("inspect_model_artifact", {
+      path: mapPath,
+      detail: "complete",
+    });
+    expect(mapMcp.isError, JSON.stringify(mapMcp)).not.toBe(true);
+    expect(parseEvidence(mapMcp.structuredContent).normalized_result).toEqual(
+      mapReport,
+    );
+
+    for (const [name, numeric] of [
+      ["decimal", "1.25"],
+      ["negative", "-1"],
+      ["overflow", "18446744073709551616"],
+    ] as const) {
+      const malformedPath = join(root, `${name}-metadata.safetensors`);
+      await writeFile(
+        malformedPath,
+        safetensorsFixture(`{"__metadata__":${numeric}}`, Buffer.alloc(0))
+          .bytes,
+      );
+      const malformedCli = await cli.run({
+        arguments: ["inspect-model-artifact", malformedPath, "--json"],
+      });
+      expect(malformedCli.exitCode).toBe(1);
+      expect(malformedCli.json).toMatchObject({
+        code: "target_unavailable",
+        category: "unsupported_target",
+        details: { constraint: "file_format_rejected" },
+      });
+      const malformedMcp = parseMcpToolError(
+        await call("inspect_model_artifact", { path: malformedPath }),
+      );
+      expect(malformedMcp.error).toMatchObject({
+        code: "target_unavailable",
+        category: "unsupported_target",
+        details: { constraint: "file_format_rejected" },
+      });
+    }
+  },
+);
+
 cliTest(
   "exports complete model Evidence for portable paging and returns typed malformed errors",
   async ({ cli }) => {

tests/boundary/model/modelArtifact.test.ts

@@ -5,6 +5,7 @@ import { expect, it, onTestFinished } from "vitest";
 import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js";
 import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js";
 import {
+  MODEL_JSON_NESTING,
   MODEL_METADATA_NESTING,
   MODEL_REPORT_BYTES,
   ModelFormatFailure,
@@ -228,6 +229,28 @@ it.each([8, 24, 40])(
   },
 );
 
+it("aligns the GGUF data section after an empty tensor directory", async () => {
+  const base = ggufFixture({ tensors: [], metadata: [] });
+  const dataOffset = Math.ceil(base.header.length / 32) * 32;
+  const bytes = Buffer.concat([
+    base.header,
+    Buffer.alloc(dataOffset - base.header.length),
+  ]);
+  const { path, provider } = await providerAt(bytes);
+  const result = await provider.inspect({ path, hash: "header" });
+  if (!result.ok) throw result.error;
+  expect(result.value.tensors).toEqual([]);
+  expect(result.value.tensor_data_offset).toBe(dataOffset);
+  expect(result.value.artifact.header_length).toBe(base.header.length);
+});
+
+it("rejects an empty GGUF whose required directory padding is truncated", async () => {
+  const fixture = ggufFixture({ tensors: [], metadata: [] });
+  const { path, provider } = await providerAt(fixture.header);
+  const result = await provider.inspect({ path, hash: "header" });
+  expect(result.ok).toBe(false);
+});
+
 it.each([0, 1, 2, 4, 12])(
   "rejects GGUF alignment %s outside positive multiples of eight",
   async (alignment) => {
@@ -585,12 +608,18 @@ it("chooses model recovery from the producing limit", () => {
   const nesting = modelCapacityRemediation("metadata-nesting");
   expect(nesting).toContain(`${String(MODEL_METADATA_NESTING)} array frames`);
   expect(nesting).not.toContain(String(MODEL_REPORT_BYTES));
+  const jsonNesting = modelCapacityRemediation("json-nesting");
+  expect(jsonNesting).toContain(`${String(MODEL_JSON_NESTING)} levels`);
+  expect(jsonNesting).not.toContain(String(MODEL_REPORT_BYTES));
   expect(modelCapacityReportedLimits("report-bytes")).toEqual({
     maximum_report_bytes: MODEL_REPORT_BYTES,
   });
   expect(modelCapacityReportedLimits("exact-offset")).toEqual({
     maximum_exact_offset: Number.MAX_SAFE_INTEGER,
   });
+  expect(modelCapacityReportedLimits("json-nesting")).toEqual({
+    maximum_json_depth: MODEL_JSON_NESTING,
+  });
 });
 
 it("refuses an oversized declared header before reading or expanding it", async () => {
@@ -742,6 +771,44 @@ const malformed = [
     "safetensors non-string metadata",
     () => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes,
   ],
+  [
+    "safetensors decimal metadata container",
+    () => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes,
+  ],
+  [
+    "safetensors negative metadata container",
+    () => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes,
+  ],
+  [
+    "safetensors overflowing metadata container",
+    () =>
+      safetensorsFixture(
+        '{"__metadata__":18446744073709551616}',
+        Buffer.alloc(0),
+      ).bytes,
+  ],
+  [
+    "safetensors noninteger shape sentinel",
+    () =>
+      safetensorsFixture(
+        '{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}',
+        Buffer.alloc(0),
+      ).bytes,
+  ],
+  [
+    "safetensors negative data-offset sentinel",
+    () =>
+      safetensorsFixture(
+        '{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}',
+        Buffer.alloc(0),
+      ).bytes,
+  ],
+  [
+    "safetensors tensor missing a required declaration",
+    () =>
+      safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0))
+        .bytes,
+  ],
   [
     "safetensors unpaired surrogate",
     () =>
@@ -762,6 +829,53 @@ it.each(malformed)(
   },
 );
 
+it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
+  const fixture = safetensorsFixture(
+    '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
+    Buffer.alloc(4),
+  );
+  const { provider, path } = await providerAt(fixture.bytes);
+  const result = await provider.inspect({ path, hash: "header" });
+  if (!result.ok) throw result.error;
+  expect(result.value.metadata).toEqual([]);
+  expect(
+    result.value.tensors.map(({ name, type, shape }) => ({
+      name,
+      type,
+      shape,
+    })),
+  ).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]);
+});
+
+it("accepts nested JSON values in ignored Safetensors extension fields", async () => {
+  const nested = `${"[".repeat(12)}0${"]".repeat(12)}`;
+  const fixture = safetensorsFixture(
+    `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
+    Buffer.alloc(4),
+  );
+  const { provider, path } = await providerAt(fixture.bytes);
+  const result = await provider.inspect({ path, hash: "header" });
+  expect(result.ok).toBe(true);
+});
+
+it("classifies over-depth Safetensors extension JSON as a resource limit", async () => {
+  const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`;
+  const fixture = safetensorsFixture(
+    `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
+    Buffer.alloc(4),
+  );
+  const { provider, path } = await providerAt(fixture.bytes);
+  const result = await provider.inspect({ path, hash: "header" });
+  expect(result.ok).toBe(false);
+  if (result.ok) throw new Error("Over-depth JSON was accepted");
+  expect(result.error).toMatchObject({
+    _tag: "AnalysisResourceConstraintError",
+    resource: "memory",
+    reason: expect.stringContaining("Safetensors JSON nesting"),
+    reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING },
+  });
+});
+
 it.each([640, 896, 1152, 1408, 66_176])(
   "admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker",
   async (length) => {

tests/fixtures/model/model.ts

@@ -77,10 +77,7 @@ export const ggufFixture = (
       ]),
     ),
   ]);
-  const dataOffset =
-    tensors.length === 0
-      ? header.length
-      : Math.ceil(header.length / alignment) * alignment;
+  const dataOffset = Math.ceil(header.length / alignment) * alignment;
   const length = Math.max(
     dataOffset,
     ...tensors.map((t) => dataOffset + Number(t.offset) + t.bytes),