From 5c7755636ddb85c4e1cbd7eb667f914cecca2605 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:19:06 +0800 Subject: [PATCH] fix(model): preserve optional metadata and tensor alignment Fixes #6656 Fixes #6637 Fixes #6636 --- docs/model-artifacts.md | 8 +- src/model/GgufHeader.ts | 6 +- src/model/ModelFormat.ts | 8 ++ src/model/SafetensorsHeader.ts | 79 ++++++++++--- tests/acceptance/modelArtifactCli.test.ts | 128 +++++++++++++++++++++ tests/boundary/model/modelArtifact.test.ts | 114 ++++++++++++++++++ tests/fixtures/model/model.ts | 5 +- 7 files changed, 323 insertions(+), 25 deletions(-) diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md index 63adc66c07..fc0323851c 100644 --- a/docs/model-artifacts.md +++ b/docs/model-artifacts.md @@ -30,6 +30,9 @@ result records the exact selected path, file size, header length, header SHA-256 bytes read and read scope. GGUF alignment padding is outside this prefix. `artifact.sha256` and the Evidence subject are null: a header digest is not a commitment to the weights. Different files can have identical headers and sizes. +The reported GGUF tensor-data offset is aligned after the tensor directory, +including when the directory contains no tensors; `header_length` still ends at +the directory, before that padding. `hash: "full"` additionally streams the remaining file bytes through SHA-256. It returns a full-file digest and Evidence subject, while retaining the same @@ -59,7 +62,10 @@ declared float type; negative zero is `"-0"`. Unknown tensor types retain their lengths; affected totals and storage validation remain unknown. Safetensors reports the header extent, exact string `__metadata__` values, dtype, -shape and declared data ranges. JSON duplicate keys, invalid UTF-8, out-of-file +shape and declared data ranges. A null `__metadata__` is treated as omitted; +unknown fields on tensor entries are ignored within the reader's 32-level JSON +nesting limit, while the required dtype, shape and data offsets are still +validated. JSON duplicate keys, invalid UTF-8, out-of-file ranges, overlaps, gaps, trailing unindexed data and known dtype/shape byte-length contradictions are rejected with a typed `file_format_rejected` constraint (public code `target_unavailable`, category `unsupported_target`). A missing, non-regular or otherwise unusable selected path is also `target_unavailable`; diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index d6c05b81d5..1c5fa8c618 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -384,9 +384,9 @@ export class GgufHeader { progress?.onTensorDeclaration(tensor); } const tensor_data_offset = - tensorCount === 0 - ? this.reader.position - : Math.ceil(this.reader.position / alignment) * alignment; + Math.ceil(this.reader.position / alignment) * alignment; + if (tensor_data_offset > this.reader.size) + rejectModel("GGUF tensor data section begins outside the file"); for (const tensor of tensors) { const offset = BigInt(tensor_data_offset) + BigInt(tensor.data_offset); if ( diff --git a/src/model/ModelFormat.ts b/src/model/ModelFormat.ts index 560415fb18..f6fc0dee60 100644 --- a/src/model/ModelFormat.ts +++ b/src/model/ModelFormat.ts @@ -5,6 +5,7 @@ export const MODEL_CAPACITY_LIMITS = [ "representation-bytes", "exact-offset", "metadata-nesting", + "json-nesting", ] as const; /** Which fixed model allowance refused the request. */ @@ -49,6 +50,7 @@ export const MODEL_HEADER_BYTES = 100_000_000; export const MODEL_REPORT_BYTES = 32 * 1024 * 1024; export const MODEL_ARRAY_PREVIEW = 8; export const MODEL_METADATA_NESTING = 32; +export const MODEL_JSON_NESTING = 32; const fixedModelLimit = (allowance: string): string => `${allowance}. This release has no runtime override. Additional host memory does not raise it. Repeating the unchanged request fails the same way.`; @@ -74,6 +76,10 @@ export const modelCapacityRemediation = (limit: ModelCapacityLimit): string => { return fixedModelLimit( `GGUF metadata nesting is limited to ${String(MODEL_METADATA_NESTING)} array frames`, ); + case "json-nesting": + return fixedModelLimit( + `Safetensors JSON nesting is limited to ${String(MODEL_JSON_NESTING)} levels`, + ); } }; @@ -92,6 +98,8 @@ export const modelCapacityReportedLimits = ( return { maximum_exact_offset: Number.MAX_SAFE_INTEGER }; case "metadata-nesting": return { maximum_array_frames: MODEL_METADATA_NESTING }; + case "json-nesting": + return { maximum_json_depth: MODEL_JSON_NESTING }; } }; diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts index 04adfedd17..d25fc25440 100644 --- a/src/model/SafetensorsHeader.ts +++ b/src/model/SafetensorsHeader.ts @@ -8,6 +8,7 @@ import type { import { ModelHeaderReader } from "./ModelHeaderReader.js"; import { MODEL_HEADER_BYTES, + MODEL_JSON_NESTING, ModelFormatFailure, ModelReportBudget, rejectModel, @@ -15,9 +16,15 @@ import { } from "./ModelFormat.js"; import { safetensorsBits } from "./ModelTensorTypes.js"; +class JsonNumber { + constructor(readonly raw: string) {} +} type HeaderValue = | string | bigint + | boolean + | null + | JsonNumber | HeaderValue[] | { [key: string]: HeaderValue }; @@ -64,10 +71,26 @@ class HeaderJson { if (++this.#nodes % 4096 === 0) yield; this.#space(); this.budget.charge(64); - if (depth > 4) - rejectModel("Safetensors header has an invalid nested structure"); + if (depth > MODEL_JSON_NESTING) + throw new ModelFormatFailure( + "capacity", + `Safetensors JSON nesting exceeds ${String(MODEL_JSON_NESTING)} levels`, + "json-nesting", + ); const c = this.text[this.#position]; if (c === '"') return this.#string(); + if (this.text.startsWith("null", this.#position)) { + this.#position += 4; + return null; + } + if (this.text.startsWith("true", this.#position)) { + this.#position += 4; + return true; + } + if (this.text.startsWith("false", this.#position)) { + this.#position += 5; + return false; + } if (c === "{" || c === "[") { this.#position++; const object: { [key: string]: HeaderValue } = Object.create(null) as { @@ -100,20 +123,37 @@ class HeaderJson { } } const start = this.#position; - while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) { + if (this.text[this.#position] === "-") this.#position++; + const integerStart = this.#position; + if (this.text[this.#position] === "0") this.#position++; + else if (/^[1-9]$/u.test(this.text[this.#position] ?? "")) { this.#position++; - if (this.#position - start > 20) - rejectModel("Safetensors unsigned integer exceeds uint64"); + while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++; } - const raw = this.text.slice(start, this.#position); - if (!/^(?:0|[1-9][0-9]*)$/u.test(raw)) - return rejectModel( - "Safetensors header requires unsigned integer declarations", - ); - const integer = BigInt(raw); - if (integer > 0xffffffffffffffffn) - rejectModel("Safetensors unsigned integer exceeds uint64"); - return integer; + if (this.#position === integerStart) + return rejectModel("Invalid safetensors JSON value"); + if (this.text[this.#position] === ".") { + this.#position++; + const fractionStart = this.#position; + while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++; + if (this.#position === fractionStart) + return rejectModel("Invalid safetensors JSON value"); + } + if (/[eE]/u.test(this.text[this.#position] ?? "")) { + this.#position++; + if (/[+-]/u.test(this.text[this.#position] ?? "")) this.#position++; + const exponentStart = this.#position; + while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++; + if (this.#position === exponentStart) + return rejectModel("Invalid safetensors JSON value"); + } + const numeric = this.text.slice(start, this.#position); + if (/^(?:0|[1-9][0-9]*)$/u.test(numeric)) { + if (numeric.length > 20) return new JsonNumber(numeric); + const integer = BigInt(numeric); + if (integer <= 0xffffffffffffffffn) return integer; + } + return new JsonNumber(numeric); } *read(): Generator { if (this.text[0] !== "{") @@ -129,7 +169,13 @@ class HeaderJson { const objectOf = ( value: HeaderValue | undefined, ): { [key: string]: HeaderValue } => { - if (value === undefined || typeof value !== "object" || Array.isArray(value)) + if ( + value === undefined || + value === null || + typeof value !== "object" || + Array.isArray(value) || + value instanceof JsonNumber + ) return rejectModel("Expected safetensors header object"); return value; }; @@ -211,7 +257,7 @@ export const readSafetensorsHeader = async ( const root = objectOf(step.value); const source = { source_offset: 8, source_length: size }; const metadata: ModelMetadata[] = []; - if (Object.hasOwn(root, "__metadata__")) { + if (Object.hasOwn(root, "__metadata__") && root.__metadata__ !== null) { for (const [key, value] of Object.entries(objectOf(root.__metadata__))) { if (typeof value !== "string") rejectModel("Safetensors __metadata__ values must be strings"); @@ -237,7 +283,6 @@ export const readSafetensorsHeader = async ( for (const name of names) { const entry = objectOf(root[name]); if ( - Object.keys(entry).length !== 3 || !Object.hasOwn(entry, "dtype") || !Object.hasOwn(entry, "shape") || !Object.hasOwn(entry, "data_offsets") diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index f5b5d9d668..feb84649b7 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -549,6 +549,134 @@ cliTest( }, ); +cliTest( + "preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-model-boundary-compat-"); + const { call } = await connectLocalToolsMcp(); + const safetensors = safetensorsFixture( + '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}', + Buffer.alloc(4), + ); + const emptyGgufHeader = ggufFixture({ tensors: [], metadata: [] }); + const alignedOffset = Math.ceil(emptyGgufHeader.header.length / 32) * 32; + const emptyGguf = Buffer.concat([ + emptyGgufHeader.header, + Buffer.alloc(alignedOffset - emptyGgufHeader.header.length), + ]); + + for (const [name, bytes] of [ + ["optional.safetensors", safetensors.bytes], + ["empty.gguf", emptyGguf], + ] as const) { + const path = join(root, name); + await writeFile(path, bytes); + const result = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse(result.json) + .normalized_result, + ); + if (name.endsWith("safetensors")) { + expect(report.metadata).toEqual([]); + expect(report.tensors[0]?.name).toBe("weight"); + expect(report.artifact.header_sha256).toBe( + createHash("sha256").update(safetensors.header).digest("hex"), + ); + } else { + expect(report.tensors).toEqual([]); + expect(report.tensor_data_offset).toBe(alignedOffset); + expect(report.artifact.header_length).toBe( + emptyGgufHeader.header.length, + ); + expect(report.artifact.header_sha256).toBe( + createHash("sha256").update(emptyGgufHeader.header).digest("hex"), + ); + } + const response = await call("inspect_model_artifact", { + path, + detail: "complete", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toEqual(report); + } + + const mapPath = join(root, "same-key-map.safetensors"); + await writeFile( + mapPath, + safetensorsFixture( + '{"__metadata__":{"json_number":"1.25"}}', + Buffer.alloc(0), + ).bytes, + ); + const mapCli = await cli.run({ + arguments: [ + "inspect-model-artifact", + mapPath, + "--detail", + "complete", + "--json", + ], + }); + expect(mapCli.exitCode, mapCli.stderr).toBe(0); + const mapReport = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json) + .normalized_result, + ); + expect( + mapReport.metadata.map(({ key, value }) => ({ key, value })), + ).toEqual([{ key: "json_number", value: "1.25" }]); + const mapMcp = await call("inspect_model_artifact", { + path: mapPath, + detail: "complete", + }); + expect(mapMcp.isError, JSON.stringify(mapMcp)).not.toBe(true); + expect(parseEvidence(mapMcp.structuredContent).normalized_result).toEqual( + mapReport, + ); + + for (const [name, numeric] of [ + ["decimal", "1.25"], + ["negative", "-1"], + ["overflow", "18446744073709551616"], + ] as const) { + const malformedPath = join(root, `${name}-metadata.safetensors`); + await writeFile( + malformedPath, + safetensorsFixture(`{"__metadata__":${numeric}}`, Buffer.alloc(0)) + .bytes, + ); + const malformedCli = await cli.run({ + arguments: ["inspect-model-artifact", malformedPath, "--json"], + }); + expect(malformedCli.exitCode).toBe(1); + expect(malformedCli.json).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + details: { constraint: "file_format_rejected" }, + }); + const malformedMcp = parseMcpToolError( + await call("inspect_model_artifact", { path: malformedPath }), + ); + expect(malformedMcp.error).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + details: { constraint: "file_format_rejected" }, + }); + } + }, +); + cliTest( "exports complete model Evidence for portable paging and returns typed malformed errors", async ({ cli }) => { diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index 35cb78959b..7c9a29ec98 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -5,6 +5,7 @@ import { expect, it, onTestFinished } from "vitest"; import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js"; import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js"; import { + MODEL_JSON_NESTING, MODEL_METADATA_NESTING, MODEL_REPORT_BYTES, ModelFormatFailure, @@ -228,6 +229,28 @@ it.each([8, 24, 40])( }, ); +it("aligns the GGUF data section after an empty tensor directory", async () => { + const base = ggufFixture({ tensors: [], metadata: [] }); + const dataOffset = Math.ceil(base.header.length / 32) * 32; + const bytes = Buffer.concat([ + base.header, + Buffer.alloc(dataOffset - base.header.length), + ]); + const { path, provider } = await providerAt(bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (!result.ok) throw result.error; + expect(result.value.tensors).toEqual([]); + expect(result.value.tensor_data_offset).toBe(dataOffset); + expect(result.value.artifact.header_length).toBe(base.header.length); +}); + +it("rejects an empty GGUF whose required directory padding is truncated", async () => { + const fixture = ggufFixture({ tensors: [], metadata: [] }); + const { path, provider } = await providerAt(fixture.header); + const result = await provider.inspect({ path, hash: "header" }); + expect(result.ok).toBe(false); +}); + it.each([0, 1, 2, 4, 12])( "rejects GGUF alignment %s outside positive multiples of eight", async (alignment) => { @@ -585,12 +608,18 @@ it("chooses model recovery from the producing limit", () => { const nesting = modelCapacityRemediation("metadata-nesting"); expect(nesting).toContain(`${String(MODEL_METADATA_NESTING)} array frames`); expect(nesting).not.toContain(String(MODEL_REPORT_BYTES)); + const jsonNesting = modelCapacityRemediation("json-nesting"); + expect(jsonNesting).toContain(`${String(MODEL_JSON_NESTING)} levels`); + expect(jsonNesting).not.toContain(String(MODEL_REPORT_BYTES)); expect(modelCapacityReportedLimits("report-bytes")).toEqual({ maximum_report_bytes: MODEL_REPORT_BYTES, }); expect(modelCapacityReportedLimits("exact-offset")).toEqual({ maximum_exact_offset: Number.MAX_SAFE_INTEGER, }); + expect(modelCapacityReportedLimits("json-nesting")).toEqual({ + maximum_json_depth: MODEL_JSON_NESTING, + }); }); it("refuses an oversized declared header before reading or expanding it", async () => { @@ -742,6 +771,44 @@ const malformed = [ "safetensors non-string metadata", () => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes, ], + [ + "safetensors decimal metadata container", + () => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes, + ], + [ + "safetensors negative metadata container", + () => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes, + ], + [ + "safetensors overflowing metadata container", + () => + safetensorsFixture( + '{"__metadata__":18446744073709551616}', + Buffer.alloc(0), + ).bytes, + ], + [ + "safetensors noninteger shape sentinel", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "safetensors negative data-offset sentinel", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "safetensors tensor missing a required declaration", + () => + safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0)) + .bytes, + ], [ "safetensors unpaired surrogate", () => @@ -762,6 +829,53 @@ it.each(malformed)( }, ); +it("accepts null safetensors metadata and ignores tensor extension fields", async () => { + const fixture = safetensorsFixture( + '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}', + Buffer.alloc(4), + ); + const { provider, path } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (!result.ok) throw result.error; + expect(result.value.metadata).toEqual([]); + expect( + result.value.tensors.map(({ name, type, shape }) => ({ + name, + type, + shape, + })), + ).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]); +}); + +it("accepts nested JSON values in ignored Safetensors extension fields", async () => { + const nested = `${"[".repeat(12)}0${"]".repeat(12)}`; + const fixture = safetensorsFixture( + `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`, + Buffer.alloc(4), + ); + const { provider, path } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + expect(result.ok).toBe(true); +}); + +it("classifies over-depth Safetensors extension JSON as a resource limit", async () => { + const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`; + const fixture = safetensorsFixture( + `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`, + Buffer.alloc(4), + ); + const { provider, path } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Over-depth JSON was accepted"); + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reason: expect.stringContaining("Safetensors JSON nesting"), + reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING }, + }); +}); + it.each([640, 896, 1152, 1408, 66_176])( "admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker", async (length) => { diff --git a/tests/fixtures/model/model.ts b/tests/fixtures/model/model.ts index c73d25c2e5..012bb7865a 100644 --- a/tests/fixtures/model/model.ts +++ b/tests/fixtures/model/model.ts @@ -77,10 +77,7 @@ export const ggufFixture = ( ]), ), ]); - const dataOffset = - tensors.length === 0 - ? header.length - : Math.ceil(header.length / alignment) * alignment; + const dataOffset = Math.ceil(header.length / alignment) * alignment; const length = Math.max( dataOffset, ...tensors.map((t) => dataOffset + Number(t.offset) + t.bytes),