fix(browser): preserve unknown storage inventory and fingerprint coverage (#2491)

* fix(browser): preserve unknown storage inventory and fingerprint coverage

* fix(browser): compare known storage edits with incomplete inventories

* test(browser): consolidate storage completeness regression workflows

* test(ida): admit aligned FAT thinning fixtures on every platform

---------

Co-authored-by: chAngE <172472630+chAng-L19@users.noreply.github.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-10-14 02:50:03 +08:00
1 parent 090b454350
commit c7ceaadb70
6 files changed
+506 -82

No files matched your search

+25 -3
View File
@@ -1,5 +1,6 @@
import type { InspectWebPageInput } from "../domain/browserObservation.js";
import type { WebPageInspection } from "../domain/browserObservationSchemas.js";
import type { BrowserExclusionReason } from "../domain/browserCompleteness.js";
import { CdpConnection } from "./CdpConnection.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
import {
@@ -26,6 +27,7 @@ export const captureStorage = async (
limitations: string[],
): Promise<{
readonly value: WebPageInspection["storage"];
readonly keysExclusion: BrowserExclusionReason | undefined;
}> => {
const origin = new URL(pageUrl).origin;
const quota = recordValue(
@@ -82,7 +84,18 @@ export const captureStorage = async (
// list is an incomplete enumeration rather than an empty one.
Array.isArray(cacheField) &&
caches.length === cacheField.length;
const keysExclusion = !context.input.include_storage_keys
? undefined
: local.exclusion === "provider_unavailable" ||
session.exclusion === "provider_unavailable" ||
indexedRaw === undefined ||
cacheRaw === undefined
? "provider_unavailable"
: !local.complete || !session.complete || !structuredStorageComplete
? "invalid_protocol_value"
: undefined;
return {
keysExclusion,
value: {
origin,
usage_bytes: numberValue(quota?.usage) ?? null,
@@ -99,7 +112,11 @@ export const captureStorage = async (
};
};
const emptyStorageItems: CapturedStorageItems = {
type StorageItemsCapture = CapturedStorageItems & {
readonly exclusion?: "provider_unavailable" | "invalid_protocol_value";
};
const emptyStorageItems: StorageItemsCapture = {
items: [],
complete: false,
};
@@ -118,7 +135,7 @@ const storageItems = async (
origin: string,
isLocalStorage: boolean,
limitations: string[],
): Promise<CapturedStorageItems> => {
): Promise<StorageItemsCapture> => {
const raw = await optionalCdpCommand(
context,
"DOMStorage.getDOMStorageItems",
@@ -127,7 +144,12 @@ const storageItems = async (
);
const entries = recordValue(raw)?.entries;
if (raw === undefined || !Array.isArray(entries))
return { items: [], complete: false };
return {
items: [],
complete: false,
exclusion:
raw === undefined ? "provider_unavailable" : "invalid_protocol_value",
};
const items = entries.flatMap((entry) => {
if (!Array.isArray(entry)) return [];
const key = cdpStringValue(entry[0]);
+66 -11
View File
@@ -193,7 +193,9 @@ const addIndexedDb = async (
scope: "indexed_db_schema",
identity: databaseName,
value: stores.map(indexedDbStoreSchema),
complete: indexedDbSchemaComplete(stores),
complete:
stores.length === database.objectStores.length &&
indexedDbSchemaComplete(stores),
});
for (const store of stores) {
const objectStoreName = cdpStringValue(store.name);
@@ -228,19 +230,37 @@ const indexedDbSchemaComplete = (
stores.every(
(store) =>
cdpStringValue(store.name) !== undefined &&
stableValueComplete(store.keyPath) &&
indexedDbKeyPathComplete(store.keyPath) &&
typeof store.autoIncrement === "boolean" &&
Array.isArray(store.indexes) &&
recordsValue(store.indexes).length === store.indexes.length &&
recordsValue(store.indexes).every(
(index) =>
cdpStringValue(index.name) !== undefined &&
stableValueComplete(index.keyPath) &&
indexedDbKeyPathComplete(index.keyPath) &&
typeof index.unique === "boolean" &&
typeof index.multiEntry === "boolean",
),
);
const indexedDbKeyPathComplete = (value: unknown): boolean => {
const keyPath = recordValue(value);
if (keyPath === undefined) return false;
switch (keyPath.type) {
case "null":
return true;
case "string":
return typeof keyPath.string === "string";
case "array":
return (
Array.isArray(keyPath.array) &&
keyPath.array.every((key) => typeof key === "string")
);
default:
return false;
}
};
const addIndexedDbRecords = async (
run: FingerprintRun,
input: {
@@ -474,14 +494,49 @@ const stableValueComplete = (value: unknown): boolean => {
const remoteObjectComplete = (value: unknown): boolean => {
const record = recordValue(value);
if (record === undefined) return true;
return (
record.objectId === undefined &&
record.preview === undefined &&
record.customPreview === undefined &&
(record.type !== "object" || record.value !== undefined) &&
stableValueComplete(record)
);
if (
record === undefined ||
record.objectId !== undefined ||
record.preview !== undefined ||
record.customPreview !== undefined ||
!stableValueComplete(record)
)
return false;
switch (record.type) {
case "undefined":
return (
record.value === undefined && record.unserializableValue === undefined
);
case "string":
case "boolean":
return (
typeof record.value === record.type &&
record.unserializableValue === undefined
);
case "number":
return record.unserializableValue === undefined
? typeof record.value === "number" && Number.isFinite(record.value)
: record.value === undefined &&
typeof record.unserializableValue === "string" &&
["-0", "NaN", "Infinity", "-Infinity"].includes(
record.unserializableValue,
);
case "bigint":
return (
record.value === undefined &&
typeof record.unserializableValue === "string" &&
/^-?\d+n$/u.test(record.unserializableValue)
);
case "object":
return (
record.unserializableValue === undefined &&
(record.subtype === "null"
? record.value === null
: record.value !== null && typeof record.value === "object")
);
default:
return false;
}
};
const decodeBase64 = (value: string): Buffer | null => {
+6
View File
@@ -155,6 +155,12 @@ const captureAuthorizedPage = async (
);
if (!input.include_storage_keys)
state.events.completeness.exclude("storage_keys", "not_approved", null);
else if (storageCapture.keysExclusion !== undefined)
state.events.completeness.exclude(
"storage_keys",
storageCapture.keysExclusion,
null,
);
await report(context.progress, 3, "Normalizing browser evidence");
const completedFrameResult = await authorizedMainFrame({
connection: context.connection,
+6 -5
View File
@@ -683,16 +683,17 @@ const storageMap = (
}),
],
]);
if (!includeKeys) return map;
const add = (kind: string, keys: readonly string[]) => {
for (const key of keys) {
map.set(`storage:${kind}:${key}`, digestCanonicalValue(key));
}
};
add("local_storage", storage.local_storage_keys);
add("session_storage", storage.session_storage_keys);
add("indexed_db", storage.indexed_db_names);
add("cache", storage.cache_names);
if (includeKeys) {
add("local_storage", storage.local_storage_keys);
add("session_storage", storage.session_storage_keys);
add("indexed_db", storage.indexed_db_names);
add("cache", storage.cache_names);
}
for (const fingerprint of storage.content_fingerprints) {
const identity = `${fingerprint.scope}:${fingerprint.identity_sha256}`;
if (!fingerprintIdentities.has(identity)) continue;
+48 -63
View File
@@ -21,12 +21,13 @@ afterEach(async () => {
}
});
/** Write a two-slice universal fixture and resolve it for the given host. */
const universalTarget = async (hostArchitecture: "arm64" | "x64") => {
const resolveUniversalTarget = async (
bytes: Buffer,
hostArchitecture: "arm64" | "x64",
) => {
const directory = await createTestTempDirectory("rea-ida-thin-");
roots.push(directory);
const path = join(directory, "universal");
const bytes = fat([0x01000007, 0x0100000c]);
await writeFile(path, bytes);
const resolved = await parseBinaryTarget(path, { hostArchitecture });
if (!resolved.ok) throw resolved.error;
@@ -35,6 +36,27 @@ const universalTarget = async (hostArchitecture: "arm64" | "x64") => {
return { bytes, target: resolved.value };
};
/** Write a two-slice universal fixture and resolve it for the given host. */
const universalTarget = (hostArchitecture: "arm64" | "x64") =>
resolveUniversalTarget(fat([0x01000007, 0x0100000c]), hostArchitecture);
const copySelectedSlice = async ({
bytes,
target,
}: Awaited<ReturnType<typeof universalTarget>>) => {
const workspace = await IdaWorkspace.allocate(target);
roots.push(workspace.directory);
await workspace.copyInput();
const input = await readFile(workspace.inputPath);
const { offset, size } = target.universalSlice;
expect(input.length, "no byte after the slice is copied").toBe(size);
expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true);
expect(workspace.inputSha256).toBe(
createHash("sha256").update(input).digest("hex"),
);
return { workspace, input };
};
/** One slice of a synthetic universal file: CPU type, offset and size. */
interface Layout {
readonly cpu: number;
@@ -43,6 +65,12 @@ interface Layout {
readonly alignmentExponent?: number;
}
const multiSliceLayout: readonly Layout[] = [
{ cpu: 0x01000007, offset: 4096, size: 8480, alignmentExponent: 12 },
{ cpu: 0x0100000c, offset: 16384, size: 33464 },
{ cpu: 0x00000007, offset: 65536, size: 33464 },
];
/**
* Write a universal file whose slices span more than one 64 KiB read, each
* filled with its own byte so bytes outside the selected slice are visible.
@@ -51,9 +79,6 @@ const largeUniversalTarget = async (
layout: readonly Layout[],
hostArchitecture: "arm64" | "x64",
) => {
const directory = await createTestTempDirectory("rea-ida-thin-large-");
roots.push(directory);
const path = join(directory, "universal");
const total = Math.max(...layout.map(({ offset, size }) => offset + size));
const bytes = Buffer.alloc(total);
bytes.writeUInt32BE(0xcafebabe, 0);
@@ -67,37 +92,27 @@ const largeUniversalTarget = async (
bytes.fill(0x11 * (index + 1), offset, offset + size);
thinMach(0xfeedfacf, cpu).copy(bytes, offset);
});
await writeFile(path, bytes);
const resolved = await parseBinaryTarget(path, { hostArchitecture });
if (!resolved.ok) throw resolved.error;
if (resolved.value.format !== "mach-o-universal")
throw new Error("the universal fixture did not resolve as a FAT container");
return { bytes, target: resolved.value };
return await resolveUniversalTarget(bytes, hostArchitecture);
};
describe("IDA workspace input thinning", () => {
it("admits the multi-read FAT fixture on every host platform", async () => {
const { target } = await largeUniversalTarget(multiSliceLayout, "arm64");
expect(target.universalSlice).toMatchObject({ offset: 16384, size: 33464 });
});
it.skipIf(process.platform === "win32")(
"thins the selected arm64 slice into the private headless input",
async () => {
const { bytes, target } = await universalTarget("arm64");
const workspace = await IdaWorkspace.allocate(target);
roots.push(workspace.directory);
await workspace.copyInput();
const input = await readFile(workspace.inputPath);
const { offset, size } = target.universalSlice;
expect(offset).toBeGreaterThan(0);
expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true);
expect(input.length).toBe(size);
const { workspace, input } = await copySelectedSlice({ bytes, target });
expect(target.universalSlice.offset).toBeGreaterThan(0);
expect(input.readUInt32BE(0)).toBe(0xfeedfacf);
expect(input.readUInt32BE(4)).toBe(0x0100000c);
expect(parseExecutableHeader(input, "arm64")).toMatchObject({
ok: true,
value: { format: "mach-o", architecture: "arm64" },
});
expect(createHash("sha256").update(input).digest("hex")).toBe(
workspace.inputSha256,
);
await workspace.remove();
},
);
@@ -106,13 +121,7 @@ describe("IDA workspace input thinning", () => {
"thins the x86_64 slice when the host is intel",
async () => {
const { bytes, target } = await universalTarget("x64");
const workspace = await IdaWorkspace.allocate(target);
roots.push(workspace.directory);
await workspace.copyInput();
const input = await readFile(workspace.inputPath);
const { offset, size } = target.universalSlice;
expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true);
const { workspace, input } = await copySelectedSlice({ bytes, target });
expect(input.readUInt32BE(4)).toBe(0x01000007);
await workspace.remove();
},
@@ -122,31 +131,12 @@ describe("IDA workspace input thinning", () => {
"copies exactly the selected slice when later slices follow it across read boundaries",
async () => {
const { bytes, target } = await largeUniversalTarget(
[
{
cpu: 0x01000007,
offset: 4096,
size: 8480,
alignmentExponent: 12,
},
{ cpu: 0x0100000c, offset: 16384, size: 33464 },
{ cpu: 0x00000007, offset: 65536, size: 33464 },
],
multiSliceLayout,
"arm64",
);
const workspace = await IdaWorkspace.allocate(target);
roots.push(workspace.directory);
await workspace.copyInput();
const input = await readFile(workspace.inputPath);
const { offset, size } = target.universalSlice;
expect(size, "the arm64 slice is selected").toBe(33464);
expect(input.length, "no byte after the slice is copied").toBe(size);
expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true);
expect(workspace.inputSha256).toBe(
createHash("sha256")
.update(bytes.subarray(offset, offset + size))
.digest("hex"),
const { workspace } = await copySelectedSlice({ bytes, target });
expect(target.universalSlice.size, "the arm64 slice is selected").toBe(
33464,
);
await workspace.remove();
},
@@ -162,15 +152,10 @@ describe("IDA workspace input thinning", () => {
],
"x64",
);
const workspace = await IdaWorkspace.allocate(target);
roots.push(workspace.directory);
await workspace.copyInput();
const input = await readFile(workspace.inputPath);
const { offset, size } = target.universalSlice;
expect(size, "the x86_64 slice is selected").toBe(37456);
expect(input.length, "no byte after the slice is copied").toBe(size);
expect(input.equals(bytes.subarray(offset, offset + size))).toBe(true);
const { workspace } = await copySelectedSlice({ bytes, target });
expect(target.universalSlice.size, "the x86_64 slice is selected").toBe(
37456,
);
await workspace.remove();
},
);
@@ -0,0 +1,355 @@
import { expect, it } from "vitest";
import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";
import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js";
import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js";
import {
startFakeCdpBrowser,
type FakeOptions,
} from "../../fixtures/fakeCdpBrowser.js";
import { trackBrowser } from "./cdpBrowserProvider.support.js";
const inspect = async (options: FakeOptions, includeFingerprints = true) => {
const browser = await startFakeCdpBrowser(options);
trackBrowser(browser);
const input = inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
include_storage_keys: true,
include_storage_fingerprints: includeFingerprints,
});
const capture = async () => {
const result = await new CdpBrowserProvider().inspectPage(input);
if (!result.ok) throw result.error;
return result.value;
};
return { capture };
};
const compareStorage = (before: WebPageInspection, after: WebPageInspection) =>
compareWebCaptures(
compareWebCapturesInputSchema.parse({
before: { inspection: before },
after: { inspection: after },
}),
).dimensions.storage;
it.each([
["CacheStorage.requestCacheNames", {}],
["CacheStorage.requestCacheNames", { caches: null }],
["CacheStorage.requestCacheNames", { caches: [null] }],
[
"CacheStorage.requestCacheNames",
{ caches: [{ cacheName: "assets-v1", cacheId: "secret-id" }, null] },
],
["CacheStorage.requestCacheNames", { caches: [{ cacheName: "assets-v1" }] }],
["IndexedDB.requestDatabaseNames", { databaseNames: [null] }],
["DOMStorage.getDOMStorageItems", { entries: [null] }],
] as const)(
"preserves unknown inventory coverage from %s: %j",
async (method, reply) => {
let malformed = false;
const { capture } = await inspect({
commandResult: (command) =>
malformed && command.method === method ? reply : undefined,
});
const before = await capture();
malformed = true;
const after = await capture();
expect(after.storage.fingerprints_complete).toBe(false);
expect(after.completeness.unavailable_sections).toContain("storage_keys");
expect(after.completeness.excluded).toContainEqual({
section: "storage_keys",
reason: "invalid_protocol_value",
count: null,
});
expect(compareStorage(before, after)).toMatchObject({
status: "unknown",
total_changes: 0,
});
},
);
it("preserves unavailable storage-key coverage without requesting fingerprints", async () => {
const { capture } = await inspect(
{ unsupportedMethods: ["CacheStorage.requestCacheNames"] },
false,
);
const captured = await capture();
expect(captured.completeness.excluded).toContainEqual({
section: "storage_keys",
reason: "provider_unavailable",
count: null,
});
expect(captured.storage.local_storage_keys).toEqual(["public-key"]);
});
const validKey = { type: "string", value: "row-1" };
it.each(["unavailable", "malformed"] as const)(
"compares shared complete values with %s cache inventory",
async (inventory) => {
let entries: unknown[][] = [["shared-key", "before"]];
const { capture } = await inspect({
...(inventory === "unavailable"
? { unsupportedMethods: ["CacheStorage.requestCacheNames"] }
: {}),
commandResult: (command) => {
if (command.method === "CacheStorage.requestCacheNames")
return { caches: [null] };
if (command.method === "DOMStorage.getDOMStorageItems")
return { entries };
return undefined;
},
});
const before = await capture();
expect(before.completeness.unavailable_sections).toContain("storage_keys");
expect(before.storage.fingerprints_complete).toBe(false);
expect(compareStorage(before, await capture())).toMatchObject({
status: "unknown",
total_changes: 0,
});
entries = [["shared-key", "after"]];
const changed = await capture();
expect(changed.storage.usage_bytes).toBe(before.storage.usage_bytes);
expect(changed.storage.quota_bytes).toBe(before.storage.quota_bytes);
for (const [left, right] of [
[before, changed],
[changed, before],
] as const)
expect(compareStorage(left, right)).toMatchObject({
status: "changed",
total_changes: 2,
changes: [
expect.objectContaining({ change: "modified" }),
expect.objectContaining({ change: "modified" }),
],
});
const incompleteValues = structuredClone(changed);
for (const fingerprint of incompleteValues.storage.content_fingerprints)
if (["local_storage", "session_storage"].includes(fingerprint.scope))
fingerprint.complete = false;
expect(compareStorage(before, incompleteValues)).toMatchObject({
status: "unknown",
total_changes: 0,
});
// Missing or invalid peer values establish neither removal nor addition.
for (const partial of [[], [["shared-key", null]]]) {
entries = partial;
const incomplete = await capture();
for (const [left, right] of [
[before, incomplete],
[incomplete, before],
] as const)
expect(compareStorage(left, right)).toMatchObject({
status: "unknown",
total_changes: 0,
changes: [],
});
}
},
);
it.each(["key", "primaryKey"])(
"requires IndexedDB %s identity evidence",
async (field) => {
const { capture } = await inspect({
commandResult: (command) =>
command.method === "IndexedDB.requestData"
? {
objectStoreDataEntries: [
{
key: validKey,
primaryKey: validKey,
value: validKey,
[field]: {},
},
],
hasMore: false,
}
: undefined,
});
const captured = await capture();
expect(captured.storage.fingerprints_complete).toBe(false);
expect(
captured.storage.content_fingerprints.find(
({ scope }) => scope === "indexed_db_record",
)?.complete,
).toBe(false);
},
);
it("does not fingerprint a filtered partial object-store schema as complete", async () => {
let malformed = false;
const store = {
name: "records",
keyPath: { type: "string", string: "id" },
autoIncrement: false,
indexes: [],
};
const { capture } = await inspect({
commandResult: (command) =>
command.method === "IndexedDB.requestDatabase"
? {
databaseWithObjectStores: {
name: "app-db",
version: 1,
objectStores: [
store,
malformed ? null : { ...store, name: "other" },
],
},
}
: undefined,
});
const before = await capture();
malformed = true;
const after = await capture();
expect(
after.storage.content_fingerprints.find(
({ scope }) => scope === "indexed_db_schema",
)?.complete,
).toBe(false);
expect(compareStorage(before, after)).toMatchObject({
status: "unknown",
total_changes: 0,
});
});
const incompleteRemoteObjects = [
undefined,
null,
"raw",
{},
{ type: "string" },
{ type: "string", value: 1 },
{ type: "boolean", value: "false" },
{ type: "number" },
{ type: "number", unserializableValue: "invalid" },
{ type: "number", unserializableValue: ["NaN"] },
{ type: "function", description: "function f() {}" },
{ type: "symbol", description: "Symbol(x)" },
{ type: "object", subtype: "null" },
{ type: "bigint", unserializableValue: "broken" },
] as const;
const completeRemoteObjects = [
{ type: "string", value: "" },
{ type: "boolean", value: false },
{ type: "number", value: 0 },
{ type: "undefined" },
{ type: "number", unserializableValue: "NaN" },
{ type: "number", unserializableValue: "-0" },
{ type: "bigint", unserializableValue: "-42n" },
{ type: "object", subtype: "null", value: null },
{ type: "object", value: { nested: [false, null, 0] } },
] as const;
it.each([
...incompleteRemoteObjects.map((value) => [value, false] as const),
...completeRemoteObjects.map((value) => [value, true] as const),
])(
"validates IndexedDB RemoteObject completeness: %j => %s",
async (value, complete) => {
const { capture } = await inspect({
commandResult: (command) =>
command.method === "IndexedDB.requestData"
? {
objectStoreDataEntries: [
{ key: validKey, primaryKey: validKey, value },
],
hasMore: false,
}
: undefined,
});
const before = await capture();
const after = await capture();
const records = before.storage.content_fingerprints.filter(
({ scope }) => scope === "indexed_db_record",
);
expect(records).toHaveLength(1);
expect(records[0]?.complete).toBe(complete);
expect(before.storage.fingerprints_complete).toBe(complete);
expect(compareStorage(before, after)).toMatchObject({
status: complete ? "unchanged" : "unknown",
total_changes: 0,
});
},
);
it.each([
[undefined, false],
[{}, false],
[{ type: "string" }, false],
[{ type: "array", array: [null] }, false],
[{ type: "null" }, true],
[{ type: "string", string: "id" }, true],
[{ type: "array", array: ["id", "name"] }, true],
] as const)(
"requires complete IndexedDB key paths: %j",
async (keyPath, complete) => {
const { capture } = await inspect({
commandResult: (command) =>
command.method === "IndexedDB.requestDatabase"
? {
databaseWithObjectStores: {
name: "app-db",
version: 1,
objectStores: [
{
name: "records",
keyPath,
autoIncrement: false,
indexes: [
{
name: "by-id",
keyPath,
unique: false,
multiEntry: false,
},
],
},
],
},
}
: undefined,
});
const captured = await capture();
expect(
captured.storage.content_fingerprints.find(
({ scope }) => scope === "indexed_db_schema",
)?.complete,
).toBe(complete);
expect(captured.storage.fingerprints_complete).toBe(complete);
},
);
it("accepts genuinely empty inventories as complete", async () => {
const { capture } = await inspect({
commandResult: (command) => {
switch (command.method) {
case "CacheStorage.requestCacheNames":
return { caches: [] };
case "IndexedDB.requestDatabaseNames":
return { databaseNames: [] };
case "DOMStorage.getDOMStorageItems":
return { entries: [] };
case "Network.getCookies":
return { cookies: [] };
default:
return undefined;
}
},
});
const before = await capture();
const after = await capture();
expect(before.storage.fingerprints_complete).toBe(true);
expect(compareStorage(before, after).status).toBe("unchanged");
});