diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index ec2c139326..ab67d458cf 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -8,6 +8,7 @@ import { classifyParsedJavaScriptOpenReceiversSteps, } from "../../domain/javascript/javascriptSemanticAnalysis.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; +import { projectJavaScriptModuleSemantics } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js"; import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; import { flattenSourceMapLeaves } from "../../domain/sourceMapEnvelope.js"; @@ -95,27 +96,7 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async ( ): Generator { const projection = projectSemantics(file, ir); if (projection !== undefined) while (projection.next().done !== true) yield; - const programScopes = ir.scopes.filter(({ kind }) => kind === "program"); - const programScopeId = programScopes[0]?.scopeId; - const localNames = new Set( - ir.moduleLinks.map(({ localName }) => localName), - ); - const callableIds = new Set( - ir.moduleLinks.map(({ callableId }) => callableId), - ); - return { - scopes: programScopes, - bindings: ir.bindings.filter( - ({ name, scopeId }) => - scopeId === programScopeId && localNames.has(name), - ), - callables: ir.callables.filter(({ callableId }) => - callableIds.has(callableId), - ), - moduleLinks: ir.moduleLinks, - coverage: ir.coverage, - limitations: ir.limitations, - }; + return projectJavaScriptModuleSemantics(ir); }; const context = { state, projectSemantics: projectFileSemantics }; for (const [index, file] of fileSet.files.entries()) { diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index 261ac88ec2..719ea1824b 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -4,6 +4,7 @@ import type { JavaScriptStaticAnalysis, } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; interface JavaScriptStructuredObservation { @@ -89,15 +90,7 @@ export type JavaScriptJsonModuleObservation = JavaScriptStructuredObservation & ); /** Module facts needed after file-local semantic projection has finished. */ -export type JavaScriptModuleSemanticIr = Pick< - JavaScriptSemanticIr, - | "scopes" - | "bindings" - | "callables" - | "moduleLinks" - | "coverage" - | "limitations" ->; +export type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; /** One relevant file plus optional AST-only JavaScript facts. */ export interface AnalyzedJavaScriptArtifactFile< diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 3ed0cafdae..265592b9ed 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -55,6 +55,7 @@ import { bindSemanticGraphApplicationNodes, type SemanticGraphProjectionState as BuilderState, } from "./JavaScriptSemanticGraphConstruction.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; interface BuilderInput { readonly rootArtifactSha256: string; @@ -126,6 +127,12 @@ export const buildJavaScriptSemanticGraph = ({ /** File-local semantic projection that does not retain consumed source IR. */ export interface JavaScriptSemanticGraphProjection { + /** Node capacity offered to the next independently analyzed source file. */ + readonly remainingFileNodeBudget: () => number; + /** Adopt decoded file-local facts; final validation and proof remain in this realm. */ + readonly adoptFileProjection: ( + projection: JavaScriptSemanticFileProjection, + ) => void; readonly projectFile: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, @@ -252,6 +259,36 @@ export const createJavaScriptSemanticGraphProjection = return graph; }; return { + remainingFileNodeBudget: () => + Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size), + ), + adoptFileProjection: (projection) => { + const newNodes = projection.nodes.filter( + ({ node_id }) => !state.nodes.has(node_id), + ); + if ( + newNodes.length > + Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size, + ) + ) + throw new RangeError( + "Transferred semantic nodes exceed the offered tree capacity", + ); + for (const context of projection.evidenceContexts) + state.evidenceContexts.adopt(context); + for (const node of newNodes) state.nodes.set(node.node_id, node); + for (const relation of projection.relations) + state.relations.set(relation.relation_id, relation); + for (const unknown of projection.unknowns) + state.unknowns.set(unknown.unknown_id, unknown); + for (const root of projection.roots) state.roots.add(root); + fingerprints.push(...projection.fingerprints); + if (projection.truncated) truncatedFiles += 1; + }, projectFile: (file, ir) => { const steps = projectSourceSteps(file, ir); while (steps.next().done !== true); @@ -272,6 +309,35 @@ export const createJavaScriptSemanticGraphProjection = }; }; +/** Project one source with the parent's remaining node capacity, retaining no full IR. */ +export function* projectJavaScriptSemanticFileSteps( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + nodeBudget: number, +): Generator { + if ( + !Number.isSafeInteger(nodeBudget) || + nodeBudget < 0 || + nodeBudget > SEMANTIC_GRAPH_FILE_NODE_CEILING + ) + throw new RangeError( + "File semantic projection requires a valid offered node capacity", + ); + const state = emptyState(); + state.fileNodeBudget = nodeBudget; + state.fileNodesDropped = false; + const fingerprints = yield* projectFileSteps(file, ir, state); + return { + roots: [...state.roots], + evidenceContexts: state.evidenceContexts.contexts, + nodes: [...state.nodes.values()], + relations: [...state.relations.values()], + unknowns: [...state.unknowns.values()], + fingerprints, + truncated: Boolean(state.fileNodesDropped), + }; +} + function* projectFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, diff --git a/src/domain/javascript/javascriptModuleSemanticIr.ts b/src/domain/javascript/javascriptModuleSemanticIr.ts new file mode 100644 index 0000000000..e2fc94e1ff --- /dev/null +++ b/src/domain/javascript/javascriptModuleSemanticIr.ts @@ -0,0 +1,36 @@ +import type { JavaScriptSemanticIr } from "./javascriptSemanticIr.js"; + +/** Module facts needed after file-local semantic projection has finished. */ +export type JavaScriptModuleSemanticIr = Pick< + JavaScriptSemanticIr, + | "scopes" + | "bindings" + | "callables" + | "moduleLinks" + | "coverage" + | "limitations" +>; + +/** Retain facts consumed by source-module and export-return composition. */ +export const projectJavaScriptModuleSemantics = ( + ir: JavaScriptSemanticIr, +): JavaScriptModuleSemanticIr => { + const scopes = ir.scopes.filter(({ kind }) => kind === "program"); + const programScopeId = scopes[0]?.scopeId; + const localNames = new Set(ir.moduleLinks.map(({ localName }) => localName)); + const callableIds = new Set( + ir.moduleLinks.map(({ callableId }) => callableId), + ); + return { + scopes, + bindings: ir.bindings.filter( + ({ name, scopeId }) => scopeId === programScopeId && localNames.has(name), + ), + callables: ir.callables.filter(({ callableId }) => + callableIds.has(callableId), + ), + moduleLinks: ir.moduleLinks, + coverage: ir.coverage, + limitations: ir.limitations, + }; +}; diff --git a/src/domain/javascript/javascriptSemanticFileProjection.ts b/src/domain/javascript/javascriptSemanticFileProjection.ts new file mode 100644 index 0000000000..8a7a4bb5e0 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticFileProjection.ts @@ -0,0 +1,18 @@ +import type { + JavaScriptSemanticEvidenceContext, + JavaScriptSemanticFingerprint, + JavaScriptSemanticGraphNode, + JavaScriptSemanticGraphRelation, + JavaScriptSemanticGraphUnknown, +} from "./javascriptSemanticGraphSchemas.js"; + +/** File-local facts transported before binding them to the containing application graph. */ +export interface JavaScriptSemanticFileProjection { + readonly roots: readonly string[]; + readonly evidenceContexts: readonly JavaScriptSemanticEvidenceContext[]; + readonly nodes: readonly JavaScriptSemanticGraphNode[]; + readonly relations: readonly JavaScriptSemanticGraphRelation[]; + readonly unknowns: readonly JavaScriptSemanticGraphUnknown[]; + readonly fingerprints: readonly JavaScriptSemanticFingerprint[]; + readonly truncated: boolean; +} diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index ce7eb6ae85..cdaa8c28b8 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -19,6 +19,7 @@ import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, JAVASCRIPT_SEMANTIC_RELATION_FAMILY, javaScriptSemanticFingerprintInputSchema, + javaScriptSemanticEvidenceContextSchema, javaScriptSemanticGraphInputSchema, javaScriptSemanticGraphRecordSchema, javaScriptSemanticNodeInputSchema, @@ -109,6 +110,33 @@ export class JavaScriptSemanticEvidenceContextRegistry { this.#contexts.clear(); this.#idsByCanonicalContext.clear(); } + + /** Validate and intern provenance decoded in this realm from an analysis worker. */ + adopt(input: unknown): void { + const context = javaScriptSemanticEvidenceContextSchema.parse(input); + const { context_id: identifier, ...value } = context; + if (identifier !== evidenceContextId(value)) + throw new TypeError( + "Transferred semantic evidence context identifier is stale", + ); + const canonical = canonicalJson( + value, + "JavaScript semantic evidence context", + ); + const existing = this.#contexts.get(identifier); + if (existing !== undefined) { + const { context_id: _existingIdentifier, ...existingValue } = existing; + if ( + canonical !== + canonicalJson(existingValue, "JavaScript semantic evidence context") + ) + throw new TypeError( + "Transferred semantic evidence context digest collision", + ); + } + this.#contexts.set(identifier, context); + this.#idsByCanonicalContext.set(canonical, identifier); + } } /** Derive a semantic node ID from its kind and exact artifact identity. */ diff --git a/src/javascript/analysis/JavaScriptAnalysisTransfer.ts b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts new file mode 100644 index 0000000000..a2be62d2c8 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts @@ -0,0 +1,276 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { open, stat } from "node:fs/promises"; +import { z } from "zod"; + +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; +import { digestSchema } from "../../domain/digests.js"; +import { + javaScriptWorkerGraphRecordSchema, + javaScriptWorkerModuleHeaderSchema, + javaScriptWorkerModuleRecordSchema, + javaScriptWorkerProjectionHeaderSchema, +} from "./JavaScriptAnalysisWorkerSchemas.js"; + +// Budgets bound transfer representation, independently of source length. +const FRAME_BYTES = 2 * 1024 * 1024; +const TRANSFER_BYTES = 512 * 1024 * 1024; +const BATCH_BYTES = 1024 * 1024; +const recordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("projection-header"), + value: javaScriptWorkerProjectionHeaderSchema, + }), + z.strictObject({ + kind: z.literal("module-header"), + value: javaScriptWorkerModuleHeaderSchema, + }), + ...javaScriptWorkerGraphRecordSchema.options, + ...javaScriptWorkerModuleRecordSchema.options, +]); + +/** Complete-file coordinates authenticated before adopting any worker facts. */ +export const javaScriptAnalysisTransferDescriptorSchema = z.strictObject({ + bytes: z.number().int().nonnegative().max(TRANSFER_BYTES), + sha256: digestSchema, + records: z.number().int().nonnegative(), +}); +type Descriptor = z.output; + +/** Encode validated facts in bounded records, without serializing a whole graph or IR. */ +export const writeJavaScriptAnalysisTransfer = async ( + path: string, + module: JavaScriptModuleSemanticIr, + projection: JavaScriptSemanticFileProjection, +): Promise => { + const output = await open(path, "wx", 0o600); + const digest = createHash("sha256"); + let bytes = 0; + let records = 0; + let batch: string[] = []; + let batchBytes = 0; + const flush = async (): Promise => { + if (batch.length === 0) return; + const buffer = Buffer.from(batch.join("")); + // FileHandle.write can be short; writeFile completes the buffer. + await output.writeFile(buffer); + digest.update(buffer); + batch = []; + batchBytes = 0; + }; + const append = async (input: unknown): Promise => { + const record = recordSchema.parse(input); + const line = `${JSON.stringify(record)}\n`; + const size = Buffer.byteLength(line); + if (size > FRAME_BYTES || bytes + size > TRANSFER_BYTES) + throw new RangeError( + "JavaScript analysis transfer exceeds its record or file byte budget", + ); + bytes += size; + records += 1; + batch.push(line); + batchBytes += size; + if (batchBytes >= BATCH_BYTES) await flush(); + }; + try { + await append({ + kind: "projection-header", + value: { roots: projection.roots, truncated: projection.truncated }, + }); + await append({ + kind: "module-header", + value: { coverage: module.coverage, limitations: module.limitations }, + }); + for (const value of module.scopes) await append({ kind: "scope", value }); + for (const value of module.bindings) + await append({ kind: "binding", value }); + for (const value of module.callables) + await append({ kind: "callable", value }); + for (const value of module.moduleLinks) + await append({ kind: "module-link", value }); + for (const value of projection.evidenceContexts) + await append({ kind: "evidence-context", value }); + for (const value of projection.nodes) await append({ kind: "node", value }); + for (const value of projection.relations) + await append({ kind: "relation", value }); + for (const value of projection.unknowns) + await append({ kind: "unknown", value }); + for (const value of projection.fingerprints) + await append({ kind: "fingerprint", value }); + await flush(); + return { bytes, sha256: digest.digest("hex"), records }; + } finally { + await output.close(); + } +}; + +/** Decode producer records into realm-local, source-bound facts; sealing follows in the consumer. */ +export const readJavaScriptAnalysisTransfer = async ( + path: string, + descriptor: Descriptor, + source: Pick, + signal?: AbortSignal, +): Promise<{ + readonly module: JavaScriptModuleSemanticIr; + readonly projection: JavaScriptSemanticFileProjection; +}> => { + const expected = javaScriptAnalysisTransferDescriptorSchema.parse(descriptor); + if ((await stat(path)).size !== expected.bytes) + throw new TypeError( + "JavaScript analysis transfer byte count does not match", + ); + const scopes: JavaScriptModuleSemanticIr["scopes"][number][] = []; + const bindings: JavaScriptModuleSemanticIr["bindings"][number][] = []; + const callables: JavaScriptModuleSemanticIr["callables"][number][] = []; + const moduleLinks: JavaScriptModuleSemanticIr["moduleLinks"][number][] = []; + const evidenceContexts: JavaScriptSemanticFileProjection["evidenceContexts"][number][] = + []; + const nodes: JavaScriptSemanticFileProjection["nodes"][number][] = []; + const relations: JavaScriptSemanticFileProjection["relations"][number][] = []; + const unknowns: JavaScriptSemanticFileProjection["unknowns"][number][] = []; + const fingerprints: JavaScriptSemanticFileProjection["fingerprints"][number][] = + []; + let projectionHeader: + | z.output + | undefined; + let moduleHeader: + | z.output + | undefined; + const identities = new Map>(); + const retainIdentity = (kind: string, identifier: string): void => { + let seen = identities.get(kind); + if (seen === undefined) { + seen = new Set(); + identities.set(kind, seen); + } + if (seen.has(identifier)) + throw new TypeError( + `Duplicate JavaScript transfer ${kind} identity: ${identifier}`, + ); + seen.add(identifier); + }; + let records = 0; + const digest = createHash("sha256"); + const stream = createReadStream(path, { + highWaterMark: 64 * 1024, + ...(signal === undefined ? {} : { signal }), + }); + let bytes = 0; + let pending = ""; + stream.setEncoding("utf8"); + try { + for await (const chunk of stream) { + signal?.throwIfAborted(); + if (typeof chunk !== "string") + throw new TypeError("JavaScript transfer stream is not UTF-8 text"); + bytes += Buffer.byteLength(chunk); + if (bytes > expected.bytes) + throw new RangeError("JavaScript transfer grew while being read"); + digest.update(chunk); + pending += chunk; + let newline: number; + while ((newline = pending.indexOf("\n")) >= 0) { + const line = pending.slice(0, newline); + pending = pending.slice(newline + 1); + if (Buffer.byteLength(line) + 1 > FRAME_BYTES) + throw new RangeError( + "JavaScript transfer record exceeds its byte budget", + ); + const input: unknown = JSON.parse(line); + const record = recordSchema.parse(input); + if (record.kind === "projection-header") { + if (records !== 0 || projectionHeader !== undefined) + throw new TypeError( + "Duplicate or late JavaScript projection header", + ); + projectionHeader = record.value; + } else if (record.kind === "module-header") { + if (records !== 1 || moduleHeader !== undefined) + throw new TypeError("Duplicate or late JavaScript module header"); + moduleHeader = record.value; + } else { + if (projectionHeader === undefined || moduleHeader === undefined) + throw new TypeError( + "JavaScript transfer facts precede their headers", + ); + switch (record.kind) { + case "scope": + retainIdentity(record.kind, record.value.scopeId); + scopes.push(record.value); + break; + case "binding": + retainIdentity(record.kind, record.value.bindingId); + bindings.push(record.value); + break; + case "callable": + retainIdentity(record.kind, record.value.callableId); + callables.push(record.value); + break; + case "module-link": + moduleLinks.push(record.value); + break; + case "evidence-context": + retainIdentity(record.kind, record.value.context_id); + evidenceContexts.push(record.value); + break; + case "node": { + retainIdentity(record.kind, record.value.node_id); + if ( + record.value.identity.artifact_sha256 !== source.sha256 || + record.value.identity.module_path !== source.path + ) + throw new TypeError( + "JavaScript transfer node belongs to a different artifact source", + ); + nodes.push(record.value); + break; + } + case "relation": + retainIdentity(record.kind, record.value.relation_id); + relations.push(record.value); + break; + case "unknown": + retainIdentity(record.kind, record.value.unknown_id); + unknowns.push(record.value); + break; + case "fingerprint": + retainIdentity(record.kind, record.value.fingerprint_id); + fingerprints.push(record.value); + break; + } + } + records += 1; + } + if (Buffer.byteLength(pending) + 1 > FRAME_BYTES) + throw new RangeError( + "Unterminated JavaScript transfer record exceeds its byte budget", + ); + } + } finally { + stream.destroy(); + } + if ( + pending !== "" || + records !== expected.records || + bytes !== expected.bytes || + digest.digest("hex") !== expected.sha256 + ) + throw new TypeError( + "JavaScript analysis transfer is incomplete or its digest does not match", + ); + if (projectionHeader === undefined || moduleHeader === undefined) + throw new TypeError("JavaScript analysis transfer is missing its headers"); + return { + module: { ...moduleHeader, scopes, bindings, callables, moduleLinks }, + projection: { + ...projectionHeader, + evidenceContexts, + nodes, + relations, + unknowns, + fingerprints, + }, + }; +}; diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts new file mode 100644 index 0000000000..92bfaec352 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -0,0 +1,582 @@ +import { z } from "zod"; +import { digestSchema } from "../../domain/digests.js"; +import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import type { + JavaScriptSemanticProperty, + JavaScriptSemanticValue, +} from "../../domain/javascript/javascriptSemanticValueTypes.js"; +import { + javaScriptSemanticNodeSchema, + javaScriptSemanticRelationSchema, + javaScriptSemanticUnknownSchema, + javaScriptSemanticFingerprintSchema, + javaScriptSemanticEvidenceContextSchema, +} from "../../domain/javascript/javascriptSemanticGraphSchemas.js"; + +const range = + javaScriptSemanticNodeSchema.shape.identity.shape.source_range.unwrap(); +const count = z.number().int().nonnegative(); +const strings = z.array(z.string()); +const pathContext = z.enum(["module-specifier", "filesystem-expression"]); +const findingContext = { + module_key: z.string().nullable(), + module_runtime: z.string().optional(), + location: range, +}; +const withRuntime = ( + value: Value, +) => { + const { module_runtime, ...rest } = value; + return { + ...rest, + ...(module_runtime === undefined ? {} : { module_runtime }), + }; +}; +const staticValue = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("literal"), + value: z.union([z.string(), z.number(), z.boolean(), z.null()]), + expression: z.null(), + }), + z.strictObject({ + status: z.literal("dynamic"), + value: z.null(), + expression: z.string(), + }), +]); +const nullHandler = { handler_kind: z.null(), handler_location: z.null() }; +const missingHandler = { + handler_kind: z.literal("missing"), + handler_location: z.null(), +}; +const locatedHandler = { + handler_kind: z.enum([ + "inline-function", + "identifier", + "member-expression", + "dynamic-expression", + ]), + handler_location: range, +}; +const ipcChannel = (shape: Shape) => + z.union([ + z.strictObject({ + ...findingContext, + ...shape, + channel: z.string(), + channel_expression: z.null(), + }), + z.strictObject({ + ...findingContext, + ...shape, + channel: z.null(), + channel_expression: z.string(), + }), + ]); +const ipc = z + .union([ + ipcChannel({ + side: z.literal("renderer"), + operation: z.enum(["send", "send-sync", "post-message", "send-to-host"]), + mode: z.literal("send"), + ...nullHandler, + }), + ipcChannel({ + side: z.literal("renderer"), + operation: z.literal("invoke"), + mode: z.literal("invoke"), + ...nullHandler, + }), + ipcChannel({ + side: z.enum(["renderer", "main"]), + operation: z.enum(["on", "once"]), + mode: z.literal("listen"), + ...missingHandler, + }), + ipcChannel({ + side: z.enum(["renderer", "main"]), + operation: z.enum(["on", "once"]), + mode: z.literal("listen"), + ...locatedHandler, + }), + ipcChannel({ + side: z.literal("main"), + operation: z.enum(["handle", "handle-once"]), + mode: z.literal("handle"), + ...missingHandler, + }), + ipcChannel({ + side: z.literal("main"), + operation: z.enum(["handle", "handle-once"]), + mode: z.literal("handle"), + ...locatedHandler, + }), + ]) + .transform((value) => { + const { module_runtime, ...rest } = value; + return { + ...rest, + ...(module_runtime === undefined ? {} : { module_runtime }), + }; + }); +const windowShape = { + ...findingContext, + options_status: z.enum(["object-literal", "dynamic", "missing"]), + web_preferences_status: z.enum(["object-literal", "dynamic", "missing"]), + web_preferences: z.array( + z.strictObject({ name: z.string(), value: staticValue }), + ), +}; +const bridgeShape = { + ...findingContext, + world: z.enum(["main", "isolated"]), + world_id: staticValue.nullable(), + api_status: z.enum(["object-literal", "dynamic", "missing"]), + members: strings, + unknown_members: count, +}; +const utilityShape = { ...findingContext, service_name: z.string().nullable() }; +const electron = z.strictObject({ + browser_windows: z.array( + z.union([ + z + .strictObject({ + ...windowShape, + preload_path: z.string(), + preload_resolution_context: pathContext, + }) + .transform(withRuntime), + z + .strictObject({ + ...windowShape, + preload_path: z.null(), + preload_resolution_context: z.null(), + }) + .transform(withRuntime), + ]), + ), + context_bridge_apis: z.array( + z.union([ + z + .strictObject({ + ...bridgeShape, + api_key: z.string(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.string(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + ]), + ), + ipc: z.array(ipc), + sender_validations: z.array( + z.strictObject({ + ...findingContext, + subject: z.enum([ + "sender-url", + "sender-origin", + "sender-frame", + "sender-id", + "frame-id", + "process-id", + ]), + mechanism: z.string(), + expected: staticValue, + enforcement: z.literal("unknown"), + }), + ), + utility_processes: z.array( + z.union([ + z + .strictObject({ + ...utilityShape, + module_path: z.string(), + module_resolution_context: pathContext, + module_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...utilityShape, + module_path: z.null(), + module_resolution_context: z.null(), + module_expression: z.string(), + }) + .transform(withRuntime), + ]), + ), + native_addon_bindings: z.array( + z.strictObject({ + ...findingContext, + specifier: z.string(), + binding_kind: z.enum(["import", "require", "re-export"]), + module_kind: z.enum(["import", "require"]), + members: strings, + namespace_access: z.boolean().optional(), + dynamic_member_access: z.boolean().optional(), + }), + ), +}); +const referenceShape = { + ...findingContext, + kind: z.enum([ + "static-import", + "dynamic-import", + "require", + "worker", + "service-worker", + ]), +}; + +/** Exact producer facts recoverable before a later semantic worker failure. */ +export const javaScriptWorkerStaticAnalysisSchema = z + .strictObject({ + parse_status: z.enum(["complete", "partial", "failed"]), + parse_error_count: count, + visited_ast_nodes: count, + references: z.array( + z.union([ + z + .strictObject({ + ...referenceShape, + specifier: z.string(), + expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...referenceShape, + specifier: z.null(), + expression: z.string(), + }) + .transform(withRuntime), + ]), + ), + endpoints: z.array( + z.strictObject({ + ...findingContext, + kind: z.enum(["route", "network"]), + value: z.string(), + mechanism: z.string(), + }), + ), + storage: z.array( + z.strictObject({ + ...findingContext, + kind: z.enum([ + "local-storage", + "session-storage", + "indexed-db", + "cache-storage", + "sqlite", + ]), + name: z.string().nullable(), + mechanism: z.string(), + }), + ), + bundler_registrations: z.array( + z.strictObject({ + bundler: z.enum(["webpack", "rspack", "esbuild"]), + runtime: z.string(), + chunk_keys: strings, + unknown_chunk_keys: count, + runtime_require_name: z.string().nullable(), + runtime_module_cache_status: z.enum(["observed", "not-observed"]), + entry_module_keys: strings, + unknown_entry_module_keys: count, + async_chunk_keys: strings, + unknown_async_chunk_keys: count, + modules: z.array( + z.strictObject({ + module_key: z.string(), + factory_require_name: z.string().nullable(), + source_sha256: digestSchema, + exports: strings, + location: range, + structural_fingerprint_sha256: digestSchema, + structural_fingerprint_algorithm: z.literal("babel-ast-v1"), + }), + ), + location: range, + }), + ), + role_paths: z.array( + z.strictObject({ + ...findingContext, + role: z.enum(["preload", "renderer"]), + path: z.string(), + resolution_context: pathContext, + mechanism: z.string(), + }), + ), + source_map_urls: z.array( + z.strictObject({ declared_url: z.string(), location: range }), + ), + vendors: strings, + electron, + limitations: strings, + }) + .transform((analysis) => ({ + ...analysis, + endpoints: analysis.endpoints.map(withRuntime), + storage: analysis.storage.map(withRuntime), + role_paths: analysis.role_paths.map(withRuntime), + electron: { + ...analysis.electron, + sender_validations: analysis.electron.sender_validations.map(withRuntime), + native_addon_bindings: analysis.electron.native_addon_bindings.map( + (value) => { + const { namespace_access, dynamic_member_access, ...rest } = + withRuntime(value); + return { + ...rest, + ...(namespace_access === undefined ? {} : { namespace_access }), + ...(dynamic_member_access === undefined + ? {} + : { dynamic_member_access }), + }; + }, + ), + }, + })) satisfies z.ZodType; + +const resourceLimit = z.enum([ + "primitive-candidates", + "primitive-bytes", + "expression-depth", +]); +const primitive = z.union([z.string(), z.number(), z.boolean(), z.null()]); +const property: z.ZodType = z.lazy(() => + z.strictObject({ + name: z.string(), + value: semanticValue, + presence: z.enum(["present", "absent", "unknown-coverage"]), + }), +); +const semanticValue: z.ZodType = z.lazy(() => + z + .discriminatedUnion("status", [ + z.strictObject({ status: z.literal("literal"), value: primitive }), + z.strictObject({ + status: z.literal("union"), + values: z.array(primitive), + }), + z.discriminatedUnion("unknownProperties", [ + z.strictObject({ + status: z.literal("object"), + properties: z.array(property), + unknownProperties: z.literal(false), + omittedProperties: z.literal(0), + }), + z.strictObject({ + status: z.literal("object"), + properties: z.array(property), + unknownProperties: z.literal(true), + omittedProperties: count.nullable(), + }), + ]), + z.discriminatedUnion("unknownItems", [ + z.strictObject({ + status: z.literal("array"), + items: z.array(property), + unknownItems: z.literal(false), + omittedItems: z.literal(0), + }), + z.strictObject({ + status: z.literal("array"), + items: z.array(property), + unknownItems: z.literal(true), + omittedItems: count.nullable(), + }), + ]), + z.strictObject({ + status: z.enum(["unknown", "ambiguous", "cycle"]), + reason: z.string(), + resourceLimit: resourceLimit.optional(), + }), + ]) + .transform((value) => { + if ( + value.status === "literal" || + value.status === "union" || + value.status === "object" || + value.status === "array" + ) + return value; + const { resourceLimit: limit, ...rest } = value; + return { + ...rest, + ...(limit === undefined ? {} : { resourceLimit: limit }), + }; + }), +); +const origin = z.strictObject({ specifier: z.string(), importedPath: strings }); +const provenance = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("module"), + origins: z.tuple([origin]), + reason: z.null(), + }), + z.strictObject({ + status: z.literal("local"), + origins: z.tuple([]), + reason: z.null(), + }), + z.strictObject({ + status: z.literal("ambiguous"), + origins: z.array(origin), + reason: z.string(), + }), + z.strictObject({ + status: z.enum(["unknown", "cycle"]), + origins: z.tuple([]), + reason: z.string(), + }), +]); +const coverage = z + .discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("complete"), + omittedCount: z.literal(0), + resourceLimits: z.array(resourceLimit).optional(), + }), + z.strictObject({ + status: z.literal("partial"), + omittedCount: z.union([z.literal(0), z.null()]), + resourceLimits: z.array(resourceLimit).optional(), + }), + z.strictObject({ + status: z.literal("failed"), + omittedCount: z.null(), + resourceLimits: z.array(resourceLimit).optional(), + }), + ]) + .transform((value) => { + const { resourceLimits, ...rest } = value; + return { + ...rest, + ...(resourceLimits === undefined ? {} : { resourceLimits }), + }; + }); +const moduleLink = z.strictObject({ + kind: z.enum(["import", "require", "export", "re-export", "commonjs-export"]), + specifier: z.string().nullable(), + importedName: z.string().nullable(), + localName: z.string().nullable(), + exportedName: z.string().nullable(), + callableId: z.string().nullable(), + location: range, +}); +const definitionKind = z.enum([ + "import", + "variable", + "parameter", + "function", + "class", + "catch", + "assignment", +]); +const scope = z.strictObject({ + scopeId: z.string(), + parentScopeId: z.string().nullable(), + kind: z.enum([ + "program", + "function", + "block", + "static-block", + "class", + "catch", + ]), + location: range, + bindingsComplete: z.boolean(), + bindingIds: strings, +}); +const binding = z.strictObject({ + bindingId: z.string(), + scopeId: z.string(), + name: z.string(), + kind: definitionKind, + mutable: z.boolean(), + definitions: z.array( + z.strictObject({ kind: definitionKind, location: range }), + ), + value: semanticValue, + provenance, +}); +const callable = z.strictObject({ + callableId: z.string(), + kind: z.enum(["function", "class", "method"]), + name: z.string().nullable(), + containerScopeId: z.string(), + bodyScopeId: z.string().nullable(), + location: range, + returnSites: z.array( + z.strictObject({ + returnSiteId: z.string(), + location: range, + identityReferenceLocation: range.nullable(), + value: semanticValue, + }), + ), + returnCoverage: z.strictObject({ + retainedCount: count, + status: z.enum(["complete", "partial"]), + omittedCount: z.union([z.literal(0), z.null()]), + }), +}); + +/** Validate each transferred graph fact independently of its emitting realm. */ +export const javaScriptWorkerGraphRecordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("evidence-context"), + value: javaScriptSemanticEvidenceContextSchema, + }), + z.strictObject({ + kind: z.literal("node"), + value: javaScriptSemanticNodeSchema, + }), + z.strictObject({ + kind: z.literal("relation"), + value: javaScriptSemanticRelationSchema, + }), + z.strictObject({ + kind: z.literal("unknown"), + value: javaScriptSemanticUnknownSchema, + }), + z.strictObject({ + kind: z.literal("fingerprint"), + value: javaScriptSemanticFingerprintSchema, + }), +]); + +/** Portable per-file header, before any graph binding or immutable proof. */ +export const javaScriptWorkerProjectionHeaderSchema = z.strictObject({ + roots: z.array(javaScriptSemanticNodeSchema.shape.node_id), + truncated: z.boolean(), +}); + +/** Module facts are transferred individually to avoid a whole-IR JSON string. */ +export const javaScriptWorkerModuleRecordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("scope"), value: scope }), + z.strictObject({ kind: z.literal("binding"), value: binding }), + z.strictObject({ kind: z.literal("callable"), value: callable }), + z.strictObject({ kind: z.literal("module-link"), value: moduleLink }), +]); + +/** Coverage and limitations are independent of module collection sizes. */ +export const javaScriptWorkerModuleHeaderSchema = z.strictObject({ + coverage, + limitations: strings, +}); diff --git a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts index bb41d5c0b7..bc8f836eb2 100644 --- a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -3,7 +3,7 @@ import { primitiveByteExpansionSource, } from "../../fixtures/javascriptPrimitiveExpansion.js"; import { createHash } from "node:crypto"; -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createPackageWithOptions } from "@electron/asar"; @@ -23,7 +23,15 @@ import { buildJavaScriptArtifactGraph } from "../../../src/application/javascrip import { buildJavaScriptSemanticGraph, createJavaScriptSemanticGraphProjection, + projectJavaScriptSemanticFileSteps, } from "../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; +import { projectJavaScriptModuleSemantics } from "../../../src/domain/javascript/javascriptModuleSemanticIr.js"; +import { completeJavaScriptAnalysisSteps } from "../../../src/application/javascript/JavaScriptAnalysisControl.js"; +import { + writeJavaScriptAnalysisTransfer, + readJavaScriptAnalysisTransfer, +} from "../../../src/javascript/analysis/JavaScriptAnalysisTransfer.js"; +import { javaScriptWorkerStaticAnalysisSchema } from "../../../src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.js"; import { createJavaScriptArtifactReader } from "../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; import { scanCanonicalArtifactInventory, @@ -31,7 +39,10 @@ import { } from "../../fixtures/artifactInventory.js"; import { parseJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; -import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraph.js"; +import { + createJavaScriptSemanticGraph, + isValidatedImmutableJavaScriptSemanticGraph, +} from "../../../src/domain/javascript/javascriptSemanticGraph.js"; import { parseJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraphSerialization.js"; import { writeJavaScriptArtifactFixture } from "../../fixtures/javascriptArtifactApplication.js"; @@ -61,6 +72,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR", import { readFile } from "node:fs"; export { readFile as read }; export function create(value) { return { value, nested: { enabled: true } }; } + export function configured(condition) { return { values: [1, null, undefined], choice: condition ? 1 : 2, ...external }; } function unexported() { return "private"; } `, ); @@ -96,6 +108,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR", ); expect(exported?.semantic?.ir.callables.map(({ name }) => name)).toEqual([ "create", + "configured", ]); expect( full.files @@ -103,6 +116,81 @@ it("preserves graph commitments and export shapes when consuming file-local IR", ?.semantic?.ir.callables.map(({ name }) => name), ).toContain("unexported"); expect(parseJavaScriptSemanticGraph(semantic)).toEqual(semantic); + const transferred = createJavaScriptSemanticGraphProjection(); + const wireRoot = await createTestTempDirectory("rea-javascript-transfer-"); + let rejectedInvalidTransfer = false; + for (const [index, analyzed] of full.files.entries()) { + if (analyzed.javascript !== null) { + const raw: unknown = JSON.parse(JSON.stringify(analyzed.javascript)); + expect(javaScriptWorkerStaticAnalysisSchema.parse(raw)).toEqual( + analyzed.javascript, + ); + } + if (analyzed.semantic === null) continue; + const fragment = await completeJavaScriptAnalysisSteps( + projectJavaScriptSemanticFileSteps( + analyzed.file, + analyzed.semantic.ir, + transferred.remainingFileNodeBudget(), + ), + ); + const module = projectJavaScriptModuleSemantics(analyzed.semantic.ir); + const wirePath = join(wireRoot, `${String(index)}.jsonl`); + const descriptor = await writeJavaScriptAnalysisTransfer( + wirePath, + module, + fragment, + ); + const decoded = await readJavaScriptAnalysisTransfer( + wirePath, + descriptor, + analyzed.file, + ); + expect(decoded.module).toEqual(module); + transferred.adoptFileProjection(decoded.projection); + if (!rejectedInvalidTransfer) { + await expect( + readJavaScriptAnalysisTransfer(wirePath, descriptor, { + ...analyzed.file, + sha256: "0".repeat(64), + }), + ).rejects.toThrow("different artifact source"); + await expect( + readJavaScriptAnalysisTransfer( + wirePath, + { ...descriptor, sha256: "0".repeat(64) }, + analyzed.file, + ), + ).rejects.toThrow("digest does not match"); + const wire = await readFile(wirePath, "utf8"); + const duplicate = wire + .split("\n") + .find((line) => line.startsWith('{"kind":"node"')); + expect(duplicate).toBeDefined(); + if (duplicate === undefined) + throw new Error("Fixture emitted no semantic node record"); + const invalid = `${wire}${duplicate}\n`; + const invalidPath = join(wireRoot, "duplicate.jsonl"); + await writeFile(invalidPath, invalid); + await expect( + readJavaScriptAnalysisTransfer( + invalidPath, + { + bytes: Buffer.byteLength(invalid), + records: descriptor.records + 1, + sha256: createHash("sha256").update(invalid).digest("hex"), + }, + analyzed.file, + ), + ).rejects.toThrow("Duplicate JavaScript transfer node identity"); + rejectedInvalidTransfer = true; + } + } + const resealed = await completeJavaScriptAnalysisSteps( + transferred.finishImmutableSteps(snapshot.manifest.root_sha256, graph), + ); + expect(resealed).toEqual(semantic); + expect(isValidatedImmutableJavaScriptSemanticGraph(resealed)).toBe(true); } finally { await reader.close(); }