diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index de24a0fc3d..ee0709ca1f 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -114,29 +114,30 @@ const accessibilityDimension = ( const afterAccess = accessibilityComparable(after); const textComparable = beforeAccess.text && afterAccess.text; const nodesComparable = beforeAccess.nodes && afterAccess.nodes; + const reason = + "Accessibility tree or text capture was incomplete in at least one observation."; + if (!nodesComparable) + return { + status: "unknown", + total_changes: 0, + changes: [], + reason, + }; return compareDimension( singleton( "accessibility_tree", digestCanonicalValue( - accessibilityProjection( - before.accessibility, - textComparable, - nodesComparable, - ), + accessibilityProjection(before.accessibility, textComparable), ), ), singleton( "accessibility_tree", digestCanonicalValue( - accessibilityProjection( - after.accessibility, - textComparable, - nodesComparable, - ), + accessibilityProjection(after.accessibility, textComparable), ), ), beforeAccess.complete && afterAccess.complete, - "Accessibility tree or text capture was incomplete in at least one observation.", + reason, ); }; @@ -850,20 +851,32 @@ const accessibilityComparable = ( const text = inspection.accessibility.text_capture.status === "included" && inspection.accessibility.text_capture.excluded_fields === 0; + const { completeness } = inspection; + const exclusions = completeness.excluded.filter( + ({ section }) => section === "accessibility", + ); + const textOnlyPolicyFilter = + inspection.accessibility.text_capture.status === "not_approved" && + exclusions.length > 0 && + exclusions.every(({ reason }) => reason === "not_approved"); const nodes = + [ + completeness.truncated_sections, + completeness.unavailable_sections, + completeness.attach_limited_sections, + ].every((sections) => !sections.includes("accessibility")) && + (!completeness.policy_filtered_sections.includes("accessibility") || + textOnlyPolicyFilter) && inspection.accessibility.total_nodes === - inspection.accessibility.nodes.length; - const complete = - text && nodes && sectionsComplete(inspection, ["accessibility"]); + inspection.accessibility.nodes.length; + const complete = text && nodes; return { text, nodes, complete }; }; const accessibilityProjection = ( accessibility: WebPageInspection["accessibility"], includeText: boolean, - includeNodes: boolean, ): unknown => { - if (!includeNodes) return { total_nodes: accessibility.total_nodes }; const nodeIndexes = new Map( accessibility.nodes.map((node, index) => [node.node_id, index]), ); diff --git a/tests/boundary/cli/accessibilityCaptureComparison.test.ts b/tests/boundary/cli/accessibilityCaptureComparison.test.ts new file mode 100644 index 0000000000..61dc571ac7 --- /dev/null +++ b/tests/boundary/cli/accessibilityCaptureComparison.test.ts @@ -0,0 +1,218 @@ +import { expect, it } from "vitest"; + +import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js"; +import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js"; +import { runCliJson } from "../../fixtures/cliJsonProcess.js"; +import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js"; + +type Capture = WebPageInspection; +type Coverage = + | "truncated" + | "unavailable" + | "attach_limited" + | "policy_filtered"; + +const accessibleCapture = (): Capture => { + const capture = passiveNetworkCapture(); + capture.accessibility = { + total_nodes: 2, + text_capture: { + status: "included", + retained_bytes: 10, + excluded_fields: 0, + }, + nodes: [ + { + node_id: "root", + parent_id: null, + role: "RootWebArea", + name: "Page", + description: null, + ignored: false, + states: [], + }, + { + node_id: "button", + parent_id: "root", + role: "button", + name: "Submit", + description: null, + ignored: false, + states: [], + }, + ], + }; + return capture; +}; + +const setCoverage = (capture: Capture, coverage: Coverage): void => { + capture.completeness = classifyBrowserCompleteness({ + policyFilteredSections: new Set( + coverage === "policy_filtered" ? ["accessibility"] : [], + ), + attachLimitedSections: new Set( + coverage === "attach_limited" ? ["accessibility"] : [], + ), + truncatedSections: new Set( + coverage === "truncated" ? ["accessibility"] : [], + ), + unavailableSections: new Set( + coverage === "unavailable" ? ["accessibility"] : [], + ), + excluded: + coverage === "policy_filtered" + ? [{ section: "accessibility", reason: "disallowed_origin", count: 1 }] + : coverage === "unavailable" + ? [ + { + section: "accessibility", + reason: "invalid_protocol_value", + count: null, + }, + ] + : [], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }); +}; + +const compare = async (before: Capture, after: Capture) => + await runCliJson( + [ + "compare-web-captures", + JSON.stringify({ inspection: before }), + JSON.stringify({ inspection: after }), + "--json", + ], + process.env, + ); + +it.each([ + "truncated", + "unavailable", + "attach_limited", + "policy_filtered", +] as const)( + "does not infer an accessibility change from a %s tree", + async (coverage) => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.nodes.pop(); + after.accessibility.total_nodes = after.accessibility.nodes.length; + after.accessibility.text_capture.retained_bytes = 4; + setCoverage(after, coverage); + + const result = await compare(before, after); + + expect(result).toMatchObject({ + operation: "compare_web_captures", + normalized_result: { + dimensions: { + accessibility: { + status: "unknown", + total_changes: 0, + changes: [], + reason: expect.stringContaining("incomplete"), + }, + }, + }, + }); + }, + 60_000, +); + +it("keeps a count-mismatched tree unknown without a completeness marker", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.total_nodes += 1; + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + dimensions: { + accessibility: { status: "unknown", total_changes: 0, changes: [] }, + }, + }, + }); +}); + +it("compares known structure when only text was not approved", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.text_capture = { + status: "not_approved", + retained_bytes: 0, + excluded_fields: 2, + }; + for (const node of after.accessibility.nodes) { + node.name = null; + node.description = null; + } + after.completeness = classifyBrowserCompleteness({ + policyFilteredSections: new Set(["accessibility"]), + attachLimitedSections: new Set(), + truncatedSections: new Set(), + unavailableSections: new Set(), + excluded: [{ section: "accessibility", reason: "not_approved", count: 2 }], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }); + after.accessibility.nodes[1]!.role = "menuitem"; + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + dimensions: { + accessibility: { + status: "changed", + total_changes: 1, + changes: [{ identity: "accessibility_tree", change: "modified" }], + }, + }, + }, + }); +}); + +it("keeps independent resource changes visible beside an unknown tree", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.nodes.pop(); + after.accessibility.total_nodes = after.accessibility.nodes.length; + setCoverage(after, "truncated"); + after.resources.push({ + resource_key: `res_${"a".repeat(64)}`, + url: "https://app.example.test/new.css", + origin: "https://app.example.test", + type: "Stylesheet", + mime_type: "text/css", + content_size: 10, + }); + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + overall_status: "changed", + dimensions: { + accessibility: { status: "unknown", total_changes: 0 }, + resources: { status: "changed", total_changes: 1 }, + }, + }, + }); +});