From 0123d5976828bfe621bd51daa4cf39fee48a1349 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:17:51 +0800 Subject: [PATCH] fix(browser): compare accessibility only with observed coverage Return unknown before fingerprinting incomplete accessibility trees. Keep known structural changes comparable when only text permission is absent, and preserve independent changes in other dimensions. Validated through seven CLI cases and existing network comparison semantics. Refs #3385, #3824. Co-authored-by: Liu Jie --- src/domain/webCaptureDiff.ts | 45 ++-- .../accessibilityCaptureComparison.test.ts | 218 ++++++++++++++++++ 2 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 tests/boundary/cli/accessibilityCaptureComparison.test.ts diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index de24a0fc3d..ee0709ca1f 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -114,29 +114,30 @@ const accessibilityDimension = ( const afterAccess = accessibilityComparable(after); const textComparable = beforeAccess.text && afterAccess.text; const nodesComparable = beforeAccess.nodes && afterAccess.nodes; + const reason = + "Accessibility tree or text capture was incomplete in at least one observation."; + if (!nodesComparable) + return { + status: "unknown", + total_changes: 0, + changes: [], + reason, + }; return compareDimension( singleton( "accessibility_tree", digestCanonicalValue( - accessibilityProjection( - before.accessibility, - textComparable, - nodesComparable, - ), + accessibilityProjection(before.accessibility, textComparable), ), ), singleton( "accessibility_tree", digestCanonicalValue( - accessibilityProjection( - after.accessibility, - textComparable, - nodesComparable, - ), + accessibilityProjection(after.accessibility, textComparable), ), ), beforeAccess.complete && afterAccess.complete, - "Accessibility tree or text capture was incomplete in at least one observation.", + reason, ); }; @@ -850,20 +851,32 @@ const accessibilityComparable = ( const text = inspection.accessibility.text_capture.status === "included" && inspection.accessibility.text_capture.excluded_fields === 0; + const { completeness } = inspection; + const exclusions = completeness.excluded.filter( + ({ section }) => section === "accessibility", + ); + const textOnlyPolicyFilter = + inspection.accessibility.text_capture.status === "not_approved" && + exclusions.length > 0 && + exclusions.every(({ reason }) => reason === "not_approved"); const nodes = + [ + completeness.truncated_sections, + completeness.unavailable_sections, + completeness.attach_limited_sections, + ].every((sections) => !sections.includes("accessibility")) && + (!completeness.policy_filtered_sections.includes("accessibility") || + textOnlyPolicyFilter) && inspection.accessibility.total_nodes === - inspection.accessibility.nodes.length; - const complete = - text && nodes && sectionsComplete(inspection, ["accessibility"]); + inspection.accessibility.nodes.length; + const complete = text && nodes; return { text, nodes, complete }; }; const accessibilityProjection = ( accessibility: WebPageInspection["accessibility"], includeText: boolean, - includeNodes: boolean, ): unknown => { - if (!includeNodes) return { total_nodes: accessibility.total_nodes }; const nodeIndexes = new Map( accessibility.nodes.map((node, index) => [node.node_id, index]), ); diff --git a/tests/boundary/cli/accessibilityCaptureComparison.test.ts b/tests/boundary/cli/accessibilityCaptureComparison.test.ts new file mode 100644 index 0000000000..61dc571ac7 --- /dev/null +++ b/tests/boundary/cli/accessibilityCaptureComparison.test.ts @@ -0,0 +1,218 @@ +import { expect, it } from "vitest"; + +import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js"; +import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js"; +import { runCliJson } from "../../fixtures/cliJsonProcess.js"; +import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js"; + +type Capture = WebPageInspection; +type Coverage = + | "truncated" + | "unavailable" + | "attach_limited" + | "policy_filtered"; + +const accessibleCapture = (): Capture => { + const capture = passiveNetworkCapture(); + capture.accessibility = { + total_nodes: 2, + text_capture: { + status: "included", + retained_bytes: 10, + excluded_fields: 0, + }, + nodes: [ + { + node_id: "root", + parent_id: null, + role: "RootWebArea", + name: "Page", + description: null, + ignored: false, + states: [], + }, + { + node_id: "button", + parent_id: "root", + role: "button", + name: "Submit", + description: null, + ignored: false, + states: [], + }, + ], + }; + return capture; +}; + +const setCoverage = (capture: Capture, coverage: Coverage): void => { + capture.completeness = classifyBrowserCompleteness({ + policyFilteredSections: new Set( + coverage === "policy_filtered" ? ["accessibility"] : [], + ), + attachLimitedSections: new Set( + coverage === "attach_limited" ? ["accessibility"] : [], + ), + truncatedSections: new Set( + coverage === "truncated" ? ["accessibility"] : [], + ), + unavailableSections: new Set( + coverage === "unavailable" ? ["accessibility"] : [], + ), + excluded: + coverage === "policy_filtered" + ? [{ section: "accessibility", reason: "disallowed_origin", count: 1 }] + : coverage === "unavailable" + ? [ + { + section: "accessibility", + reason: "invalid_protocol_value", + count: null, + }, + ] + : [], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }); +}; + +const compare = async (before: Capture, after: Capture) => + await runCliJson( + [ + "compare-web-captures", + JSON.stringify({ inspection: before }), + JSON.stringify({ inspection: after }), + "--json", + ], + process.env, + ); + +it.each([ + "truncated", + "unavailable", + "attach_limited", + "policy_filtered", +] as const)( + "does not infer an accessibility change from a %s tree", + async (coverage) => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.nodes.pop(); + after.accessibility.total_nodes = after.accessibility.nodes.length; + after.accessibility.text_capture.retained_bytes = 4; + setCoverage(after, coverage); + + const result = await compare(before, after); + + expect(result).toMatchObject({ + operation: "compare_web_captures", + normalized_result: { + dimensions: { + accessibility: { + status: "unknown", + total_changes: 0, + changes: [], + reason: expect.stringContaining("incomplete"), + }, + }, + }, + }); + }, + 60_000, +); + +it("keeps a count-mismatched tree unknown without a completeness marker", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.total_nodes += 1; + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + dimensions: { + accessibility: { status: "unknown", total_changes: 0, changes: [] }, + }, + }, + }); +}); + +it("compares known structure when only text was not approved", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.text_capture = { + status: "not_approved", + retained_bytes: 0, + excluded_fields: 2, + }; + for (const node of after.accessibility.nodes) { + node.name = null; + node.description = null; + } + after.completeness = classifyBrowserCompleteness({ + policyFilteredSections: new Set(["accessibility"]), + attachLimitedSections: new Set(), + truncatedSections: new Set(), + unavailableSections: new Set(), + excluded: [{ section: "accessibility", reason: "not_approved", count: 2 }], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }); + after.accessibility.nodes[1]!.role = "menuitem"; + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + dimensions: { + accessibility: { + status: "changed", + total_changes: 1, + changes: [{ identity: "accessibility_tree", change: "modified" }], + }, + }, + }, + }); +}); + +it("keeps independent resource changes visible beside an unknown tree", async () => { + const before = accessibleCapture(); + const after = structuredClone(before); + after.accessibility.nodes.pop(); + after.accessibility.total_nodes = after.accessibility.nodes.length; + setCoverage(after, "truncated"); + after.resources.push({ + resource_key: `res_${"a".repeat(64)}`, + url: "https://app.example.test/new.css", + origin: "https://app.example.test", + type: "Stylesheet", + mime_type: "text/css", + content_size: 10, + }); + + const result = await compare(before, after); + + expect(result).toMatchObject({ + normalized_result: { + overall_status: "changed", + dimensions: { + accessibility: { status: "unknown", total_changes: 0 }, + resources: { status: "changed", total_changes: 1 }, + }, + }, + }); +});