fix(browser): compare accessibility only with observed coverage

Return unknown before fingerprinting incomplete accessibility trees. Keep
known structural changes comparable when only text permission is absent,
and preserve independent changes in other dimensions.

Validated through seven CLI cases and existing network comparison semantics.
Refs #3385, #3824.

Co-authored-by: Liu Jie <maxazure@gmail.com>

src/domain/webCaptureDiff.ts

@@ -114,29 +114,30 @@ const accessibilityDimension = (
   const afterAccess = accessibilityComparable(after);
   const textComparable = beforeAccess.text && afterAccess.text;
   const nodesComparable = beforeAccess.nodes && afterAccess.nodes;
+  const reason =
+    "Accessibility tree or text capture was incomplete in at least one observation.";
+  if (!nodesComparable)
+    return {
+      status: "unknown",
+      total_changes: 0,
+      changes: [],
+      reason,
+    };
   return compareDimension(
     singleton(
       "accessibility_tree",
       digestCanonicalValue(
-        accessibilityProjection(
-          before.accessibility,
-          textComparable,
-          nodesComparable,
-        ),
+        accessibilityProjection(before.accessibility, textComparable),
       ),
     ),
     singleton(
       "accessibility_tree",
       digestCanonicalValue(
-        accessibilityProjection(
-          after.accessibility,
-          textComparable,
-          nodesComparable,
-        ),
+        accessibilityProjection(after.accessibility, textComparable),
       ),
     ),
     beforeAccess.complete && afterAccess.complete,
-    "Accessibility tree or text capture was incomplete in at least one observation.",
+    reason,
   );
 };
 
@@ -850,20 +851,32 @@ const accessibilityComparable = (
   const text =
     inspection.accessibility.text_capture.status === "included" &&
     inspection.accessibility.text_capture.excluded_fields === 0;
+  const { completeness } = inspection;
+  const exclusions = completeness.excluded.filter(
+    ({ section }) => section === "accessibility",
+  );
+  const textOnlyPolicyFilter =
+    inspection.accessibility.text_capture.status === "not_approved" &&
+    exclusions.length > 0 &&
+    exclusions.every(({ reason }) => reason === "not_approved");
   const nodes =
+    [
+      completeness.truncated_sections,
+      completeness.unavailable_sections,
+      completeness.attach_limited_sections,
+    ].every((sections) => !sections.includes("accessibility")) &&
+    (!completeness.policy_filtered_sections.includes("accessibility") ||
+      textOnlyPolicyFilter) &&
     inspection.accessibility.total_nodes ===
-    inspection.accessibility.nodes.length;
-  const complete =
-    text && nodes && sectionsComplete(inspection, ["accessibility"]);
+      inspection.accessibility.nodes.length;
+  const complete = text && nodes;
   return { text, nodes, complete };
 };
 
 const accessibilityProjection = (
   accessibility: WebPageInspection["accessibility"],
   includeText: boolean,
-  includeNodes: boolean,
 ): unknown => {
-  if (!includeNodes) return { total_nodes: accessibility.total_nodes };
   const nodeIndexes = new Map(
     accessibility.nodes.map((node, index) => [node.node_id, index]),
   );

tests/boundary/cli/accessibilityCaptureComparison.test.ts

@@ -0,0 +1,218 @@
+import { expect, it } from "vitest";
+
+import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js";
+import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";
+import { runCliJson } from "../../fixtures/cliJsonProcess.js";
+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";
+
+type Capture = WebPageInspection;
+type Coverage =
+  | "truncated"
+  | "unavailable"
+  | "attach_limited"
+  | "policy_filtered";
+
+const accessibleCapture = (): Capture => {
+  const capture = passiveNetworkCapture();
+  capture.accessibility = {
+    total_nodes: 2,
+    text_capture: {
+      status: "included",
+      retained_bytes: 10,
+      excluded_fields: 0,
+    },
+    nodes: [
+      {
+        node_id: "root",
+        parent_id: null,
+        role: "RootWebArea",
+        name: "Page",
+        description: null,
+        ignored: false,
+        states: [],
+      },
+      {
+        node_id: "button",
+        parent_id: "root",
+        role: "button",
+        name: "Submit",
+        description: null,
+        ignored: false,
+        states: [],
+      },
+    ],
+  };
+  return capture;
+};
+
+const setCoverage = (capture: Capture, coverage: Coverage): void => {
+  capture.completeness = classifyBrowserCompleteness({
+    policyFilteredSections: new Set(
+      coverage === "policy_filtered" ? ["accessibility"] : [],
+    ),
+    attachLimitedSections: new Set(
+      coverage === "attach_limited" ? ["accessibility"] : [],
+    ),
+    truncatedSections: new Set(
+      coverage === "truncated" ? ["accessibility"] : [],
+    ),
+    unavailableSections: new Set(
+      coverage === "unavailable" ? ["accessibility"] : [],
+    ),
+    excluded:
+      coverage === "policy_filtered"
+        ? [{ section: "accessibility", reason: "disallowed_origin", count: 1 }]
+        : coverage === "unavailable"
+          ? [
+              {
+                section: "accessibility",
+                reason: "invalid_protocol_value",
+                count: null,
+              },
+            ]
+          : [],
+    droppedEvents: {
+      scripts: 0,
+      network_requests: 0,
+      console_events: 0,
+      websocket_connections: 0,
+      websocket_frames: 0,
+      webmcp_tools: 0,
+      timeline_events: 0,
+    },
+  });
+};
+
+const compare = async (before: Capture, after: Capture) =>
+  await runCliJson(
+    [
+      "compare-web-captures",
+      JSON.stringify({ inspection: before }),
+      JSON.stringify({ inspection: after }),
+      "--json",
+    ],
+    process.env,
+  );
+
+it.each([
+  "truncated",
+  "unavailable",
+  "attach_limited",
+  "policy_filtered",
+] as const)(
+  "does not infer an accessibility change from a %s tree",
+  async (coverage) => {
+    const before = accessibleCapture();
+    const after = structuredClone(before);
+    after.accessibility.nodes.pop();
+    after.accessibility.total_nodes = after.accessibility.nodes.length;
+    after.accessibility.text_capture.retained_bytes = 4;
+    setCoverage(after, coverage);
+
+    const result = await compare(before, after);
+
+    expect(result).toMatchObject({
+      operation: "compare_web_captures",
+      normalized_result: {
+        dimensions: {
+          accessibility: {
+            status: "unknown",
+            total_changes: 0,
+            changes: [],
+            reason: expect.stringContaining("incomplete"),
+          },
+        },
+      },
+    });
+  },
+  60_000,
+);
+
+it("keeps a count-mismatched tree unknown without a completeness marker", async () => {
+  const before = accessibleCapture();
+  const after = structuredClone(before);
+  after.accessibility.total_nodes += 1;
+
+  const result = await compare(before, after);
+
+  expect(result).toMatchObject({
+    normalized_result: {
+      dimensions: {
+        accessibility: { status: "unknown", total_changes: 0, changes: [] },
+      },
+    },
+  });
+});
+
+it("compares known structure when only text was not approved", async () => {
+  const before = accessibleCapture();
+  const after = structuredClone(before);
+  after.accessibility.text_capture = {
+    status: "not_approved",
+    retained_bytes: 0,
+    excluded_fields: 2,
+  };
+  for (const node of after.accessibility.nodes) {
+    node.name = null;
+    node.description = null;
+  }
+  after.completeness = classifyBrowserCompleteness({
+    policyFilteredSections: new Set(["accessibility"]),
+    attachLimitedSections: new Set(),
+    truncatedSections: new Set(),
+    unavailableSections: new Set(),
+    excluded: [{ section: "accessibility", reason: "not_approved", count: 2 }],
+    droppedEvents: {
+      scripts: 0,
+      network_requests: 0,
+      console_events: 0,
+      websocket_connections: 0,
+      websocket_frames: 0,
+      webmcp_tools: 0,
+      timeline_events: 0,
+    },
+  });
+  after.accessibility.nodes[1]!.role = "menuitem";
+
+  const result = await compare(before, after);
+
+  expect(result).toMatchObject({
+    normalized_result: {
+      dimensions: {
+        accessibility: {
+          status: "changed",
+          total_changes: 1,
+          changes: [{ identity: "accessibility_tree", change: "modified" }],
+        },
+      },
+    },
+  });
+});
+
+it("keeps independent resource changes visible beside an unknown tree", async () => {
+  const before = accessibleCapture();
+  const after = structuredClone(before);
+  after.accessibility.nodes.pop();
+  after.accessibility.total_nodes = after.accessibility.nodes.length;
+  setCoverage(after, "truncated");
+  after.resources.push({
+    resource_key: `res_${"a".repeat(64)}`,
+    url: "https://app.example.test/new.css",
+    origin: "https://app.example.test",
+    type: "Stylesheet",
+    mime_type: "text/css",
+    content_size: 10,
+  });
+
+  const result = await compare(before, after);
+
+  expect(result).toMatchObject({
+    normalized_result: {
+      overall_status: "changed",
+      dimensions: {
+        accessibility: { status: "unknown", total_changes: 0 },
+        resources: { status: "changed", total_changes: 1 },
+      },
+    },
+  });
+});