fix(evaluation): canonicalize Unicode argument keys (#980)

src/domain/unicodeCodePointOrder.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+
+import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
+
+describe("Unicode code-point ordering", () => {
+  it("orders supplementary characters by scalar value rather than UTF-16 units", () => {
+    expect(compareUnicodeCodePoints("\uE000", "\u{10000}")).toBeLessThan(0);
+    expect(compareUnicodeCodePoints("\u{10000}", "\uE000")).toBeGreaterThan(0);
+  });
+
+  it.each([
+    ["", "a", -1],
+    ["same", "same", 0],
+    ["a", "aa", -1],
+    ["e\u0301", "\u00e9", -132],
+    ["\uD800", "\uD801", -1],
+  ])("orders %j and %j by scalar value", (left, right, expectedSign) => {
+    expect(Math.sign(compareUnicodeCodePoints(left, right))).toBe(
+      Math.sign(expectedSign),
+    );
+  });
+});

src/domain/unicodeCodePointOrder.ts

@@ -3,12 +3,16 @@ export const compareUnicodeCodePoints = (
   left: string,
   right: string,
 ): number => {
-  const leftPoints = [...left].map((value) => value.codePointAt(0) ?? 0);
-  const rightPoints = [...right].map((value) => value.codePointAt(0) ?? 0);
-  const length = Math.min(leftPoints.length, rightPoints.length);
-  for (let index = 0; index < length; index += 1) {
-    const difference = (leftPoints[index] ?? 0) - (rightPoints[index] ?? 0);
+  let leftIndex = 0;
+  let rightIndex = 0;
+  while (leftIndex < left.length && rightIndex < right.length) {
+    const leftPoint = left.codePointAt(leftIndex) ?? 0;
+    const rightPoint = right.codePointAt(rightIndex) ?? 0;
+    const difference = leftPoint - rightPoint;
     if (difference !== 0) return difference;
+    leftIndex += leftPoint > 0xffff ? 2 : 1;
+    rightIndex += rightPoint > 0xffff ? 2 : 1;
   }
-  return leftPoints.length - rightPoints.length;
+  if (leftIndex === left.length && rightIndex === right.length) return 0;
+  return leftIndex === left.length ? -1 : 1;
 };

src/evaluation/CodexAgentEval.ts

@@ -1,3 +1,5 @@
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+
 /** One REA MCP invocation observed in a Codex JSONL transcript. */
 interface CodexMcpCall {
   readonly id: string | null;
@@ -47,7 +49,7 @@ const canonicalValue = (value: unknown): unknown => {
   if (object === undefined) return value;
   return Object.fromEntries(
     Object.entries(object)
-      .sort(([left], [right]) => left.localeCompare(right))
+      .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
       .map(([key, child]) => [key, canonicalValue(child)]),
   );
 };

tests/evaluation/model-evals/codexAgentEval.test.ts

@@ -100,6 +100,29 @@ describe("Codex agent release evaluation", () => {
         .repeatedCallCount,
     ).toBe(1);
   });
+
+  it("counts identical arguments with reversed Unicode key insertion order", () => {
+    const call = (id: string, arguments_: Record<string, number>) => ({
+      type: "item.completed",
+      item: {
+        id,
+        type: "mcp_tool_call",
+        server: "rea",
+        tool: "inspect_artifact",
+        arguments: arguments_,
+      },
+    });
+
+    expect(
+      evaluateCodexEvents(
+        [
+          call("item-1", { "e\u0301": 1, "\u00e9": 2 }),
+          call("item-2", { "\u00e9": 2, "e\u0301": 1 }),
+        ],
+        "inspect_artifact",
+      ).repeatedCallCount,
+    ).toBe(1);
+  });
 });
 
 describe("Codex agent completion evaluation", () => {