From 0a4caf4a0a1355b6a121fc88886b22e55843ac76 Mon Sep 17 00:00:00 2001
From: morluto <76467478+morluto@users.noreply.github.com>
Date: Wed, 7 Oct 2026 23:19:49 +0800
Subject: [PATCH] fix(evaluation): canonicalize Unicode argument keys (#980)

---
 src/domain/unicodeCodePointOrder.test.ts      | 22 ++++++++++++++++++
 src/domain/unicodeCodePointOrder.ts           | 16 ++++++++-----
 src/evaluation/CodexAgentEval.ts              |  4 +++-
 .../model-evals/codexAgentEval.test.ts        | 23 +++++++++++++++++++
 4 files changed, 58 insertions(+), 7 deletions(-)
 create mode 100644 src/domain/unicodeCodePointOrder.test.ts

diff --git a/src/domain/unicodeCodePointOrder.test.ts b/src/domain/unicodeCodePointOrder.test.ts
new file mode 100644
index 0000000000..9ac4acf682
--- /dev/null
+++ b/src/domain/unicodeCodePointOrder.test.ts
@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+
+import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
+
+describe("Unicode code-point ordering", () => {
+  it("orders supplementary characters by scalar value rather than UTF-16 units", () => {
+    expect(compareUnicodeCodePoints("\uE000", "\u{10000}")).toBeLessThan(0);
+    expect(compareUnicodeCodePoints("\u{10000}", "\uE000")).toBeGreaterThan(0);
+  });
+
+  it.each([
+    ["", "a", -1],
+    ["same", "same", 0],
+    ["a", "aa", -1],
+    ["e\u0301", "\u00e9", -132],
+    ["\uD800", "\uD801", -1],
+  ])("orders %j and %j by scalar value", (left, right, expectedSign) => {
+    expect(Math.sign(compareUnicodeCodePoints(left, right))).toBe(
+      Math.sign(expectedSign),
+    );
+  });
+});
diff --git a/src/domain/unicodeCodePointOrder.ts b/src/domain/unicodeCodePointOrder.ts
index 42acf2b068..163039364b 100644
--- a/src/domain/unicodeCodePointOrder.ts
+++ b/src/domain/unicodeCodePointOrder.ts
@@ -3,12 +3,16 @@ export const compareUnicodeCodePoints = (
   left: string,
   right: string,
 ): number => {
-  const leftPoints = [...left].map((value) => value.codePointAt(0) ?? 0);
-  const rightPoints = [...right].map((value) => value.codePointAt(0) ?? 0);
-  const length = Math.min(leftPoints.length, rightPoints.length);
-  for (let index = 0; index < length; index += 1) {
-    const difference = (leftPoints[index] ?? 0) - (rightPoints[index] ?? 0);
+  let leftIndex = 0;
+  let rightIndex = 0;
+  while (leftIndex < left.length && rightIndex < right.length) {
+    const leftPoint = left.codePointAt(leftIndex) ?? 0;
+    const rightPoint = right.codePointAt(rightIndex) ?? 0;
+    const difference = leftPoint - rightPoint;
     if (difference !== 0) return difference;
+    leftIndex += leftPoint > 0xffff ? 2 : 1;
+    rightIndex += rightPoint > 0xffff ? 2 : 1;
   }
-  return leftPoints.length - rightPoints.length;
+  if (leftIndex === left.length && rightIndex === right.length) return 0;
+  return leftIndex === left.length ? -1 : 1;
 };
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts
index e5fc0c7d7c..e193761b32 100644
--- a/src/evaluation/CodexAgentEval.ts
+++ b/src/evaluation/CodexAgentEval.ts
@@ -1,3 +1,5 @@
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+
 /** One REA MCP invocation observed in a Codex JSONL transcript. */
 interface CodexMcpCall {
   readonly id: string | null;
@@ -47,7 +49,7 @@ const canonicalValue = (value: unknown): unknown => {
   if (object === undefined) return value;
   return Object.fromEntries(
     Object.entries(object)
-      .sort(([left], [right]) => left.localeCompare(right))
+      .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
       .map(([key, child]) => [key, canonicalValue(child)]),
   );
 };
diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts
index c068878f60..82100cea6b 100644
--- a/tests/evaluation/model-evals/codexAgentEval.test.ts
+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts
@@ -100,6 +100,29 @@ describe("Codex agent release evaluation", () => {
         .repeatedCallCount,
     ).toBe(1);
   });
+
+  it("counts identical arguments with reversed Unicode key insertion order", () => {
+    const call = (id: string, arguments_: Record<string, number>) => ({
+      type: "item.completed",
+      item: {
+        id,
+        type: "mcp_tool_call",
+        server: "rea",
+        tool: "inspect_artifact",
+        arguments: arguments_,
+      },
+    });
+
+    expect(
+      evaluateCodexEvents(
+        [
+          call("item-1", { "e\u0301": 1, "\u00e9": 2 }),
+          call("item-2", { "\u00e9": 2, "e\u0301": 1 }),
+        ],
+        "inspect_artifact",
+      ).repeatedCallCount,
+    ).toBe(1);
+  });
 });
 
 describe("Codex agent completion evaluation", () => {