test(clients): verify native Gemini CLI integration (#1541)

docs/testing.md

@@ -105,6 +105,19 @@ home; this does not prove their default home-directory discovery.
 Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of
 a named sticky profile with `HERMES_HOME` still pointing to its root.
 
+`verify:gemini-client` requires an installed Gemini CLI (verified with
+`@google/gemini-cli@0.63.0`); select it with `REA_VERIFY_GEMINI_COMMAND`.
+The optional POSIX lane uses native `GEMINI_CLI_HOME` discovery in an isolated
+Git project, checks setup plans, backups and idempotence, activates the installed
+personal skill, validates all forwarded input JSON Schemas against their declared
+dialect, forwards the complete REA catalog and checks full JavaScript
+Evidence for a Unicode path. Use `-- chat` for ordinary chat with REA enabled,
+or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed REA package.
+The local fixture exercises the real Gemini API adapter, including its
+`untrusted_context` tool-result envelope. Token counts are synthetic; live
+Google API and native Windows compatibility remain unverified. The lane disables
+the client's memory-based relaunch to preserve the caller's Node heap budget.
+
 `verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`;
 verified with `@deepseek-ai/dsh@0.2.0-rc.2`) and Git. Set
 `REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane

package.json

@@ -180,6 +180,7 @@
     "verify:package": "npm run build:cached && node scripts/verify-package.mjs",
     "verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs",
     "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
+    "verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
     "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",
     "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes",
     "verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs",

scripts/verify-gemini-client.mjs

@@ -0,0 +1,378 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import Ajv2020 from "ajv/dist/2020.js";
+import {
+  TOOL_CONTRACTS,
+  toolContract,
+} from "../dist/contracts/toolContracts.js";
+import { exec } from "./lib/verify-package-core.mjs";
+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs";
+
+assert(
+  process.platform !== "win32",
+  "Native Gemini verifier requires POSIX; Windows is unverified",
+);
+const repo = fileURLToPath(new URL("..", import.meta.url));
+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
+const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini";
+const mode = process.argv[2] ?? "call";
+assert(
+  ["chat", "call"].includes(mode),
+  "Usage: verify-gemini-client.mjs [chat|call]",
+);
+const verifier = createVerifierRun();
+const lab = await mkdtemp(join(tmpdir(), "rea-gemini-client-"));
+const account = join(lab, "account");
+const profile = join(lab, "selected profile");
+const workspace = join(lab, "workspace");
+const configPath = join(profile, ".gemini", "settings.json");
+const skillPath = join(
+  profile,
+  ".agents",
+  "skills",
+  "reverse-engineer-anything",
+  "SKILL.md",
+);
+await Promise.all([
+  mkdir(account),
+  mkdir(workspace),
+  mkdir(join(profile, ".gemini"), { recursive: true }),
+]);
+await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
+await writeFile(join(lab, "system-settings.json"), "{}\n");
+await writeFile(join(lab, "system-defaults.json"), "{}\n");
+const environment = {
+  PATH: process.env.PATH ?? "",
+  USERPROFILE: account,
+  GEMINI_CLI_HOME: profile,
+  GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"),
+  GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"),
+  // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing.
+  GEMINI_CLI_NO_RELAUNCH: "1",
+  GEMINI_CLI_TRUST_WORKSPACE: "true",
+  REA_PROCESS_RUN_ID: verifier.run_id,
+};
+let clientVersion;
+try {
+  clientVersion = (
+    await exec(command, ["--version"], { env: environment, timeout: 30_000 })
+  ).stdout.trim();
+} catch (cause) {
+  throw new Error(
+    `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND. ${String(cause)}`,
+  );
+}
+const original = JSON.stringify(
+  {
+    security: {
+      auth: { selectedType: "gemini-api-key" },
+      folderTrust: { enabled: false },
+    },
+    telemetry: { enabled: false },
+    advanced: { autoConfigureMemory: false },
+    model: { name: "gemini-2.5-flash", maxSessionTurns: 8 },
+    mcpServers: { other: { command: "unrelated-server" } },
+  },
+  null,
+  2,
+);
+await writeFile(configPath, original);
+for (const args of [
+  ["setup", "--client", "gemini_cli", "--dry-run"],
+  ["setup", "--client", "gemini_cli", "--yes"],
+  ["doctor", "--client", "gemini_cli", "--skill"],
+]) {
+  const result = await exec(
+    process.execPath,
+    [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
+    { env: environment, cwd: workspace, timeout: 60_000 },
+  );
+  const parsed = JSON.parse(result.stdout);
+  if (args[0] === "doctor") assert.equal(parsed.healthy, true);
+  else {
+    assert.equal(
+      parsed.status,
+      args.includes("--dry-run") ? "planned" : "ready",
+    );
+    if (args.includes("--dry-run")) {
+      assert(parsed.plannedActions.some((a) => a.target === configPath));
+      assert(
+        parsed.plannedActions.some((a) =>
+          a.target.startsWith(join(profile, ".agents", "skills") + "/"),
+        ),
+      );
+      assert(
+        parsed.plannedActions.every((a) => a.target.startsWith(profile + "/")),
+      );
+    }
+  }
+  await writeFile(
+    join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
+    result.stdout,
+  );
+}
+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+const configured = await readFile(configPath, "utf8");
+assert.deepEqual(JSON.parse(configured).mcpServers.other, {
+  command: "unrelated-server",
+});
+const repeated = await exec(
+  process.execPath,
+  [
+    join(runtimeRoot, "scripts/rea.mjs"),
+    "setup",
+    "--client",
+    "gemini_cli",
+    "--yes",
+    "--json",
+  ],
+  { env: environment, cwd: workspace, timeout: 60_000 },
+);
+assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
+assert.equal(await readFile(configPath, "utf8"), configured);
+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+assert(
+  (await readFile(skillPath, "utf8")).includes(
+    "name: reverse-engineer-anything",
+  ),
+);
+const target = join(lab, "fixture α测试");
+await mkdir(target);
+await writeFile(
+  join(target, "app.js"),
+  'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
+);
+let evidenceSeen = false;
+let skillLoaded = false;
+let skillRequested = false;
+let finalSeen = false;
+let catalogSize = 0;
+let schemasChecked = 0;
+let artifactDigest;
+const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
+const extractEvidence = (value) => {
+  if (typeof value === "string") {
+    if (
+      value.startsWith("<untrusted_context>\n") &&
+      value.endsWith("\n</untrusted_context>")
+    ) {
+      return extractEvidence(
+        value.slice(
+          "<untrusted_context>\n".length,
+          -"\n</untrusted_context>".length,
+        ),
+      );
+    }
+    try {
+      return extractEvidence(JSON.parse(value));
+    } catch {
+      return undefined;
+    }
+  }
+  if (value && typeof value === "object") {
+    if (
+      typeof value.evidence_id === "string" &&
+      value.normalized_result?.graph?.nodes?.length > 0
+    )
+      return value;
+    for (const nested of Object.values(value)) {
+      const evidence = extractEvidence(nested);
+      if (evidence) return evidence;
+    }
+  }
+  return undefined;
+};
+const fixture = await createGeminiModelFixture({
+  directory: lab,
+  onRequest: async ({ body, tools }) => {
+    const analysis = tools.find((t) =>
+      t.name.endsWith("analyze_javascript_application"),
+    );
+    assert(analysis, "Gemini must expose the REA analysis tool");
+    const prefix = analysis.name.slice(
+      0,
+      -"analyze_javascript_application".length,
+    );
+    const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort();
+    assert.deepEqual(
+      tools
+        .filter((t) => expected.includes(t.name))
+        .map((t) => t.name)
+        .sort(),
+      expected,
+      "Gemini must forward the complete REA tool catalog",
+    );
+    catalogSize = expected.length;
+    if (schemasChecked === 0) {
+      for (const tool of tools.filter((t) => expected.includes(t.name))) {
+        assert(
+          tool.parametersJsonSchema,
+          `${tool.name}: missing advertised JSON Schema`,
+        );
+        assert(
+          schemaValidator.validateSchema(tool.parametersJsonSchema),
+          `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`,
+        );
+        schemasChecked++;
+      }
+    }
+    const responses = body.contents
+      .flatMap((c) => c.parts ?? [])
+      .filter((p) => p.functionResponse)
+      .map((p) => p.functionResponse);
+    skillLoaded ||= responses.some(
+      (r) =>
+        r.name === "activate_skill" &&
+        r.response?.output?.includes(
+          `<activated_skill name="reverse-engineer-anything">`,
+        ) &&
+        r.response.output.includes(
+          join(profile, ".agents", "skills", "reverse-engineer-anything"),
+        ),
+    );
+    const observed = extractEvidence(responses);
+    if (observed) {
+      const evidence = toolContract(
+        "analyze_javascript_application",
+      ).outputSchema.parse(observed);
+      assert.equal(evidence.subject.local_path, target);
+      const shape = evidence.normalized_result.graph.nodes
+        .flatMap((n) => n.observations)
+        .map((o) => o.properties)
+        .find(
+          (p) =>
+            p.semantic_role === "export-return-shapes" &&
+            p.exported_name === "result",
+        );
+      assert(shape?.return_shape_coverage?.projection_complete);
+      const fields = shape.static_return_shapes.flatMap((s) => s.fields);
+      assert(
+        fields.some(
+          (f) =>
+            f.path === "/clientCompatibility" &&
+            f.state === "literal" &&
+            f.value === "REA_CLIENT_FIXTURE",
+        ),
+      );
+      assert(
+        fields.some(
+          (f) => f.path === "/count" && f.state === "literal" && f.value === 7,
+        ),
+      );
+      const bytes = JSON.stringify(evidence);
+      artifactDigest = createHash("sha256").update(bytes).digest("hex");
+      await writeFile(join(lab, "gemini-evidence.json"), bytes);
+      evidenceSeen = true;
+    }
+    if (mode === "chat" || evidenceSeen) {
+      finalSeen = true;
+      return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }];
+    }
+    if (!skillRequested) {
+      const skill = tools.find((t) => t.name === "activate_skill");
+      assert(skill, "Gemini must expose its native skill activation tool");
+      skillRequested = true;
+      return [
+        {
+          functionCall: {
+            name: skill.name,
+            args: { name: "reverse-engineer-anything" },
+          },
+        },
+      ];
+    }
+    assert(skillLoaded, "Gemini must load the installed personal REA skill");
+    return [
+      {
+        functionCall: {
+          name: analysis.name,
+          args: { input_path: target, format: "directory" },
+        },
+      },
+    ];
+  },
+});
+let failure;
+try {
+  const response = await exec(
+    command,
+    [
+      "--model",
+      "gemini-2.5-flash",
+      "--approval-mode",
+      mode === "chat" ? "default" : "yolo",
+      "--allowed-mcp-server-names",
+      "rea",
+      "--output-format",
+      "stream-json",
+      "-p",
+      mode === "chat"
+        ? "Say hello briefly."
+        : `Use REA to analyze the JavaScript directory ${target}. Report the result.`,
+    ],
+    {
+      env: {
+        ...environment,
+        GEMINI_API_KEY: "local-fixture-key",
+        GOOGLE_GEMINI_BASE_URL: fixture.baseUrl,
+      },
+      cwd: workspace,
+      timeout: 120_000,
+      maxBuffer: 10 * 1024 * 1024,
+    },
+  );
+  await writeFile(join(lab, `${mode}.stdout`), response.stdout);
+  await writeFile(join(lab, `${mode}.stderr`), response.stderr);
+  const events = response.stdout
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+  assert.equal(events.findLast((e) => e.type === "result")?.status, "success");
+  assert.equal(fixture.failure, undefined);
+  assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
+  if (mode === "call") assert(evidenceSeen && skillLoaded);
+} catch (cause) {
+  failure = cause;
+  if (typeof cause?.stdout === "string")
+    await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
+  if (typeof cause?.stderr === "string")
+    await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
+} finally {
+  await fixture.close();
+  const lineage = await completeVerifierRun(verifier);
+  const receipt = {
+    client: "Gemini CLI",
+    clientVersion,
+    mode,
+    passed: failure === undefined,
+    artifacts: lab,
+    runtimeRoot,
+    host: {
+      platform: process.platform,
+      architecture: process.arch,
+      node: process.versions.node,
+    },
+    modelFixture:
+      "loopback native Gemini API adapter; token counts synthetic; no live Google API claim",
+    skillDiscovery:
+      "native GEMINI_CLI_HOME personal shared directory; isolated Git project",
+    catalogSize,
+    schemasChecked,
+    evidenceSeen,
+    skillLoaded,
+    artifactDigest,
+    finalSeen,
+    requests: fixture.requests,
+    probes: fixture.probes,
+    fixtureFailure: fixture.failure,
+    verifier: lineage,
+  };
+  await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
+  console.log(JSON.stringify(receipt));
+}
+if (failure !== undefined) throw failure;

scripts/verify/clients/gemini-model-fixture.mjs

@@ -0,0 +1,104 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Exercise the native Gemini API adapter against a local model, without cloud claims. */
+export const createGeminiModelFixture = async ({ directory, onRequest }) => {
+  const requests = [];
+  const probes = [];
+  let incoming = 0;
+  let failure;
+  const server = createServer(async (request, response) => {
+    try {
+      const chunks = [];
+      let bytes = 0;
+      for await (const chunk of request) {
+        bytes += chunk.length;
+        assert(
+          bytes <= 10 * 1024 * 1024,
+          "Model fixture request exceeds 10 MiB budget",
+        );
+        chunks.push(chunk);
+      }
+      const raw = Buffer.concat(chunks).toString("utf8");
+      const body = raw.length === 0 ? null : JSON.parse(raw);
+      await writeFile(
+        join(directory, `gemini-incoming-${++incoming}.json`),
+        JSON.stringify({ method: request.method, path: request.url, body }),
+      );
+      const path = new URL(request.url, "http://localhost").pathname;
+      if (path.endsWith(":countTokens")) {
+        probes.push({ path, kind: "synthetic-token-count" });
+        response.setHeader("Content-Type", "application/json");
+        response.end(JSON.stringify({ totalTokens: 100 }));
+        return;
+      }
+      if (
+        !/\/models\/[^/]+:(?:streamGenerateContent|generateContent)$/u.test(
+          path,
+        )
+      ) {
+        probes.push({ path, status: 404 });
+        response.statusCode = 404;
+        response.end(
+          JSON.stringify({
+            error: { message: "Unsupported fixture endpoint" },
+          }),
+        );
+        return;
+      }
+      assert.equal(request.method, "POST");
+      assert(body && typeof body === "object" && !Array.isArray(body));
+      assert(Array.isArray(body.contents));
+      const tools = (body.tools ?? []).flatMap(
+        (t) => t.functionDeclarations ?? [],
+      );
+      requests.push({
+        path,
+        tools: tools.map((t) => t.name),
+        contentRoles: body.contents.map((c) => c.role),
+      });
+      const parts = await onRequest({
+        body,
+        tools,
+        requestIndex: requests.length,
+      });
+      const result = {
+        candidates: [
+          { index: 0, content: { role: "model", parts }, finishReason: "STOP" },
+        ],
+        usageMetadata: {
+          promptTokenCount: 100,
+          candidatesTokenCount: 20,
+          totalTokenCount: 120,
+        },
+        modelVersion: "local-rea-fixture",
+      };
+      if (path.endsWith(":streamGenerateContent")) {
+        response.setHeader("Content-Type", "text/event-stream");
+        response.end(`data: ${JSON.stringify(result)}\n\n`);
+      } else {
+        response.setHeader("Content-Type", "application/json");
+        response.end(JSON.stringify(result));
+      }
+    } catch (cause) {
+      failure = String(cause);
+      response.statusCode = 400;
+      response.setHeader("Content-Type", "application/json");
+      response.end(JSON.stringify({ error: { message: failure, code: 400 } }));
+    }
+  });
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const address = server.address();
+  assert(address !== null && typeof address !== "string");
+  return {
+    baseUrl: `http://127.0.0.1:${address.port}`,
+    requests,
+    probes,
+    get failure() {
+      return failure;
+    },
+    close: () => new Promise((resolve) => server.close(resolve)),
+  };
+};