fix(sqlite): distinguish query capacity and unavailable row sources

Use one bound JSON selection set for inventory metadata, keeping raw-name byte
comparison and filtering inside SQLite while preserving caller projection order
and repetition. Large requests reach native result-column capacity attribution.

Separate unavailable row sources from generated-expression failures. Preserve
SQLite's reason and selected table/path, offer schema-only recovery, and avoid
claiming corruption or recommending unreadable columns.

Validated UTF-8/UTF-16 identity, native inspection/worker tests and CLI/MCP failure
and recovery workflows. The large CLI selection uses the existing framed CLI
fixture to avoid host argv limits. Source bytes and identity remain unchanged.

Fixes #6855
Fixes #6604

docs/sqlite-databases.md

@@ -72,6 +72,15 @@ each failing selected column with SQLite's message, and recommends only columns
 whose sampled values were read; stored and other generated cells stay readable. Malformed column-name bytes
 cause a format refusal when all columns are requested. In inventory scope, an explicit valid subset can
 still be sampled without decoding unrelated malformed column names.
+
+An unavailable row source, including a `WITHOUT ROWID` primary key requiring an
+application collation, is also refused as `capability_unavailable`. The error
+retains the selected table and SQLite's reason without identifying unproven
+dependencies or diagnosing file corruption. Such a source does not provide
+readable-column recommendations. Omit `table` and select `schema_scope: "database"`
+to read stored declarations without requesting rows; the application's supported
+consistent export may be needed to obtain those values.
+
 Column selection requires a
 table and at least one name; empty-string identifiers are valid. SQLite counts
 every projected name, including a repeated name, against its native

src/sqlite/SqliteDatabaseInspection.ts

@@ -250,12 +250,20 @@ const readColumns = (
   const selection =
     selectedColumns === undefined
       ? ""
-      : ` WHERE CAST(name AS BLOB) IN (${selectedColumns.map(() => "CAST(? AS BLOB)").join(", ")})`;
+      : " WHERE CAST(name AS BLOB) IN (SELECT CAST(value AS BLOB) FROM temp.json_each(?))";
+  // One bound data value keeps metadata selection independent of SQL-variable
+  // capacity. SQLite owns conversion into the database encoding, and filters
+  // before projecting unrelated column definitions. Row projections still keep
+  // the caller's order/repetitions and enforce the native result-column limit.
+  const parameters =
+    selectedColumns === undefined
+      ? [name]
+      : [name, JSON.stringify([...new Set(selectedColumns)])];
   for (const raw of db
     .prepare(
       `SELECT cid, CAST(name AS BLOB) AS name, CAST(type AS BLOB) AS type, "notnull", CAST(dflt_value AS BLOB) AS dflt_value, pk, hidden FROM temp.pragma_table_xinfo(?, 'main')${selection}`,
     )
-    .iterate(name, ...(selectedColumns ?? []))) {
+    .iterate(...parameters)) {
     const value = columnRow.parse(raw);
     const column = {
       cid: value.cid,
@@ -599,12 +607,11 @@ const resultColumnLimitFailure = (
 };
 
 /**
- * Attribute an SQL error from selected cells to the virtual generated columns
- * that raise it on this connection. Each column is probed alone over the same
- * sampled rows, so only columns whose probe succeeds are recommended; any
- * other cause keeps its original classification.
+ * Attribute SQL failures to the row source before blaming selected generated
+ * expressions. A generic SQL error alone does not establish file corruption.
+ * Only columns whose sampled probes succeed are recommended for recovery.
  */
-const columnEvaluationFailure = (
+const rowEvaluationFailure = (
   db: DatabaseSync,
   cause: unknown,
   selection: {
@@ -619,6 +626,28 @@ const columnEvaluationFailure = (
   const columnLimit = resultColumnLimitFailure(db, cause, columns.length);
   if (columnLimit !== undefined) return columnLimit;
   if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause;
+  const unavailable = new SqliteInspectionFailure(
+    "row-evaluation",
+    `SQLite cannot prepare or evaluate rows of table ${JSON.stringify(table.name)} on REA's read-only connection: ${cause instanceof Error ? cause.message : String(cause)}`,
+  );
+  const generated = new Set(
+    table.columns
+      .filter((column) => column.hidden === 2)
+      .map((column) => column.name),
+  );
+  if (!columns.some((name) => generated.has(name))) return unavailable;
+  try {
+    // A WITHOUT ROWID source may require an unavailable application collation
+    // even for a constant projection. Do not attribute that failure to every
+    // generated column or recommend other columns from the same unreadable source.
+    db.prepare(`SELECT count(*) FROM (SELECT 1 FROM ${source} LIMIT ?)`).get(
+      limit,
+    );
+  } catch (failure: unknown) {
+    return sqlitePrimaryResultCode(failure) === SQLITE_ERROR
+      ? unavailable
+      : failure;
+  }
   // The aggregate evaluates exactly the sampled cells, as the length pass does.
   const probe = (name: string): { readonly failure: unknown } | undefined => {
     try {
@@ -635,11 +664,6 @@ const columnEvaluationFailure = (
     const failed = probe(name);
     if (failed !== undefined) failures.set(name, failed.failure);
   }
-  const generated = new Set(
-    table.columns
-      .filter((column) => column.hidden === 2)
-      .map((column) => column.name),
-  );
   if (
     ![...failures].some(
       ([name, failure]) =>
@@ -647,7 +671,7 @@ const columnEvaluationFailure = (
         sqlitePrimaryResultCode(failure) === SQLITE_ERROR,
     )
   )
-    return cause;
+    return unavailable;
   const remaining = columns.filter((name) => !failures.has(name));
   const readableStoredColumns = (): string[] => {
     try {
@@ -722,8 +746,7 @@ const readRows = (
     truncated: false,
     order: "unspecified",
   };
-  // SQLite compiles and evaluates only selected cells, so an SQL error here is
-  // attributed to the selected generated expressions rather than the file.
+  // Keep query-source, expression, capacity and malformed-data failures distinct.
   try {
     const lengths = db.prepare(
       `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`,
@@ -741,8 +764,11 @@ const readRows = (
       budget.retain(values);
       rows.values.push(values);
     }
+    rows.truncated =
+      db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !==
+      undefined;
   } catch (cause: unknown) {
-    throw columnEvaluationFailure(db, cause, {
+    throw rowEvaluationFailure(db, cause, {
       table,
       columns,
       source,
@@ -751,9 +777,6 @@ const readRows = (
     });
   }
   rows.returned_rows = rows.values.length;
-  rows.truncated =
-    db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !==
-    undefined;
   return rows;
 };
 

src/sqlite/SqliteDatabaseLimits.ts

@@ -13,6 +13,7 @@ export class SqliteInspectionFailure extends Error {
       | "selection"
       | "column-selection"
       | "column-evaluation"
+      | "row-evaluation"
       | "column-limit"
       | "unsupported"
       | "unavailable"

src/sqlite/SqliteDatabaseWorkerExecution.ts

@@ -46,6 +46,7 @@ const replySchema = z.union([
       "format",
       "selection",
       "column-selection",
+      "row-evaluation",
       "unsupported",
       "unavailable",
       "output-limit",
@@ -109,6 +110,19 @@ const rejectedReply = (
       },
     );
   }
+  if (reply.reason === "row-evaluation")
+    return new AnalysisCapabilityUnavailableError(
+      "sqlite",
+      operation,
+      reply.message,
+      {
+        capturedOutput,
+        userMessage: reply.message,
+        diagnostics: { selected_path: selectedPath },
+        remediationAction:
+          "SQLite read the selected schema but could not prepare or evaluate the requested rows. REA's private connection loads no extensions, application functions or application collations. The SQLite reason may identify a missing dependency; an unidentified query failure does not establish database corruption. Omit table and use schema_scope: database to inspect stored declarations. Obtaining the requested values may require the application's supported consistent export. Do not repeat the unchanged row request.",
+      },
+    );
   if (reply.reason === "memory")
     return new AnalysisResourceConstraintError(
       operation,

tests/acceptance/sqliteDatabaseCli.test.ts

@@ -19,11 +19,14 @@ import {
   toolAvailabilityInventory,
 } from "../../src/contracts/toolOutputSchemaPrimitives.js";
 
-import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../fixtures/sqlite/database.js";
+import {
+  createSqliteApplicationCollationFixture,
+  SQLITE_NATIVE_LIMITS_AVAILABLE,
+} from "../fixtures/sqlite/database.js";
 import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js";
 import { parseMcpToolError } from "../fixtures/mcpToolError.js";
 import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
-import { cliTest } from "../support/cli/cliFixture.js";
+import { cliTest, runFileCli } from "../support/cli/cliFixture.js";
 
 const digest = (bytes: Buffer): string =>
   createHash("sha256").update(bytes).digest("hex");
@@ -797,6 +800,186 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
   },
 );
 
+cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
+  "reports inventory projections beyond SQLite bind capacity through CLI and MCP",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory(
+      "rea-sqlite-inventory-capacity-",
+    );
+    const path = join(root, "selected.db");
+    const producer = new DatabaseSync(path);
+    let selectedColumns: number;
+    let engineColumnLimit: number;
+    try {
+      producer.exec(
+        "CREATE TABLE records(value TEXT); INSERT INTO records VALUES ('retained');",
+      );
+      engineColumnLimit = producer.limits.column;
+      selectedColumns =
+        Math.max(producer.limits.variableNumber, engineColumnLimit) + 1;
+    } finally {
+      producer.close();
+    }
+    const before = await sourceState(path);
+    const { call } = await connectLocalToolsMcp();
+    const control = await cli.run({
+      arguments: [
+        "inspect-sqlite-database",
+        path,
+        "--schema-scope",
+        "inventory",
+        "--table",
+        "records",
+        "--column",
+        "value",
+        "--row-limit",
+        "1",
+        "--json",
+      ],
+      environment: { REA_LOG_LEVEL: "silent" },
+    });
+    expect(control.exitCode, control.stdout + control.stderr).toBe(0);
+    expect(control.json).toMatchObject({
+      normalized_result: {
+        schema: { scope: "inventory" },
+        rows: {
+          columns: ["value"],
+          returned_rows: 1,
+          values: [[{ type: "text", value: "retained" }]],
+        },
+      },
+    });
+    const columns = Array.from({ length: selectedColumns }, () => "value");
+    // The framed fixture executes the compiled public CLI without the host's
+    // process-argument ceiling rejecting this many repeated flags first.
+    const result = await runFileCli(cli, {
+      arguments: [
+        "inspect-sqlite-database",
+        path,
+        "--schema-scope",
+        "inventory",
+        "--table",
+        "records",
+        ...columns.flatMap((column) => ["--column", column]),
+        "--row-limit",
+        "1",
+        "--json",
+      ],
+      environment: { REA_LOG_LEVEL: "silent" },
+    });
+    const response = await call("inspect_sqlite_database", {
+      path,
+      schema_scope: "inventory",
+      table: "records",
+      columns,
+      row_limit: 1,
+    });
+    const expectedError = {
+      code: "resource_constraint",
+      category: "resource_constraint",
+      retryable: false,
+      details: {
+        resource: "capacity",
+        diagnostics: { selected_path: path },
+        reported_limits: {
+          selected_result_columns: selectedColumns,
+          engine_result_column_limit: engineColumnLimit,
+        },
+      },
+    };
+    expect(result.exitCode, result.stdout + result.stderr).toBe(1);
+    expect(result.json).toMatchObject(expectedError);
+    expect(parseMcpToolError(response).error).toMatchObject(expectedError);
+    expect(await sourceState(path)).toEqual(before);
+  },
+);
+
+cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
+  "reports unavailable application-collation rows with schema-only recovery through CLI and MCP",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-sqlite-collation-public-");
+    const path = createSqliteApplicationCollationFixture(root);
+    const before = await sourceState(path);
+    const { call } = await connectLocalToolsMcp();
+    const schema = await cli.run({
+      arguments: [
+        "inspect-sqlite-database",
+        path,
+        "--schema-scope",
+        "database",
+        "--json",
+      ],
+      environment: { REA_LOG_LEVEL: "silent" },
+    });
+    const schemaResponse = await call("inspect_sqlite_database", {
+      path,
+      schema_scope: "database",
+    });
+    const expectedSchema = {
+      subject: { local_path: path, digest: { sha256: digest(before.bytes) } },
+      normalized_result: {
+        rows: null,
+        schema: {
+          tables: [
+            {
+              name: "items",
+              without_rowid: true,
+              sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"),
+              columns: [
+                { name: "name", hidden: 0 },
+                { name: "value", hidden: 0 },
+                { name: "generated_value", hidden: 2 },
+              ],
+            },
+          ],
+        },
+      },
+    };
+    expect(schema.exitCode, schema.stdout + schema.stderr).toBe(0);
+    expect(schema.json).toMatchObject(expectedSchema);
+    expect(schemaResponse.isError).not.toBe(true);
+    expect(schemaResponse.structuredContent).toMatchObject(expectedSchema);
+
+    const result = await cli.run({
+      arguments: [
+        "inspect-sqlite-database",
+        path,
+        "--table",
+        "items",
+        "--json",
+      ],
+      environment: { REA_LOG_LEVEL: "silent" },
+    });
+    const response = await call("inspect_sqlite_database", {
+      path,
+      table: "items",
+    });
+    const expectedError = {
+      code: "capability_unavailable",
+      retryable: false,
+      message: expect.stringMatching(
+        /rows of table "items".*(?:no query solution|no such collation sequence)/,
+      ),
+      details: {
+        provider_id: "sqlite",
+        operation: "inspect_sqlite_database",
+        diagnostics: { selected_path: path },
+      },
+      remediation: {
+        action: expect.stringContaining("schema_scope: database"),
+      },
+    };
+    expect(result.exitCode, result.stdout + result.stderr).toBe(1);
+    expect(result.json).toMatchObject(expectedError);
+    const error = parseMcpToolError(response).error;
+    expect(error).toMatchObject(expectedError);
+    expect(error.remediation.action).not.toContain(
+      "Select columns whose values were read",
+    );
+    expect(await sourceState(path)).toEqual(before);
+  },
+);
+
 cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
   "attributes an unevaluable generated column to its selection, not the readable database path",
   async ({ cli }) => {

tests/boundary/sqlite/sqliteColumnIdentity.test.ts

@@ -13,7 +13,8 @@ const sample = async (encoding: string, malformed = false) => {
   try {
     db.exec(`PRAGMA encoding='${encoding}';
       CREATE TABLE t("col" TEXT, "�ol" TEXT);
-      INSERT INTO t VALUES ('first-column','replacement-column');`);
+      INSERT INTO t VALUES ('first-column','replacement-column');
+      CREATE TABLE json_each(value TEXT);`);
   } finally {
     db.close();
   }

tests/boundary/sqlite/sqliteDatabaseInspection.test.ts

@@ -1,7 +1,10 @@
 import { DatabaseSync } from "node:sqlite";
 import { readFile, writeFile } from "node:fs/promises";
 import { expect, it as nativeTest, onTestFinished } from "vitest";
-import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../../fixtures/sqlite/database.js";
+import {
+  createSqliteApplicationCollationFixture,
+  SQLITE_NATIVE_LIMITS_AVAILABLE,
+} from "../../fixtures/sqlite/database.js";
 import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js";
 import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js";
 import {
@@ -681,3 +684,45 @@ it("classifies SQLite's result-column capacity refusal without blaming the datab
     }),
   );
 });
+
+it.each(["inventory", "selected_table", "database"] as const)(
+  "keeps unavailable row sources separate from generated expressions in %s scope",
+  async (schema_scope) => {
+    const workspace = await createTestWorkspace("rea-sqlite-collation-");
+    onTestFinished(() => removeTestWorkspace(workspace.root));
+    const path = createSqliteApplicationCollationFixture(workspace.root);
+    const before = await readFile(path);
+    const schema = inspectSqliteDatabaseSnapshot(path, {
+      path,
+      schema_scope: "database",
+    });
+    expect(schema.rows).toBeNull();
+    expect(schema.schema.tables[0]).toMatchObject({
+      name: "items",
+      without_rowid: true,
+      sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"),
+      columns: [
+        expect.objectContaining({ name: "name", hidden: 0 }),
+        expect.objectContaining({ name: "value", hidden: 0 }),
+        expect.objectContaining({ name: "generated_value", hidden: 2 }),
+      ],
+    });
+    for (const columns of [undefined, ["value"], ["generated_value"]])
+      expect(() =>
+        inspectSqliteDatabaseSnapshot(path, {
+          path,
+          table: "items",
+          schema_scope,
+          ...(columns === undefined ? {} : { columns }),
+        }),
+      ).toThrow(
+        expect.objectContaining({
+          reason: "row-evaluation",
+          message: expect.stringMatching(
+            /rows of table "items".*(?:no query solution|no such collation sequence)/,
+          ),
+        }),
+      );
+    expect(await readFile(path)).toEqual(before);
+  },
+);

tests/boundary/sqlite/sqliteDatabaseProvider.test.ts

@@ -15,6 +15,7 @@ import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js";
 import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js";
 import {
   createSqliteDatabaseFixture,
+  createSqliteApplicationCollationFixture,
   SQLITE_NATIVE_LIMITS_AVAILABLE,
 } from "../../fixtures/sqlite/database.js";
 import {
@@ -850,6 +851,102 @@ it("reports a projection above SQLite's result-column capacity as capacity, not
   expect(JSON.stringify(projection)).not.toContain("invalid_format");
 });
 
+it("attributes inventory selections beyond metadata bind capacity to result-column capacity", async () => {
+  const { database } = await createFixture();
+  const reader = new DatabaseSync(database.path, { readOnly: true });
+  let selectedColumns: number;
+  let engineColumnLimit: number;
+  try {
+    engineColumnLimit = reader.limits.column;
+    selectedColumns =
+      Math.max(reader.limits.variableNumber, engineColumnLimit) + 1;
+  } finally {
+    reader.close();
+  }
+  const provider = createProvider();
+  onTestFinished(() => provider.close());
+  const original = await readFile(database.path);
+  const control = await provider.inspect({
+    path: database.path,
+    table: "records",
+    schema_scope: "inventory",
+    columns: ["text_value", "text_value"],
+    row_limit: 1,
+  });
+  if (!control.ok) throw control.error;
+  expect(control.value.result).toMatchObject({
+    rows: {
+      columns: ["text_value", "text_value"],
+      returned_rows: 1,
+    },
+  });
+  const result = await provider.inspect({
+    path: database.path,
+    table: "records",
+    schema_scope: "inventory",
+    columns: Array.from({ length: selectedColumns }, () => "text_value"),
+    row_limit: 1,
+  });
+  if (result.ok) throw new Error("Expected result-column capacity refusal");
+  const projected = projectAnalysisError(result.error);
+  expect(projected).toMatchObject({
+    code: "resource_constraint",
+    details: {
+      resource: "capacity",
+      reported_limits: {
+        selected_result_columns: selectedColumns,
+        engine_result_column_limit: engineColumnLimit,
+      },
+    },
+  });
+  expect(JSON.stringify(projected)).not.toContain("invalid_format");
+  expect(await readFile(database.path)).toEqual(original);
+});
+
+it("reports application-collation row failures as unavailable while retaining schema-only recovery", async () => {
+  const workspace = await createTestWorkspace("rea-sqlite-collation-");
+  onTestFinished(() => removeTestWorkspace(workspace.root));
+  const path = createSqliteApplicationCollationFixture(workspace.root);
+  const original = await readFile(path);
+  const provider = createProvider();
+  onTestFinished(() => provider.close());
+  const schema = await provider.inspect({ path, schema_scope: "database" });
+  if (!schema.ok) throw schema.error;
+  expect(schema.value.result).toMatchObject({
+    schema: {
+      tables: [
+        {
+          name: "items",
+          sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"),
+        },
+      ],
+    },
+    rows: null,
+  });
+  for (const columns of [undefined, ["value"], ["generated_value"]]) {
+    const result = await provider.inspect({
+      path,
+      table: "items",
+      ...(columns === undefined ? {} : { columns }),
+    });
+    if (result.ok)
+      throw new Error("Expected unavailable application collation");
+    const projected = projectAnalysisError(result.error);
+    expect(projected).toMatchObject({
+      code: "capability_unavailable",
+      message: expect.stringContaining('rows of table "items"'),
+      remediation: {
+        action: expect.stringContaining("schema_scope: database"),
+      },
+    });
+    expect(JSON.stringify(projected)).not.toContain("invalid_format");
+    expect(projected.remediation.action).not.toContain(
+      "Select columns whose values were read",
+    );
+  }
+  expect(await readFile(path)).toEqual(original);
+});
+
 it("rejects malformed databases without altering the selected file", async () => {
   const workspace = await createTestWorkspace("rea-sqlite-invalid-");
   onTestFinished(() => removeTestWorkspace(workspace.root));

tests/fixtures/sqlite/database.ts

@@ -72,3 +72,31 @@ export const createSqliteDatabaseFixture = (
     close: () => db.close(),
   };
 };
+
+/** A single stored key stays ordered when its declaration names an application collation. */
+export const createSqliteApplicationCollationFixture = (
+  root: string,
+): string => {
+  const path = join(root, "application-collation.db");
+  const producer = new DatabaseSync(path, { defensive: false });
+  try {
+    // Node exposes no collation-registration API. Changing only the stored
+    // collation name preserves this one-key table's record layout and ordering.
+    producer.exec(`
+      CREATE TABLE items(
+        name TEXT COLLATE NOCASE PRIMARY KEY,
+        value INTEGER,
+        generated_value TEXT GENERATED ALWAYS AS (upper(name)) VIRTUAL
+      ) WITHOUT ROWID;
+      INSERT INTO items(name, value) VALUES ('alpha', 1);
+      PRAGMA writable_schema = ON;
+      UPDATE sqlite_schema
+        SET sql = replace(sql, 'COLLATE NOCASE', 'COLLATE APPLICATION_COLLATION')
+        WHERE name = 'items';
+      PRAGMA writable_schema = OFF;
+    `);
+  } finally {
+    producer.close();
+  }
+  return path;
+};