diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index fb423edea0..3740320b99 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -72,6 +72,15 @@ each failing selected column with SQLite's message, and recommends only columns whose sampled values were read; stored and other generated cells stay readable. Malformed column-name bytes cause a format refusal when all columns are requested. In inventory scope, an explicit valid subset can still be sampled without decoding unrelated malformed column names. + +An unavailable row source, including a `WITHOUT ROWID` primary key requiring an +application collation, is also refused as `capability_unavailable`. The error +retains the selected table and SQLite's reason without identifying unproven +dependencies or diagnosing file corruption. Such a source does not provide +readable-column recommendations. Omit `table` and select `schema_scope: "database"` +to read stored declarations without requesting rows; the application's supported +consistent export may be needed to obtain those values. + Column selection requires a table and at least one name; empty-string identifiers are valid. SQLite counts every projected name, including a repeated name, against its native diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index 164e8ee26d..a3d9b2c78c 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -250,12 +250,20 @@ const readColumns = ( const selection = selectedColumns === undefined ? "" - : ` WHERE CAST(name AS BLOB) IN (${selectedColumns.map(() => "CAST(? AS BLOB)").join(", ")})`; + : " WHERE CAST(name AS BLOB) IN (SELECT CAST(value AS BLOB) FROM temp.json_each(?))"; + // One bound data value keeps metadata selection independent of SQL-variable + // capacity. SQLite owns conversion into the database encoding, and filters + // before projecting unrelated column definitions. Row projections still keep + // the caller's order/repetitions and enforce the native result-column limit. + const parameters = + selectedColumns === undefined + ? [name] + : [name, JSON.stringify([...new Set(selectedColumns)])]; for (const raw of db .prepare( `SELECT cid, CAST(name AS BLOB) AS name, CAST(type AS BLOB) AS type, "notnull", CAST(dflt_value AS BLOB) AS dflt_value, pk, hidden FROM temp.pragma_table_xinfo(?, 'main')${selection}`, ) - .iterate(name, ...(selectedColumns ?? []))) { + .iterate(...parameters)) { const value = columnRow.parse(raw); const column = { cid: value.cid, @@ -599,12 +607,11 @@ const resultColumnLimitFailure = ( }; /** - * Attribute an SQL error from selected cells to the virtual generated columns - * that raise it on this connection. Each column is probed alone over the same - * sampled rows, so only columns whose probe succeeds are recommended; any - * other cause keeps its original classification. + * Attribute SQL failures to the row source before blaming selected generated + * expressions. A generic SQL error alone does not establish file corruption. + * Only columns whose sampled probes succeed are recommended for recovery. */ -const columnEvaluationFailure = ( +const rowEvaluationFailure = ( db: DatabaseSync, cause: unknown, selection: { @@ -619,6 +626,28 @@ const columnEvaluationFailure = ( const columnLimit = resultColumnLimitFailure(db, cause, columns.length); if (columnLimit !== undefined) return columnLimit; if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause; + const unavailable = new SqliteInspectionFailure( + "row-evaluation", + `SQLite cannot prepare or evaluate rows of table ${JSON.stringify(table.name)} on REA's read-only connection: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + const generated = new Set( + table.columns + .filter((column) => column.hidden === 2) + .map((column) => column.name), + ); + if (!columns.some((name) => generated.has(name))) return unavailable; + try { + // A WITHOUT ROWID source may require an unavailable application collation + // even for a constant projection. Do not attribute that failure to every + // generated column or recommend other columns from the same unreadable source. + db.prepare(`SELECT count(*) FROM (SELECT 1 FROM ${source} LIMIT ?)`).get( + limit, + ); + } catch (failure: unknown) { + return sqlitePrimaryResultCode(failure) === SQLITE_ERROR + ? unavailable + : failure; + } // The aggregate evaluates exactly the sampled cells, as the length pass does. const probe = (name: string): { readonly failure: unknown } | undefined => { try { @@ -635,11 +664,6 @@ const columnEvaluationFailure = ( const failed = probe(name); if (failed !== undefined) failures.set(name, failed.failure); } - const generated = new Set( - table.columns - .filter((column) => column.hidden === 2) - .map((column) => column.name), - ); if ( ![...failures].some( ([name, failure]) => @@ -647,7 +671,7 @@ const columnEvaluationFailure = ( sqlitePrimaryResultCode(failure) === SQLITE_ERROR, ) ) - return cause; + return unavailable; const remaining = columns.filter((name) => !failures.has(name)); const readableStoredColumns = (): string[] => { try { @@ -722,8 +746,7 @@ const readRows = ( truncated: false, order: "unspecified", }; - // SQLite compiles and evaluates only selected cells, so an SQL error here is - // attributed to the selected generated expressions rather than the file. + // Keep query-source, expression, capacity and malformed-data failures distinct. try { const lengths = db.prepare( `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`, @@ -741,8 +764,11 @@ const readRows = ( budget.retain(values); rows.values.push(values); } + rows.truncated = + db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !== + undefined; } catch (cause: unknown) { - throw columnEvaluationFailure(db, cause, { + throw rowEvaluationFailure(db, cause, { table, columns, source, @@ -751,9 +777,6 @@ const readRows = ( }); } rows.returned_rows = rows.values.length; - rows.truncated = - db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !== - undefined; return rows; }; diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 05cc4c50b8..56f25d8666 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -13,6 +13,7 @@ export class SqliteInspectionFailure extends Error { | "selection" | "column-selection" | "column-evaluation" + | "row-evaluation" | "column-limit" | "unsupported" | "unavailable" diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index ea656f12b9..48ea8593a0 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -46,6 +46,7 @@ const replySchema = z.union([ "format", "selection", "column-selection", + "row-evaluation", "unsupported", "unavailable", "output-limit", @@ -109,6 +110,19 @@ const rejectedReply = ( }, ); } + if (reply.reason === "row-evaluation") + return new AnalysisCapabilityUnavailableError( + "sqlite", + operation, + reply.message, + { + capturedOutput, + userMessage: reply.message, + diagnostics: { selected_path: selectedPath }, + remediationAction: + "SQLite read the selected schema but could not prepare or evaluate the requested rows. REA's private connection loads no extensions, application functions or application collations. The SQLite reason may identify a missing dependency; an unidentified query failure does not establish database corruption. Omit table and use schema_scope: database to inspect stored declarations. Obtaining the requested values may require the application's supported consistent export. Do not repeat the unchanged row request.", + }, + ); if (reply.reason === "memory") return new AnalysisResourceConstraintError( operation, diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 77fd9e45b0..b58dad8775 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -19,11 +19,14 @@ import { toolAvailabilityInventory, } from "../../src/contracts/toolOutputSchemaPrimitives.js"; -import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../fixtures/sqlite/database.js"; +import { + createSqliteApplicationCollationFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../fixtures/sqlite/database.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); @@ -797,6 +800,186 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, ); +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reports inventory projections beyond SQLite bind capacity through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-sqlite-inventory-capacity-", + ); + const path = join(root, "selected.db"); + const producer = new DatabaseSync(path); + let selectedColumns: number; + let engineColumnLimit: number; + try { + producer.exec( + "CREATE TABLE records(value TEXT); INSERT INTO records VALUES ('retained');", + ); + engineColumnLimit = producer.limits.column; + selectedColumns = + Math.max(producer.limits.variableNumber, engineColumnLimit) + 1; + } finally { + producer.close(); + } + const before = await sourceState(path); + const { call } = await connectLocalToolsMcp(); + const control = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--schema-scope", + "inventory", + "--table", + "records", + "--column", + "value", + "--row-limit", + "1", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(control.exitCode, control.stdout + control.stderr).toBe(0); + expect(control.json).toMatchObject({ + normalized_result: { + schema: { scope: "inventory" }, + rows: { + columns: ["value"], + returned_rows: 1, + values: [[{ type: "text", value: "retained" }]], + }, + }, + }); + const columns = Array.from({ length: selectedColumns }, () => "value"); + // The framed fixture executes the compiled public CLI without the host's + // process-argument ceiling rejecting this many repeated flags first. + const result = await runFileCli(cli, { + arguments: [ + "inspect-sqlite-database", + path, + "--schema-scope", + "inventory", + "--table", + "records", + ...columns.flatMap((column) => ["--column", column]), + "--row-limit", + "1", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const response = await call("inspect_sqlite_database", { + path, + schema_scope: "inventory", + table: "records", + columns, + row_limit: 1, + }); + const expectedError = { + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { + resource: "capacity", + diagnostics: { selected_path: path }, + reported_limits: { + selected_result_columns: selectedColumns, + engine_result_column_limit: engineColumnLimit, + }, + }, + }; + expect(result.exitCode, result.stdout + result.stderr).toBe(1); + expect(result.json).toMatchObject(expectedError); + expect(parseMcpToolError(response).error).toMatchObject(expectedError); + expect(await sourceState(path)).toEqual(before); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reports unavailable application-collation rows with schema-only recovery through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-collation-public-"); + const path = createSqliteApplicationCollationFixture(root); + const before = await sourceState(path); + const { call } = await connectLocalToolsMcp(); + const schema = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--schema-scope", + "database", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const schemaResponse = await call("inspect_sqlite_database", { + path, + schema_scope: "database", + }); + const expectedSchema = { + subject: { local_path: path, digest: { sha256: digest(before.bytes) } }, + normalized_result: { + rows: null, + schema: { + tables: [ + { + name: "items", + without_rowid: true, + sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"), + columns: [ + { name: "name", hidden: 0 }, + { name: "value", hidden: 0 }, + { name: "generated_value", hidden: 2 }, + ], + }, + ], + }, + }, + }; + expect(schema.exitCode, schema.stdout + schema.stderr).toBe(0); + expect(schema.json).toMatchObject(expectedSchema); + expect(schemaResponse.isError).not.toBe(true); + expect(schemaResponse.structuredContent).toMatchObject(expectedSchema); + + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "items", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const response = await call("inspect_sqlite_database", { + path, + table: "items", + }); + const expectedError = { + code: "capability_unavailable", + retryable: false, + message: expect.stringMatching( + /rows of table "items".*(?:no query solution|no such collation sequence)/, + ), + details: { + provider_id: "sqlite", + operation: "inspect_sqlite_database", + diagnostics: { selected_path: path }, + }, + remediation: { + action: expect.stringContaining("schema_scope: database"), + }, + }; + expect(result.exitCode, result.stdout + result.stderr).toBe(1); + expect(result.json).toMatchObject(expectedError); + const error = parseMcpToolError(response).error; + expect(error).toMatchObject(expectedError); + expect(error.remediation.action).not.toContain( + "Select columns whose values were read", + ); + expect(await sourceState(path)).toEqual(before); + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "attributes an unevaluable generated column to its selection, not the readable database path", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteColumnIdentity.test.ts b/tests/boundary/sqlite/sqliteColumnIdentity.test.ts index 1f83454115..5f13efad0b 100644 --- a/tests/boundary/sqlite/sqliteColumnIdentity.test.ts +++ b/tests/boundary/sqlite/sqliteColumnIdentity.test.ts @@ -13,7 +13,8 @@ const sample = async (encoding: string, malformed = false) => { try { db.exec(`PRAGMA encoding='${encoding}'; CREATE TABLE t("col" TEXT, "�ol" TEXT); - INSERT INTO t VALUES ('first-column','replacement-column');`); + INSERT INTO t VALUES ('first-column','replacement-column'); + CREATE TABLE json_each(value TEXT);`); } finally { db.close(); } diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index db19bdf80f..f30e311ab1 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -1,7 +1,10 @@ import { DatabaseSync } from "node:sqlite"; import { readFile, writeFile } from "node:fs/promises"; import { expect, it as nativeTest, onTestFinished } from "vitest"; -import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../../fixtures/sqlite/database.js"; +import { + createSqliteApplicationCollationFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js"; import { @@ -681,3 +684,45 @@ it("classifies SQLite's result-column capacity refusal without blaming the datab }), ); }); + +it.each(["inventory", "selected_table", "database"] as const)( + "keeps unavailable row sources separate from generated expressions in %s scope", + async (schema_scope) => { + const workspace = await createTestWorkspace("rea-sqlite-collation-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = createSqliteApplicationCollationFixture(workspace.root); + const before = await readFile(path); + const schema = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope: "database", + }); + expect(schema.rows).toBeNull(); + expect(schema.schema.tables[0]).toMatchObject({ + name: "items", + without_rowid: true, + sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"), + columns: [ + expect.objectContaining({ name: "name", hidden: 0 }), + expect.objectContaining({ name: "value", hidden: 0 }), + expect.objectContaining({ name: "generated_value", hidden: 2 }), + ], + }); + for (const columns of [undefined, ["value"], ["generated_value"]]) + expect(() => + inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + schema_scope, + ...(columns === undefined ? {} : { columns }), + }), + ).toThrow( + expect.objectContaining({ + reason: "row-evaluation", + message: expect.stringMatching( + /rows of table "items".*(?:no query solution|no such collation sequence)/, + ), + }), + ); + expect(await readFile(path)).toEqual(before); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 619cd9fc0f..04f574a8c4 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -15,6 +15,7 @@ import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; import { createSqliteDatabaseFixture, + createSqliteApplicationCollationFixture, SQLITE_NATIVE_LIMITS_AVAILABLE, } from "../../fixtures/sqlite/database.js"; import { @@ -850,6 +851,102 @@ it("reports a projection above SQLite's result-column capacity as capacity, not expect(JSON.stringify(projection)).not.toContain("invalid_format"); }); +it("attributes inventory selections beyond metadata bind capacity to result-column capacity", async () => { + const { database } = await createFixture(); + const reader = new DatabaseSync(database.path, { readOnly: true }); + let selectedColumns: number; + let engineColumnLimit: number; + try { + engineColumnLimit = reader.limits.column; + selectedColumns = + Math.max(reader.limits.variableNumber, engineColumnLimit) + 1; + } finally { + reader.close(); + } + const provider = createProvider(); + onTestFinished(() => provider.close()); + const original = await readFile(database.path); + const control = await provider.inspect({ + path: database.path, + table: "records", + schema_scope: "inventory", + columns: ["text_value", "text_value"], + row_limit: 1, + }); + if (!control.ok) throw control.error; + expect(control.value.result).toMatchObject({ + rows: { + columns: ["text_value", "text_value"], + returned_rows: 1, + }, + }); + const result = await provider.inspect({ + path: database.path, + table: "records", + schema_scope: "inventory", + columns: Array.from({ length: selectedColumns }, () => "text_value"), + row_limit: 1, + }); + if (result.ok) throw new Error("Expected result-column capacity refusal"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "resource_constraint", + details: { + resource: "capacity", + reported_limits: { + selected_result_columns: selectedColumns, + engine_result_column_limit: engineColumnLimit, + }, + }, + }); + expect(JSON.stringify(projected)).not.toContain("invalid_format"); + expect(await readFile(database.path)).toEqual(original); +}); + +it("reports application-collation row failures as unavailable while retaining schema-only recovery", async () => { + const workspace = await createTestWorkspace("rea-sqlite-collation-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = createSqliteApplicationCollationFixture(workspace.root); + const original = await readFile(path); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const schema = await provider.inspect({ path, schema_scope: "database" }); + if (!schema.ok) throw schema.error; + expect(schema.value.result).toMatchObject({ + schema: { + tables: [ + { + name: "items", + sql: expect.stringContaining("COLLATE APPLICATION_COLLATION"), + }, + ], + }, + rows: null, + }); + for (const columns of [undefined, ["value"], ["generated_value"]]) { + const result = await provider.inspect({ + path, + table: "items", + ...(columns === undefined ? {} : { columns }), + }); + if (result.ok) + throw new Error("Expected unavailable application collation"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining('rows of table "items"'), + remediation: { + action: expect.stringContaining("schema_scope: database"), + }, + }); + expect(JSON.stringify(projected)).not.toContain("invalid_format"); + expect(projected.remediation.action).not.toContain( + "Select columns whose values were read", + ); + } + expect(await readFile(path)).toEqual(original); +}); + it("rejects malformed databases without altering the selected file", async () => { const workspace = await createTestWorkspace("rea-sqlite-invalid-"); onTestFinished(() => removeTestWorkspace(workspace.root)); diff --git a/tests/fixtures/sqlite/database.ts b/tests/fixtures/sqlite/database.ts index ae174c44d7..53773c8fe2 100644 --- a/tests/fixtures/sqlite/database.ts +++ b/tests/fixtures/sqlite/database.ts @@ -72,3 +72,31 @@ export const createSqliteDatabaseFixture = ( close: () => db.close(), }; }; + +/** A single stored key stays ordered when its declaration names an application collation. */ +export const createSqliteApplicationCollationFixture = ( + root: string, +): string => { + const path = join(root, "application-collation.db"); + const producer = new DatabaseSync(path, { defensive: false }); + try { + // Node exposes no collation-registration API. Changing only the stored + // collation name preserves this one-key table's record layout and ordering. + producer.exec(` + CREATE TABLE items( + name TEXT COLLATE NOCASE PRIMARY KEY, + value INTEGER, + generated_value TEXT GENERATED ALWAYS AS (upper(name)) VIRTUAL + ) WITHOUT ROWID; + INSERT INTO items(name, value) VALUES ('alpha', 1); + PRAGMA writable_schema = ON; + UPDATE sqlite_schema + SET sql = replace(sql, 'COLLATE NOCASE', 'COLLATE APPLICATION_COLLATION') + WHERE name = 'items'; + PRAGMA writable_schema = OFF; + `); + } finally { + producer.close(); + } + return path; +};