merge(main): take CI cache and WebMCP coverage fixes

Co-authored-by: morluto <morluto@users.noreply.github.com>
This commit is contained in:
Cursor Agentandmorluto committed 2026-10-20 20:03:08 +00:00
commit 2775bf4e41
32 files changed
+3257 -593

No files matched your search

+13 -3
View File
@@ -1,5 +1,10 @@
name: Cache npm package downloads
description: Restore npm's download cache and save it only from trusted default-branch events.
description: Restore npm's download cache. Trusted default-branch jobs that opt in may save it.
inputs:
save:
description: Save the shared download cache when this event is allowed to write it.
required: false
default: "false"
outputs:
can-save:
description: Whether this run may update the shared npm download cache.
@@ -23,20 +28,25 @@ runs:
can_save=true
fi
cache_key="npm-download-${RUNNER_OS}-${runtime_arch}-${LOCKFILES_HASH}"
# Same platform only. A lockfile change can reuse tarballs from the previous key.
restore_key="npm-download-${RUNNER_OS}-${runtime_arch}-"
{
printf 'can-save=%s\n' "${can_save}"
printf 'path=%s\n' "${cache_path}"
printf 'key=%s\n' "${cache_key}"
printf 'restore-key=%s\n' "${restore_key}"
} >> "${GITHUB_OUTPUT}"
- name: Restore and save npm package downloads
if: steps.context.outputs.can-save == 'true'
if: steps.context.outputs.can-save == 'true' && inputs.save == 'true'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ steps.context.outputs.path }}
key: ${{ steps.context.outputs.key }}
restore-keys: ${{ steps.context.outputs.restore-key }}
- name: Restore npm package downloads without saving
if: steps.context.outputs.can-save != 'true'
if: steps.context.outputs.can-save != 'true' || inputs.save != 'true'
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ steps.context.outputs.path }}
key: ${{ steps.context.outputs.key }}
restore-keys: ${{ steps.context.outputs.restore-key }}
+57
View File
@@ -0,0 +1,57 @@
name: Cache retention
on:
workflow_run:
workflows: ["CI"]
types: [completed]
# Qualification calls ci.yml with actions: read. A nested workflow cannot raise
# that token, so cache deletion stays in this workflow.
permissions:
contents: read
actions: write
concurrency:
group: cache-retention-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: false
jobs:
retain:
name: Retain Turbo snapshots and npm download caches
# Checkout and run only a same-repository default-branch commit. Pull
# requests, including forks, do not reach these steps. Static and build
# save Turbo snapshots before later lanes finish, so a failed, timed-out,
# or cancelled run still retains those snapshots.
if: >-
(github.event.workflow_run.conclusion == 'success' ||
github.event.workflow_run.conclusion == 'failure' ||
github.event.workflow_run.conclusion == 'timed_out' ||
github.event.workflow_run.conclusion == 'cancelled') &&
github.event.workflow_run.head_repository.full_name == github.repository &&
github.event.workflow_run.head_branch == github.event.repository.default_branch &&
(github.event.workflow_run.event == 'push' ||
github.event.workflow_run.event == 'schedule' ||
github.event.workflow_run.event == 'workflow_dispatch')
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
ref: ${{ github.event.workflow_run.head_sha }}
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Retain current Turbo snapshots and npm download caches
env:
GH_TOKEN: ${{ github.token }}
CI_REPOSITORY: ${{ github.repository }}
CACHE_PREFIX: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-
CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
CACHE_DEEP_PREFIX: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-
CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
CACHE_KNIP_PREFIX: knip-meta-v1-${{ runner.os }}-
CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
CACHE_PORTABLE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-
DEDUPE_NPM_DOWNLOADS: "true"
run: node scripts/ci/cache-retention.mjs
+13 -3
View File
@@ -146,6 +146,9 @@ jobs:
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
with:
# Linux x64 writer. Parallel Linux jobs restore this key.
save: "true"
- run: npm ci
- name: Restore static task cache
if: needs.changes.outputs.code == 'true'
@@ -299,9 +302,11 @@ jobs:
# every shard's diagnostics and coverage. Passing PRs run the full suite.
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
matrix:
# Fast feedback spends the same six test jobs on one source group and
# five built groups; full qualification retains six complete shards.
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
# Fast feedback uses one source group and eight built groups so the
# slowest built shard job stays inside a ten-minute execution budget.
# Classification, build, and report merge sit outside that job.
# Full qualification retains six complete shards with coverage.
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/8","2/8","3/8","4/8","5/8","6/8","7/8","8/8"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
@@ -658,6 +663,9 @@ jobs:
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
with:
# ubuntu-latest is the Linux x64 writer above. Other hosts save their own key.
save: ${{ matrix.os != 'ubuntu-latest' }}
- name: Verify native runtime identity
env:
EXPECTED_CI_PLATFORM: ${{ matrix.platform }}
@@ -794,6 +802,8 @@ jobs:
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
with:
save: "true"
- run: npm ci
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
+16
View File
@@ -19,8 +19,24 @@ Inspect native SAZ or HAR 1.2 offline with REA, or connect Everywhere's separate
- `analyze_web_bundle` / `rea analyze-web-bundle` parses selected script artifacts without execution and derives chunk edges, route and endpoint candidates, vendor fingerprints, static WebMCP declarations, and optionally source-map/original-source evidence.
- `observe_web_session` / `rea observe-web-session` arms an observation window of the requested duration for an external user action and records ordered reload, SPA navigation, redirect, failure, lifecycle, and target-termination metadata.
- `discover_webmcp_tools` / `rea discover-webmcp-tools` uses the experimental CDP WebMCP domain to return every in-scope registration and its complete observed `input_schema` declaration and canonical `input_schema_sha256` digest inline. Declarations are untrusted JSON data, including declared types, required fields, enums, examples and reference URLs. REA does not resolve schema references. These fields replace `input_schema_shape`. Registrations with the same name and URL in distinct observed frames retain separate identities. `tool_key` includes the observed frame ID; it is not a stable identity across navigation or browser sessions. Same-frame registration updates replace that owner only, and removal leaves other owners intact. Declared `readOnly`, `untrustedContent`, `consequential`, and `autosubmit` annotation hints are reported per registration: `true`/`false` are declared values, `null` means the declaration did not provide the hint in that capture, and a missing `consequential` key means the hint was not recorded, either in a historical capture or because the observed annotation was invalid. Consult discovery completeness and exclusions to distinguish these cases. Malformed annotation values retain the registration with incomplete coverage and an `invalid_protocol_value` exclusion. Other malformed protocol payloads can fail discovery explicitly. REA never exposes `WebMCP.invokeTool`.
- `compare_web_captures` / `rea compare-web-captures` compares passive captures or exact-step-aligned browser scenarios. Scenario results commit reproducible literal normalization and expose alignment failures plus artifact-level action, screenshot, DOM, accessibility, URL, history, storage, and event differences. Missing or truncated evidence is never treated as equivalence.
For `analyze_web_bundle`, each `webmcp_declarations` entry's
`schema_property_names` is a source-derived projection of statically known own
property keys, not schema validation. `schema_property_names_coverage` reports
how complete that projection is: `complete` means a literal properties object
was fully enumerated (including an empty literal); `partial` means a literal
object was available but some computed or spread membership was unresolved,
and the name array retains keys that were known; `unknown` means the schema or
properties owner could not be enumerated as an object literal, including
explicit non-object values; `not_present` means no effective schema or own
`properties` member was present after the `inputSchema ?? input_schema` alias
selection. `not_present` describes this projection only: it does not validate a
schema or rule out properties expressed through other schema constructs. Older
observations may omit the coverage field; treat omission as `unknown` rather
than inferring completeness from the array.
Passive DOM comparison checks complete node inventories and complete frame
parent/child structure independently. Frame IDs and sibling enumeration order
are capture-local and do not establish a change; duplicate IDs, missing parents,
+75 -26
View File
@@ -471,8 +471,8 @@ partial diagnostics and preflights do not establish complete mutation outcomes
or qualify the release revision for mutation coverage.
Fast CI starts the source-only projects after classification, independently of
build and static checks, and runs five built-dependent shards after the build.
The six test jobs still select every portable test. Full qualification keeps six
build and static checks, and runs eight built-dependent shards after the build.
Those nine test jobs still select every portable test. Full qualification keeps six
complete Vitest shards with coverage. Neither tier retries tests. Canonical
project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest,
CI group execution, and development-test preparation. A test importing compiled
@@ -533,11 +533,35 @@ JSON artifact members without executing artifact content. Reports and limits
are retained in `ci-health-diagnostics`; its ten-minute job limit covers the
existing four-minute timing collection plus the bounded diagnostic stages.
Shard-count reduction remains deferred: the available successful timing sample
covers an older inventory, and file phase weights are not job wall time. The
one-source/five-built split retains six test jobs; it makes no hosted speedup
claim. Current complete timings and completed candidate hosted runs are needed
before enabling a smaller adaptive matrix.
Fast built shards target under ten minutes of uncontended shard-job execution.
That figure is the `test-shard` job. The job waits for classification and the
build, and the portable-test merge runs after the slowest shard. On 2026-10-20,
run `39451548945` measured those surrounding jobs with runner queue excluded:
classification 15 seconds, build 49 seconds, and the portable-test merge 29
seconds. Shard 1/5 of that run spent 22 seconds before built Vitest and 3
seconds after it, including `npm ci` at 9 seconds. `node_modules` is about 674
MiB, so publishing it as a shared artifact would transfer more data than that
install saves. The npm download cache stays the setup cache.
File timings from that run put the built suite at 7944 seconds of module time.
The sequencer keeps each file's full duration on one worker. Extra workers
overlap other files in the same sequence group, and `process-boundary` runs
after that group. Scheduling the unsplit inventory onto five shards peaks at
820 seconds of tests. That run's slowest built shard job lasted 925 seconds
(shard 4/5), and shard 1/5 spent 874 seconds in built Vitest, so hosted job
time sits above the file schedule. Six shards of the same inventory peak at
684 seconds of tests. Eight shards, after the 578-second loader file is split
by case count (54 cases stay in that file and 46 move into each of three
files), peak at 513 seconds of tests. With the measured 25-second shard setup,
that shard job is about 538 seconds. Adding the measured classification, build,
and merge makes an uncontended portable critical path of about 631 seconds,
still before the gap between this file schedule and hosted Vitest. Runner queue
is separate and grows when more shard jobs compete for runners.
`process-boundary` stays serial because process-tree sampling shares the host.
Raising the worker cap from two to four on this four-core host cut a six-file
acceptance sample from 96–101 seconds to 71 seconds and did not remove the
single-file floor, so the cap stays two. Full qualification keeps six covered
shards; its longer instrumentation is outside this fast budget.
The separate `CI timings` workflow batches recent completed CI and Release runs
hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and
@@ -579,8 +603,9 @@ metadata, and available manifests into `.cache/ci-build-cache/`.
`portable-build-cache.json` records availability per task. Evicted or partially
missing entries are omitted as whole entries, and consumers rebuild cache misses,
including when every entry is unavailable. Malformed plans, non-file entries,
and I/O errors other than disappearance still fail publication. Older cross-run entries stay in the
original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
and I/O errors other than disappearance still fail publication. After a trusted
static run, superseded cross-run Turbo snapshots for that OS and architecture
are deleted down to the two newest generations. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
commands, which restore matching outputs or rebuild on a cache miss. Native
artifacts, dependency installs, package archives, and runtime verification remain
host-local. Keep platform-dependent outputs out of this shared cache.
@@ -1129,7 +1154,7 @@ suite, without the source-test shards or native package
lanes. Classification compares the PR head with its merge base, so later
base-branch changes do not expand that scope. Static and portable-test aggregate jobs
remain present and fail if classification fails. Every code change retains the
complete six-shard Linux suite; individual test files are not selected by
complete portable Linux suite; individual test files are not selected by
imports or filenames.
Ordinary PRs also produce an **observational** test-selection plan. The candidate
@@ -1140,7 +1165,7 @@ tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall
back to the complete inventory. Renames account for both paths. Main, scheduled,
manual, and `ci:full` runs always propose the full suite.
All six shards still execute the complete suite. The existing report merge
The source group and every built shard together still execute the complete suite. The existing report merge
compares the proposal with actual module outcomes and retains
`.cache/ci-test-selection/report.json` in `test-results`, including selected and
omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and
@@ -1222,7 +1247,7 @@ push: changes from failed or cancelled intermediate runs remain in scope.
Missing, untrusted, or non-ancestor history selects the fast Linux x64
baseline: portable checks and the Linux runtime lane. The lookup is
bounded to the last 100 successful push runs; lookup failures broaden selection.
Every code change still selects all six portable shards, without coverage
Every code change still selects the source group and all eight fast built shards, without coverage
instrumentation in this fast tier. Ordinary PR matrices cancel remaining
shards after the first failure; passing PRs execute the complete suite. Main,
`ci:full`, scheduled, manual, and release qualification retain every shard's
@@ -1335,13 +1360,15 @@ producer; newer schedules that reuse qualification without producing timings do
not hide the last full run's artifact.
CI assigns discovered files by historical duration, with a deterministic
fallback weight for new files, rather than dividing only by file count.
`process-boundary` files are weighted at their full duration because that
project runs files serially. Other projects are weighted at half their file
duration, matching the two-worker cap, so a serial file does not look as cheap
as a file that overlaps with another. Every
discovered file still belongs to exactly one shard; worker limits and project
ordering remain unchanged. Fast qualification keeps five built shards without
coverage. The merge job rejects successful reports with missing,
Each file keeps its full duration on one worker. Extra workers overlap other
files in the same sequence group, and `process-boundary` runs after that
group. History published before the competing-loader split still names only
`javascriptLiteralPercentPaths.test.ts`. Until that artifact records one of
the three new files, the sequencer assigns that recorded duration by the
pre-split case counts: 54/192 to the remaining file and 46/192 to each new
file. Every discovered file still belongs to exactly one shard; worker limits
and project ordering remain unchanged. Fast qualification keeps eight built
shards without coverage. The merge job rejects successful reports with missing,
unexpected, or duplicate portable files. Coverage instrumentation and aggregate
thresholds remain enforced in full qualification.
Missing or unusable history falls back to Vitest's default assignment. Only
@@ -1370,13 +1397,35 @@ token leaves the local Turbo cache in place.
The shared npm-cache action caches downloads, not `node_modules`, and every
consumer still runs `npm ci`. Keys cover the runner OS, actual Node architecture,
root lockfiles, and `.npmrc`. Only push, schedule, and manual events on the
repository's default branch may save shared npm caches; PRs restore only.
Static snapshots use the same write authority and are saved only after the
static job's validation succeeds. Main warms the new cache namespace after
merge; a cold cache or changed lockfile falls back to the normal install.
This prevents each PR from saving another branch-scoped copy of the same
download cache. It does not remove runner assignment waits.
root lockfiles, and `.npmrc`. An exact-key miss restores the newest cache for
that OS and architecture, so a lockfile change reuses unchanged tarballs while
`npm ci` still verifies the lockfile. Only push, schedule, and manual events on
the repository's default branch may save shared npm caches, and only from the
opted-in writer for that host: Static checks for Linux x64, the package matrix
for every other included host, and the Windows curated lane. Every other job
restores without saving, so parallel jobs do not publish duplicate copies.
Pull requests restore only. Static snapshots are saved only after the static
job's validation succeeds. Cache deletion is a
separate workflow, because reusable qualification calls this workflow and
cannot raise its read-only token. After a push, schedule, or manual CI run on
the default branch finishes, including when a later lane fails, times out, or
is cancelled, that workflow runs on `ubuntu-latest` and deletes older
`rea-static-v1`, `rea-static-deep-v1`, and `rea-portable-build-v2` entries for
that runner's OS and architecture, plus older `knip-meta-v1` entries for that
OS. It keeps two generations plus the key that run saved. A fast run that did
not publish a deep or Knip snapshot still leaves the two newest entries in
those families. It
deletes an npm download record only when another record has the same key, cache
version, and git ref, and it keeps the most recently accessed copy. A different
version or ref is a different restore target and stays. One cache id is one
record: a repeated id keeps the later access time when its key, version, ref,
size, and creation time agree, and an inconsistent repeat deletes nothing. An
incomplete or unexpected cache list deletes nothing.
Distinct lockfile generations of the download cache stay, so the restore
prefix still has a seed.
Main warms the new cache namespace after merge; a cold cache falls back to the
normal install. This prevents each PR from saving another branch-scoped copy of
the same download cache. It does not remove runner assignment waits.
Portable owner-based test selection remains observational. Its reports compare
proposed omissions with the full run; incomplete or skipped observations do not
+61
View File
@@ -0,0 +1,61 @@
export interface RetainedCacheEntry {
readonly id: number;
readonly key: string;
readonly createdAt: number;
readonly accessedAt: number;
readonly size: number;
}
export interface CacheListRecord {
readonly id: number;
readonly key: string;
readonly version: string;
readonly ref: string;
readonly created_at: string;
readonly last_accessed_at?: string;
readonly size_in_bytes: number;
}
export interface RetentionFamily {
readonly prefix: string;
readonly keepKey?: string;
}
export type CacheRequest = (
method: string,
endpoint: string,
) => Promise<unknown>;
export function assertRetentionPrefix(prefix: string): void;
export function selectPrefixDeletions(
caches: readonly unknown[],
family?: RetentionFamily,
keepGenerations?: number,
): RetainedCacheEntry[];
export function selectNpmDownloadDuplicateDeletions(
caches: readonly unknown[],
): RetainedCacheEntry[];
export function listAllCaches(
request: CacheRequest,
repository: string,
prefix: string,
): Promise<readonly CacheListRecord[]>;
export function retainCaches(options: {
repository: string;
request: CacheRequest;
deleteCache: (id: number) => Promise<void>;
families?: readonly RetentionFamily[];
dedupeNpmDownloads?: boolean;
dryRun?: boolean;
}): Promise<{
dryRun: boolean;
plan: readonly {
prefix: string;
deletions: readonly RetainedCacheEntry[];
}[];
npmDeletions: readonly RetainedCacheEntry[];
}>;
+358
View File
@@ -0,0 +1,358 @@
import { execFile } from "node:child_process";
import { realpathSync } from "node:fs";
import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
const execFileAsync = promisify(execFile);
const MAX_BUFFER = 8 * 1024 * 1024;
const REQUEST_TIMEOUT_MS = 20_000;
const MAX_PAGES = 10;
const PAGE_SIZE = 100;
// Keep the newest snapshots, not only the key just saved. A concurrent job may
// still be restoring the previous generation; deleting that id fails its restore.
// The following trusted run drops anything older than those two.
const RETAINED_CACHE_GENERATIONS = 2;
const PREFIX_PATTERNS = [
/^rea-static-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
/^rea-static-deep-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
/^rea-portable-build-v2-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
/^knip-meta-v1-[A-Za-z0-9]+-$/u,
];
/** Reject prefixes that could match npm downloads or another platform. */
export function assertRetentionPrefix(prefix) {
if (
typeof prefix !== "string" ||
!PREFIX_PATTERNS.some((pattern) => pattern.test(prefix))
)
throw new Error(
"Cache retention prefix must be one Turbo family for one OS and architecture, or one Knip metadata family for one OS",
);
}
function normalizeCache(cache, index) {
if (typeof cache !== "object" || cache === null || Array.isArray(cache))
throw new Error(`Cache entry ${String(index + 1)} is malformed`);
const { id, key, size_in_bytes: size } = cache;
if (!Number.isSafeInteger(id) || id <= 0)
throw new Error(`Cache entry ${String(index + 1)} has an invalid id`);
if (typeof key !== "string" || key.length === 0 || /[\r\n]/u.test(key))
throw new Error(`Cache entry ${String(index + 1)} has an invalid key`);
const createdAt = Date.parse(cache.created_at);
const accessedAt = Date.parse(cache.last_accessed_at ?? cache.created_at);
if (!Number.isFinite(createdAt) || !Number.isFinite(accessedAt))
throw new Error(
`Cache entry ${String(index + 1)} has an invalid timestamp`,
);
if (!Number.isSafeInteger(size) || size < 0)
throw new Error(`Cache entry ${String(index + 1)} has an invalid size`);
// actions/cache restores a key only when this metadata hash matches.
const version = cache.version;
if (typeof version !== "string" || !/^[a-f0-9]{64}$/u.test(version))
throw new Error(`Cache entry ${String(index + 1)} has an invalid version`);
// The git ref is the cache's access scope. Another branch cannot use it.
const ref = cache.ref;
if (typeof ref !== "string" || !/^refs\/[^\s]+$/u.test(ref))
throw new Error(`Cache entry ${String(index + 1)} has an invalid ref`);
return { id, key, version, ref, createdAt, accessedAt, size };
}
function sameStableIdentity(left, right) {
return (
left.key === right.key &&
left.version === right.version &&
left.ref === right.ref &&
left.size === right.size &&
left.createdAt === right.createdAt
);
}
// The list API orders by last access. A restore between pages can place one
// id on two pages. That is still one cache: collapse it, or abort when the
// pages disagree, before any id can be both kept and deleted.
function uniqueById(entries) {
const byId = new Map();
for (const entry of entries) {
const existing = byId.get(entry.id);
if (existing === undefined) {
byId.set(entry.id, entry);
continue;
}
if (!sameStableIdentity(existing, entry))
throw new Error(
`Cache id ${String(entry.id)} was listed with inconsistent fields`,
);
if (entry.accessedAt > existing.accessedAt) byId.set(entry.id, entry);
}
return [...byId.values()];
}
function byNewest(left, right) {
if (left.createdAt !== right.createdAt)
return right.createdAt - left.createdAt;
return right.id - left.id;
}
/**
* Choose Turbo snapshot ids to delete. `keepKey` is protected even when it is
* older than the retained generations. Keys outside the prefix abort retention.
*/
export function selectPrefixDeletions(
caches,
{ prefix, keepKey } = {},
keepGenerations = RETAINED_CACHE_GENERATIONS,
) {
assertRetentionPrefix(prefix);
if (!Number.isSafeInteger(keepGenerations) || keepGenerations < 1)
throw new Error("Cache retention must keep at least one generation");
if (keepKey !== undefined) {
if (
typeof keepKey !== "string" ||
!keepKey.startsWith(prefix) ||
keepKey === prefix
)
throw new Error("Cache retention keep key must extend its prefix");
}
const entries = uniqueById(caches.map(normalizeCache));
for (const entry of entries)
if (!entry.key.startsWith(prefix))
throw new Error(
`Refusing to evaluate a cache outside ${prefix}: ${entry.key}`,
);
const protectedIds = new Set(
[...entries]
.sort(byNewest)
.slice(0, keepGenerations)
.map((entry) => entry.id),
);
if (keepKey !== undefined)
for (const entry of entries)
if (entry.key === keepKey) protectedIds.add(entry.id);
return entries.filter((entry) => !protectedIds.has(entry.id));
}
/**
* Drop extra copies of one npm restore target. Records that share a user key
* but differ by cache version or git ref stay, because a workflow can restore
* only a matching version inside its visible scope.
*/
export function selectNpmDownloadDuplicateDeletions(caches) {
const entries = uniqueById(caches.map(normalizeCache));
for (const entry of entries)
if (!entry.key.startsWith("npm-download-"))
throw new Error(
`Refusing to evaluate a non-download cache: ${entry.key}`,
);
const groups = new Map();
for (const entry of entries) {
const identity = `${entry.version}\n${entry.ref}\n${entry.key}`;
const group = groups.get(identity) ?? [];
group.push(entry);
groups.set(identity, group);
}
const deletions = [];
for (const group of groups.values()) {
if (group.length < 2) continue;
const ranked = [...group].sort((left, right) => {
if (left.accessedAt !== right.accessedAt)
return right.accessedAt - left.accessedAt;
if (left.createdAt !== right.createdAt)
return right.createdAt - left.createdAt;
return right.id - left.id;
});
deletions.push(...ranked.slice(1));
}
return deletions;
}
export async function listAllCaches(request, repository, prefix) {
const caches = [];
for (let page = 1; page <= MAX_PAGES; page += 1) {
const body = await request(
"GET",
`repos/${repository}/actions/caches?per_page=${String(PAGE_SIZE)}&page=${String(page)}&key=${encodeURIComponent(prefix)}`,
);
if (
typeof body !== "object" ||
body === null ||
Array.isArray(body) ||
!Array.isArray(body.actions_caches)
)
throw new Error("GitHub cache list is malformed");
caches.push(...body.actions_caches);
if (body.actions_caches.length < PAGE_SIZE) return caches;
}
throw new Error(
`Cache list for ${prefix} exceeded ${String(MAX_PAGES)} pages; deleted nothing`,
);
}
/**
* List every requested family before deleting. A malformed list or an unsafe
* prefix leaves the repository caches unchanged.
*/
export async function retainCaches({
repository,
request,
deleteCache,
families = [],
dedupeNpmDownloads = false,
dryRun = false,
}) {
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
for (const family of families) {
assertRetentionPrefix(family.prefix);
if (
family.keepKey !== undefined &&
(typeof family.keepKey !== "string" ||
!family.keepKey.startsWith(family.prefix) ||
family.keepKey === family.prefix)
)
throw new Error("Cache retention keep key must extend its prefix");
}
const plan = [];
for (const family of families) {
const caches = await listAllCaches(request, repository, family.prefix);
plan.push({
prefix: family.prefix,
deletions: selectPrefixDeletions(caches, family),
});
}
const npmDeletions = dedupeNpmDownloads
? selectNpmDownloadDuplicateDeletions(
await listAllCaches(request, repository, "npm-download-"),
)
: [];
if (!dryRun)
for (const entry of [
...plan.flatMap((family) => family.deletions),
...npmDeletions,
])
await deleteCache(entry.id);
return { dryRun, plan, npmDeletions };
}
function repositoryFromEnv(env) {
const repository = env.CI_REPOSITORY;
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
return repository;
}
function familyWithKeepKey(env, prefixName, keepName) {
const prefix = env[prefixName];
const keepKey = env[keepName];
const hasPrefix = typeof prefix === "string" && prefix.length > 0;
const hasKeepKey = typeof keepKey === "string" && keepKey.length > 0;
if (!hasPrefix && !hasKeepKey) return undefined;
if (!hasPrefix || !hasKeepKey)
throw new Error(`${keepName} is required with ${prefixName}`);
return { prefix, keepKey };
}
function familiesFromEnv(env) {
const families = [
familyWithKeepKey(env, "CACHE_PREFIX", "CACHE_KEEP_KEY"),
familyWithKeepKey(env, "CACHE_DEEP_PREFIX", "CACHE_DEEP_KEEP_KEY"),
familyWithKeepKey(env, "CACHE_KNIP_PREFIX", "CACHE_KNIP_KEEP_KEY"),
typeof env.CACHE_PORTABLE_PREFIX === "string" &&
env.CACHE_PORTABLE_PREFIX.length > 0
? { prefix: env.CACHE_PORTABLE_PREFIX }
: undefined,
].filter((family) => family !== undefined);
if (families.length === 0 && env.DEDUPE_NPM_DOWNLOADS !== "true")
throw new Error(
"Cache retention requires CACHE_PREFIX or CACHE_PORTABLE_PREFIX or npm dedupe",
);
return families;
}
function failureText(error) {
if (typeof error !== "object" || error === null) return String(error);
const stderr =
"stderr" in error && error.stderr !== undefined ? String(error.stderr) : "";
const message = error instanceof Error ? error.message : "";
return `${message}\n${stderr}`;
}
async function githubRequest(method, endpoint, env) {
const { stdout } = await execFileAsync(
"gh",
["api", "--method", method, endpoint],
{
env,
encoding: "utf8",
maxBuffer: MAX_BUFFER,
timeout: REQUEST_TIMEOUT_MS,
},
);
if (method === "DELETE") return undefined;
try {
return JSON.parse(stdout);
} catch (cause) {
throw new Error("GitHub cache API returned malformed JSON", { cause });
}
}
async function main() {
const env = process.env;
const dryRun = env.CACHE_RETENTION_DRY_RUN === "true";
const report = await retainCaches({
repository: repositoryFromEnv(env),
request: (method, endpoint) => githubRequest(method, endpoint, env),
deleteCache: async (id) => {
try {
await githubRequest(
"DELETE",
`repos/${repositoryFromEnv(env)}/actions/caches/${String(id)}`,
env,
);
} catch (error) {
if (/404|not found/iu.test(failureText(error))) return;
throw error;
}
},
families: familiesFromEnv(env),
dedupeNpmDownloads: env.DEDUPE_NPM_DOWNLOADS === "true",
dryRun,
});
const summary = {
dryRun: report.dryRun,
families: report.plan.map((family) => ({
prefix: family.prefix,
deleteCount: family.deletions.length,
deleteBytes: family.deletions.reduce((sum, entry) => sum + entry.size, 0),
})),
npmDuplicateDeleteCount: report.npmDeletions.length,
npmDuplicateDeleteBytes: report.npmDeletions.reduce(
(sum, entry) => sum + entry.size,
0,
),
};
process.stdout.write(`${JSON.stringify(summary)}\n`);
}
function isDirectRun() {
const entry = process.argv[1];
if (!entry) return false;
try {
return import.meta.url === pathToFileURL(realpathSync(entry)).href;
} catch {
return false;
}
}
if (isDirectRun()) await main();
+45 -1
View File
@@ -3,7 +3,7 @@ import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js";
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { afterEach, expect, it, vi } from "vitest";
import { mkdir, writeFile, rm, access } from "node:fs/promises";
import { mkdir, writeFile, rm, access, symlink } from "node:fs/promises";
import { join } from "node:path";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import type {
@@ -14,6 +14,7 @@ import type {
} from "playwright-core";
import { browserScenarioSchema } from "../domain/browserScenario.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import {
attachedScenarioLeaseKey,
failBrowserScenarioOperation,
@@ -91,6 +92,49 @@ it("passes the selected environment to the actual browser launch boundary", asyn
});
});
it.skipIf(process.platform === "win32")(
"passes the caller-selected symlink-parent executable to the launch boundary",
async () => {
const root = await createTestTempDirectory("rea-browser-caller-path-");
try {
await mkdir(join(root, "real/inner"), { recursive: true });
await symlink(join(root, "real/inner"), join(root, "link"));
const selected = absoluteCallerPath("link/../browser", { cwd: root });
let launchedExecutable: string | undefined;
const scenario = browserScenarioSchema.parse({
browser: { mode: "launch", executable_path: selected },
start_url: { url: "https://example.test" },
actions: [
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
],
});
await expect(
openPlaywrightScenarioBrowser(
scenario,
{},
{
launcher: {
connectOverCDP: async () => {
throw new Error("Unexpected connection");
},
launchPersistentContext: async (_profile, options) => {
launchedExecutable = options?.executablePath;
throw new Error("fake launch boundary reached");
},
},
},
),
).rejects.toThrow("fake launch boundary reached");
expect(launchedExecutable).toBe(selected);
expect(selected).toBe(`${root}/link/../browser`);
} finally {
await rm(root, { recursive: true, force: true });
}
},
);
it("keeps Termux execution support when launching Chromium on Android", async () => {
Object.defineProperty(process, "platform", { value: "android" });
vi.stubEnv("PREFIX", "/data/data/com.termux/files/usr");
+2 -2
View File
@@ -7,7 +7,7 @@ import {
} from "../artifacts/AnalysisCleanupLease.js";
import { chmod, mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { join } from "node:path";
import type {
Browser,
@@ -717,7 +717,7 @@ export const openPlaywrightScenarioBrowser = async (
try {
await chmod(profilePath, 0o700);
context = await launcher.launchPersistentContext(profilePath, {
executablePath: resolve(scenario.browser.executable_path),
executablePath: scenario.browser.executable_path,
headless: scenario.browser.headless,
acceptDownloads: false,
viewport: {
+19 -4
View File
@@ -24,16 +24,22 @@ import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnal
import { withCommandCancellation } from "./commandCancellation.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import { finishAnalysisCommand } from "./finishAnalysisCommand.js";
import type { ElectronActiveObservationPort } from "../application/javascript/ElectronActiveObservationPort.js";
/** Register CLI equivalents of the Electron MCP tools. */
export const registerElectronCommands = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
environment: Readonly<Record<string, string | undefined>>,
options: {
readonly createActiveProvider?: (
environment: Readonly<Record<string, string | undefined>>,
) => ElectronActiveObservationPort;
} = {},
): void => {
registerElectronObservationCommands(cli, logger);
registerElectronFuseCommand(cli, logger);
registerElectronActiveCommand(cli, logger, environment);
registerElectronActiveCommand(cli, logger, environment, options);
registerJavaScriptApplicationCommand(cli, logger);
registerJavaScriptRuntimeReconciliationCommand(cli, logger);
};
@@ -86,6 +92,11 @@ const registerElectronActiveCommand = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
environment: Readonly<Record<string, string | undefined>>,
options: {
readonly createActiveProvider?: (
environment: Readonly<Record<string, string | undefined>>,
) => ElectronActiveObservationPort;
},
): void => {
cli.command(CLI_COMMANDS.captureElectronScenario, {
description: "Run one owned Electron scenario with cancellable actions",
@@ -122,9 +133,13 @@ const registerElectronActiveCommand = (
);
const { captureElectronScenario } =
await import("../application/javascript/ElectronActiveObservationService.js");
const { createElectronScenarioProvider } =
await import("../composition/electronScenario.js");
const provider = createElectronScenarioProvider(environment);
const provider =
options.createActiveProvider === undefined
? await import("../composition/electronScenario.js").then(
({ createElectronScenarioProvider }) =>
createElectronScenarioProvider(environment),
)
: options.createActiveProvider(environment);
const result = await finishAnalysisCommand(
() => captureElectronScenario(provider, parsed.data, { signal }),
{
+13 -3
View File
@@ -11,6 +11,7 @@ import { projectAnalysisError } from "./domain/analysisErrorProjection.js";
import { BrowserObservationError } from "./domain/browserObservationError.js";
import { analysisInputErrorFromIssues } from "./domain/inputIssueProjection.js";
import { browserScenarioSchema } from "./domain/browserScenario.js";
import type { BrowserScenarioCapturePort } from "./application/BrowserScenarioCapturePort.js";
import type { JsonValue } from "./domain/jsonValue.js";
import type { Logger } from "pino";
@@ -21,6 +22,11 @@ export const registerBrowserScenarioCommands = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
environment: Readonly<Record<string, string | undefined>>,
options: {
readonly createProvider?: (
environment: Readonly<Record<string, string | undefined>>,
) => BrowserScenarioCapturePort;
} = {},
): void => {
cli.command(CLI_COMMANDS.captureBrowserScenario, {
description:
@@ -48,9 +54,13 @@ export const registerBrowserScenarioCommands = (
),
);
// Load Playwright only for this command, not during CLI startup.
const { createBrowserScenarioProvider } =
await import("./composition/browserScenario.js");
const provider = createBrowserScenarioProvider(environment);
const provider =
options.createProvider === undefined
? await import("./composition/browserScenario.js").then(
({ createBrowserScenarioProvider }) =>
createBrowserScenarioProvider(environment),
)
: options.createProvider(environment);
let result:
| Awaited<ReturnType<typeof captureBrowserScenario>>
| undefined;
+2 -2
View File
@@ -1,4 +1,4 @@
import { resolve } from "node:path";
import { absoluteCallerPath } from "./filesystem/callerPath.js";
import { sealOwnedJsonInput } from "./application/OwnedJsonInput.js";
import { readJsonInputFile } from "./application/JsonInputFile.js";
@@ -141,7 +141,7 @@ const resolveJsonPath = (
typeof entry === "string" &&
entry.trim().length > 0 &&
!URL_REFERENCE.test(entry)
? resolve(entry)
? absoluteCallerPath(entry)
: entry,
];
return [key, resolveJsonPath(entry, tail)];
+47 -1
View File
@@ -1,4 +1,6 @@
import { resolve } from "node:path";
import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { describe, expect, it } from "vitest";
@@ -57,6 +59,50 @@ describe("CLI JSON path normalization", () => {
expect(input.browser.executable_path).toBe(" browser ");
});
it.skipIf(process.platform === "win32")(
"keeps POSIX symlink-parent traversal for each selected JSON path",
async () => {
const root = await mkdtemp(join(tmpdir(), "rea-json-caller-path-"));
const originalCwd = process.cwd();
try {
await mkdir(join(root, "real/inner"), { recursive: true });
await symlink(join(root, "real/inner"), join(root, "link"));
process.chdir(root);
const electronInput = {
executable_path: "link/../tool",
application_path: "link/../app",
application_root: "link/../root",
};
const normalizedElectron = resolveCliJsonPaths(electronInput, paths);
expect(normalizedElectron).toEqual({
executable_path: `${root}/link/../tool`,
application_path: `${root}/link/../app`,
application_root: `${root}/link/../root`,
});
expect(electronInput.executable_path).toBe("link/../tool");
expect(
electronActiveObservationInputSchema.safeParse(normalizedElectron)
.success,
).toBe(true);
const browserInput = {
browser: { executable_path: "link/../browser", label: "keep" },
};
expect(
resolveCliJsonPaths(browserInput, [["browser", "executable_path"]]),
).toEqual({
browser: {
executable_path: `${root}/link/../browser`,
label: "keep",
},
});
expect(browserInput.browser.executable_path).toBe("link/../browser");
} finally {
process.chdir(originalCwd);
await rm(root, { recursive: true, force: true });
}
},
);
it.each([
"file:///opt/electron",
"FILE:electron",
@@ -4,6 +4,7 @@ import { expect, it } from "vitest";
import {
semanticStaticPropertyKey,
staticObjectProperty,
staticObjectPropertyNames,
} from "./javascriptAstValues.js";
import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js";
@@ -70,3 +71,188 @@ it("bounds nested inline object spreads without losing a shallow final property"
.status,
).toBe("dynamic");
});
it("collects complete own-key sets through bounded literal spreads", () => {
const names = staticObjectPropertyNames(
t.objectExpression([
t.spreadElement(
t.objectExpression([
t.objectProperty(t.identifier("query"), t.objectExpression([])),
t.objectProperty(t.identifier("shared"), t.numericLiteral(1)),
]),
),
t.objectProperty(t.identifier("direct"), t.objectExpression([])),
t.objectProperty(t.identifier("shared"), t.numericLiteral(2)),
]),
);
expect(names).toEqual({
status: "complete",
names: ["query", "shared", "direct"],
});
});
it("keeps key enumeration unknown when object membership is unresolved", () => {
const dynamicSpread = t.identifier("dynamicProperties");
const computedKey = t.identifier("propertyName");
const cases = [
t.objectExpression([
t.spreadElement(dynamicSpread),
t.objectProperty(t.identifier("direct"), t.objectExpression([])),
]),
t.objectExpression([
t.objectProperty(computedKey, t.objectExpression([]), true),
t.objectProperty(t.identifier("direct"), t.objectExpression([])),
]),
t.objectExpression([
t.spreadElement(
t.objectExpression([
t.objectMethod(
"get",
t.identifier("query"),
[],
t.blockStatement([]),
),
]),
),
]),
];
for (const object of cases)
expect(staticObjectPropertyNames(object).status).toBe("partial");
});
it("retains known keys alongside unresolved object membership", () => {
expect(
staticObjectPropertyNames(
t.objectExpression([
t.spreadElement(t.identifier("dynamicProperties")),
t.objectProperty(t.identifier("direct"), t.objectExpression([])),
t.objectProperty(
t.identifier("propertyName"),
t.objectExpression([]),
true,
),
]),
),
).toEqual({ status: "partial", names: ["direct"] });
});
it("does not claim a complete key set after exceeding spread depth", () => {
let object = t.objectExpression([
t.objectProperty(t.identifier("query"), t.objectExpression([])),
]);
for (let depth = 0; depth < SEMANTIC_EXPRESSION_DEPTH_LIMIT; depth++)
object = t.objectExpression([t.spreadElement(object)]);
expect(staticObjectPropertyNames(object).status).toBe("complete");
expect(
staticObjectPropertyNames(t.objectExpression([t.spreadElement(object)]))
.status,
).toBe("partial");
});
it("enumerates a wide inline spread without a variadic call overflow", () => {
const properties = Array.from({ length: 130_000 }, (_, index) =>
t.objectProperty(t.identifier(`key${index}`), t.numericLiteral(index)),
);
const result = staticObjectPropertyNames(
t.objectExpression([t.spreadElement(t.objectExpression(properties))]),
);
expect(result.status).toBe("complete");
if (result.status === "complete") {
expect(result.names).toHaveLength(properties.length);
expect(result.names[0]).toBe("key0");
expect(result.names.at(-1)).toBe("key129999");
}
});
it("excludes prototype setters from own keys but preserves other __proto__ forms", () => {
const setter = t.objectProperty(
t.identifier("__proto__"),
t.objectExpression([
t.objectProperty(t.identifier("inherited"), t.numericLiteral(1)),
]),
);
expect(
staticObjectPropertyNames(
t.objectExpression([
setter,
t.objectProperty(t.identifier("own"), t.numericLiteral(2)),
]),
),
).toEqual({ status: "complete", names: ["own"] });
const ownForms = [
t.objectProperty(t.stringLiteral("__proto__"), t.numericLiteral(1), true),
t.objectProperty(
t.identifier("__proto__"),
t.identifier("__proto__"),
false,
true,
),
t.objectMethod(
"method",
t.identifier("__proto__"),
[],
t.blockStatement([]),
),
];
for (const member of ownForms)
expect(staticObjectPropertyNames(t.objectExpression([member]))).toEqual({
status: "complete",
names: ["__proto__"],
});
});
it("keeps inherited lookup uncertain after a top-level prototype setter", () => {
const setter = t.objectProperty(
t.identifier("__proto__"),
t.objectExpression([
t.objectProperty(t.identifier("inherited"), t.numericLiteral(1)),
]),
);
expect(
staticObjectProperty(t.objectExpression([setter]), "inherited").status,
).toBe("dynamic");
expect(
staticObjectProperty(
t.objectExpression([
setter,
t.objectProperty(t.identifier("inherited"), t.numericLiteral(2)),
]),
"inherited",
).status,
).toBe("explicit");
expect(
staticObjectProperty(
t.objectExpression([
t.objectProperty(t.identifier("inherited"), t.numericLiteral(2)),
setter,
]),
"inherited",
).status,
).toBe("explicit");
const spreadSource = t.objectExpression([
setter,
t.objectProperty(t.identifier("visible"), t.numericLiteral(3)),
]);
expect(
staticObjectProperty(
t.objectExpression([t.spreadElement(spreadSource)]),
"inherited",
).status,
).toBe("missing");
});
it("omits prototype setters copied by literal object spread", () => {
const source = t.objectExpression([
t.objectProperty(
t.identifier("__proto__"),
t.objectExpression([
t.objectProperty(t.identifier("hidden"), t.numericLiteral(1)),
]),
),
t.objectProperty(t.identifier("visible"), t.numericLiteral(2)),
]);
expect(
staticObjectPropertyNames(t.objectExpression([t.spreadElement(source)])),
).toEqual({ status: "complete", names: ["visible"] });
});
+123 -42
View File
@@ -130,6 +130,79 @@ type StaticObjectProperty =
| { readonly status: "missing" | "dynamic" }
| { readonly status: "explicit"; readonly property: t.ObjectProperty };
type StaticObjectMember =
| { readonly kind: "unknown" }
| { readonly kind: "prototype-setter" }
| {
readonly kind: "property";
readonly property: t.ObjectProperty | t.ObjectMethod;
};
const UNKNOWN_STATIC_OBJECT_MEMBER: StaticObjectMember = { kind: "unknown" };
const isObjectPrototypeSetter = (property: t.ObjectProperty): boolean =>
!property.computed &&
!property.shorthand &&
semanticStaticPropertyKey(property.key, false) === "__proto__";
/** Expand only bounded literal spreads, preserving unknown writes in order. */
const appendStaticObjectMembers = (
members: StaticObjectMember[],
object: t.ObjectExpression,
depth: number,
isNullish?: (node: t.Node) => boolean,
): void => {
for (const property of object.properties) {
if (!t.isSpreadElement(property)) {
if (t.isObjectProperty(property) && isObjectPrototypeSetter(property)) {
// A top-level setter changes lookup inheritance but is never an own
// key; one copied through object spread has no effect on the target.
if (depth === 0) members.push({ kind: "prototype-setter" });
continue;
}
members.push({ kind: "property", property });
continue;
}
const { node: spread, depth: wrappers } = unwrapJavaScriptExpression(
property.argument,
);
if (t.isObjectExpression(spread)) {
const nextDepth = depth + wrappers + 1;
// Copying a getter can run code that changes another source key.
if (
nextDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT ||
spread.properties.some((entry) =>
t.isObjectMethod(entry, { kind: "get" }),
)
) {
members.push(UNKNOWN_STATIC_OBJECT_MEMBER);
continue;
}
appendStaticObjectMembers(members, spread, nextDepth, isNullish);
continue;
}
// These primitive boxes have no own keys; strings can own index keys.
if (
t.isBooleanLiteral(spread) ||
t.isNumericLiteral(spread) ||
t.isBigIntLiteral(spread) ||
isNullish?.(property.argument) === true
)
continue;
members.push(UNKNOWN_STATIC_OBJECT_MEMBER);
}
};
const staticObjectMembers = (
object: t.ObjectExpression,
depth: number,
isNullish?: (node: t.Node) => boolean,
): readonly StaticObjectMember[] => {
const members: StaticObjectMember[] = [];
appendStaticObjectMembers(members, object, depth, isNullish);
return members;
};
/** Find the effective explicit property, preserving unresolved overrides. */
export const staticObjectProperty = (
object: t.Node | undefined,
@@ -138,49 +211,57 @@ export const staticObjectProperty = (
): StaticObjectProperty => {
if (object === undefined) return { status: "missing" };
if (!t.isObjectExpression(object)) return { status: "dynamic" };
const lookup = (
current: t.ObjectExpression,
depth: number,
): StaticObjectProperty => {
for (const property of current.properties.toReversed()) {
if (t.isSpreadElement(property)) {
const { node: spread, depth: wrappers } = unwrapJavaScriptExpression(
property.argument,
);
if (t.isObjectExpression(spread)) {
const nextDepth = depth + wrappers + 1;
// Copying a getter can run code that changes another source key.
if (
nextDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT ||
spread.properties.some((entry) =>
t.isObjectMethod(entry, { kind: "get" }),
)
)
return { status: "dynamic" };
const found = lookup(spread, nextDepth);
if (found.status === "missing") continue;
return found;
}
// These primitive boxes have no own keys; strings can own index keys.
if (
t.isBooleanLiteral(spread) ||
t.isNumericLiteral(spread) ||
t.isBigIntLiteral(spread) ||
isNullish?.(property.argument) === true
)
continue;
return { status: "dynamic" };
}
const key = semanticStaticPropertyKey(property.key, property.computed);
if (key === null) return { status: "dynamic" };
if (key !== name) continue;
return t.isObjectProperty(property)
? { status: "explicit", property }
: { status: "dynamic" };
let prototypeMaySupply = false;
for (const member of staticObjectMembers(object, 0, isNullish).toReversed()) {
if (member.kind === "prototype-setter") {
prototypeMaySupply = true;
continue;
}
return { status: "missing" };
};
return lookup(object, 0);
if (member.kind === "unknown") return { status: "dynamic" };
const { property } = member;
const key = semanticStaticPropertyKey(property.key, property.computed);
if (key === null) return { status: "dynamic" };
if (key !== name) continue;
return t.isObjectProperty(property)
? { status: "explicit", property }
: { status: "dynamic" };
}
return prototypeMaySupply ? { status: "dynamic" } : { status: "missing" };
};
export type StaticObjectPropertyNames =
| { readonly status: "missing" | "dynamic" }
| { readonly status: "partial"; readonly names: readonly string[] }
| { readonly status: "complete"; readonly names: readonly string[] };
/** Retain known own names and report unresolved object membership separately. */
export const staticObjectPropertyNames = (
object: t.Node | undefined,
isNullish?: (node: t.Node) => boolean,
): StaticObjectPropertyNames => {
if (object === undefined) return { status: "missing" };
if (!t.isObjectExpression(object)) return { status: "dynamic" };
const names = new Set<string>();
let complete = true;
for (const member of staticObjectMembers(object, 0, isNullish)) {
if (member.kind === "prototype-setter") continue;
if (member.kind === "unknown") {
complete = false;
continue;
}
const key = semanticStaticPropertyKey(
member.property.key,
member.property.computed,
);
if (key === null) {
complete = false;
continue;
}
names.add(key);
}
return complete
? { status: "complete", names: [...names] }
: { status: "partial", names: [...names] };
};
/** Later properties own a key; an intervening dynamic write leaves it unknown. */
+11 -1
View File
@@ -220,7 +220,17 @@ export const webBundleAnalysisSchema = z.object({
z.object({
name: z.string().nullable(),
description: z.string().nullable(),
schema_property_names: z.array(z.string()),
schema_property_names: z
.array(z.string())
.describe(
"Statically known own property names from the declared schema projection; a partial list may omit unresolved members.",
),
schema_property_names_coverage: z
.enum(["complete", "partial", "unknown", "not_present"])
.optional()
.describe(
"Coverage of the schema property-name projection. Missing means unknown for historical observations. Not a schema-validity result.",
),
trust: z.literal("page-declared-untrusted"),
location: sourceLocationSchema,
}),
+357 -16
View File
@@ -158,6 +158,232 @@ it("discloses widened returned-reference effects only for the affected script",
).toEqual(["/candidate"]);
});
it.each([
{
label: "inline declaration and schema spreads",
source: `document.modelContext.registerTool({
...{
name: "spread-tool",
description: "Copied from a literal",
inputSchema: {
...{
type: "object",
properties: { query: { type: "string" } }
}
}
}
});`,
name: "spread-tool",
description: "Copied from a literal",
properties: ["query"],
coverage: "complete",
},
{
label: "unrelated trailing literal spread",
source: `document.modelContext.registerTool({
name: "direct-tool",
description: "Explicit metadata",
inputSchema: { properties: { query: { type: "string" } } },
...{ timeout: 5 }
});`,
name: "direct-tool",
description: "Explicit metadata",
properties: ["query"],
coverage: "complete",
},
{
label: "null spread after explicit metadata",
source: `document.modelContext.registerTool({
name: "null-safe-tool",
description: "Null spread is inert",
inputSchema: { properties: { query: { type: "string" } } },
...null
});`,
name: "null-safe-tool",
description: "Null spread is inert",
properties: ["query"],
coverage: "complete",
},
{
label: "snake-case schema property from a spread",
source: `document.modelContext.registerTool({
...{
name: "snake-tool",
description: "Snake-case schema",
input_schema: { properties: { item_id: { type: "string" } } }
}
});`,
name: "snake-tool",
description: "Snake-case schema",
properties: ["item_id"],
coverage: "complete",
},
])(
"preserves WebMCP facts through $label",
({ source, name, description, properties, coverage }) => {
const result = analyzeCapturedWebBundle(inspection(source));
expect(result.observations.webmcp_declarations).toEqual([
expect.objectContaining({
name,
description,
schema_property_names: properties,
schema_property_names_coverage: coverage,
trust: "page-declared-untrusted",
}),
]);
expect(result.completeness.status).toBe("complete");
},
);
it.each([
{
label: "unknown spread after explicit facts",
source: `document.modelContext.registerTool({
name: "possibly-overridden",
description: "Possibly overridden",
inputSchema: { properties: { query: { type: "string" } } },
...dynamicOptions
});`,
name: null,
description: null,
properties: [],
coverage: "unknown",
},
{
label: "explicit facts after an unknown spread",
source: `document.modelContext.registerTool({
...dynamicOptions,
name: "known-after-spread",
description: "Explicitly restored",
inputSchema: { properties: { query: { type: "string" } } }
});`,
name: "known-after-spread",
description: "Explicitly restored",
properties: ["query"],
coverage: "complete",
},
])(
"preserves rightmost uncertainty for $label",
({ source, name, description, properties, coverage }) => {
const result = analyzeCapturedWebBundle(inspection(source));
expect(result.observations.webmcp_declarations).toEqual([
expect.objectContaining({
name,
description,
schema_property_names: properties,
schema_property_names_coverage: coverage,
trust: "page-declared-untrusted",
}),
]);
expect(result.completeness.status).toBe("complete");
},
);
it.each([
{
label: "an unknown camel-case schema override",
source: `document.modelContext.registerTool({
...dynamicOptions,
inputSchema: { properties: { camel: {} } },
...anotherDynamicOptions,
input_schema: { properties: { snake: {} } },
name: "tool",
description: "Explicit metadata"
});`,
name: "tool",
properties: [],
coverage: "unknown",
},
{
label: "an explicit null camel-case schema",
source: `document.modelContext.registerTool({
name: "null-alias",
description: "Fallback applies",
inputSchema: null,
input_schema: { properties: { snake: {} } }
});`,
name: "null-alias",
properties: ["snake"],
coverage: "complete",
},
{
label: "literal property spreads",
source: `document.modelContext.registerTool({
name: "spread-properties",
inputSchema: { properties: { ...{ query: {} }, direct: {} } }
});`,
name: "spread-properties",
properties: ["direct", "query"],
coverage: "complete",
},
{
label: "prototype setters are not schema properties",
source: `document.modelContext.registerTool({
name: "prototype-properties",
inputSchema: { properties: { __proto__: { hidden: {} }, direct: {}, ["__proto__"]: {} } }
});`,
name: "prototype-properties",
properties: ["__proto__", "direct"],
coverage: "complete",
},
{
label: "own schema properties before a prototype setter",
source: `document.modelContext.registerTool({
name: "own-before-prototype",
inputSchema: { properties: { known: {} }, __proto__: { properties: { inherited: {} } } }
});`,
name: "own-before-prototype",
properties: ["known"],
coverage: "complete",
},
{
label: "own schema properties after a prototype setter",
source: `document.modelContext.registerTool({
name: "own-after-prototype",
inputSchema: { __proto__: { properties: { inherited: {} } }, properties: { known: {} } }
});`,
name: "own-after-prototype",
properties: ["known"],
coverage: "complete",
},
{
label: "an unresolved property spread",
source: `document.modelContext.registerTool({
name: "unknown-properties",
inputSchema: { properties: { ...dynamicProperties, direct: {} } }
});`,
name: "unknown-properties",
properties: ["direct"],
coverage: "partial",
},
{
label: "an unresolved computed property key",
source: `document.modelContext.registerTool({
name: "computed-properties",
inputSchema: { properties: { [propertyName]: {}, direct: {} } }
});`,
name: "computed-properties",
properties: ["direct"],
coverage: "partial",
},
])(
"preserves WebMCP schema ownership for $label",
({ source, name, properties, coverage }) => {
const result = analyzeCapturedWebBundle(inspection(source));
expect(result.observations.webmcp_declarations).toEqual([
expect.objectContaining({
name,
schema_property_names: properties,
schema_property_names_coverage: coverage,
trust: "page-declared-untrusted",
}),
]);
},
);
describe("web bundle analyzer", () => {
it("extracts graph, route, endpoint, vendor, and WebMCP evidence", () => {
const source = `
@@ -206,6 +432,7 @@ describe("web bundle analyzer", () => {
name: "lookup-user",
trust: "page-declared-untrusted",
schema_property_names: ["userId", "verbose"],
schema_property_names_coverage: "complete",
}),
]);
expect(result.inferences).toEqual(
@@ -266,6 +493,7 @@ describe("web bundle analyzer", () => {
name,
description,
schema_property_names: [propertyName],
schema_property_names_coverage: "complete",
trust: "page-declared-untrusted",
});
});
@@ -1481,10 +1709,24 @@ describe("web metadata key semantics", () => {
expect.objectContaining({
name: "new",
schema_property_names: ["literal"],
schema_property_names_coverage: "partial",
}),
expect.objectContaining({
name: null,
schema_property_names_coverage: "unknown",
}),
expect.objectContaining({
name: null,
schema_property_names_coverage: "not_present",
}),
expect.objectContaining({
name: "restored",
schema_property_names_coverage: "unknown",
}),
expect.objectContaining({
name: "valid",
schema_property_names_coverage: "not_present",
}),
expect.objectContaining({ name: null }),
expect.objectContaining({ name: "restored" }),
expect.objectContaining({ name: "valid" }),
]);
});
@@ -1514,20 +1756,86 @@ describe("web metadata key semantics", () => {
expect(
result.observations.webmcp_declarations.map(
({ name, schema_property_names }) => ({ name, schema_property_names }),
({ name, schema_property_names, schema_property_names_coverage }) => ({
name,
schema_property_names,
schema_property_names_coverage,
}),
),
).toEqual([
{ name: "literal", schema_property_names: ["", "properties", "type"] },
{ name: "missing properties", schema_property_names: [] },
{ name: "dynamic properties", schema_property_names: [] },
{ name: "null properties", schema_property_names: [] },
{ name: "missing schema", schema_property_names: [] },
{ name: "dynamic schema", schema_property_names: [] },
{ name: "null schema", schema_property_names: [] },
{ name: "spread schema", schema_property_names: [] },
{ name: "computed literal keys", schema_property_names: ["", "literal"] },
{
name: "literal",
schema_property_names: ["", "properties", "type"],
schema_property_names_coverage: "partial",
},
{
name: "missing properties",
schema_property_names: [],
schema_property_names_coverage: "not_present",
},
{
name: "dynamic properties",
schema_property_names: [],
schema_property_names_coverage: "unknown",
},
{
name: "null properties",
schema_property_names: [],
schema_property_names_coverage: "unknown",
},
{
name: "missing schema",
schema_property_names: [],
schema_property_names_coverage: "not_present",
},
{
name: "dynamic schema",
schema_property_names: [],
schema_property_names_coverage: "unknown",
},
{
name: "null schema",
schema_property_names: [],
schema_property_names_coverage: "not_present",
},
{
name: "spread schema",
schema_property_names: [],
schema_property_names_coverage: "unknown",
},
{
name: "computed literal keys",
schema_property_names: ["", "literal"],
schema_property_names_coverage: "partial",
},
]);
});
it("keeps missing coverage absent when parsing historical observations", () => {
const analyzed = analyzeCapturedWebBundle(
inspection(
`document.modelContext.registerTool({ name: "legacy", inputSchema: { properties: {} } });`,
),
);
const declaration = analyzed.observations.webmcp_declarations[0];
expect(declaration).toBeDefined();
if (declaration === undefined) throw new Error("Expected declaration");
const { schema_property_names_coverage: _coverage, ...legacyDeclaration } =
declaration;
const parsed = webBundleAnalysisSchema.parse({
...analyzed,
observations: {
...analyzed.observations,
webmcp_declarations: [legacyDeclaration],
},
});
expect(
Object.hasOwn(
parsed.observations.webmcp_declarations[0] ?? {},
"schema_property_names_coverage",
),
).toBe(false);
});
});
it("retains WebMCP declarations through optional member and call chaining", () => {
@@ -1695,6 +2003,8 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen
document.modelContext.registerTool({ name: "lookup", description: "second", inputSchema: { properties: { alpha: {} } } });
document.modelContext.registerTool({ name: "equivalent", inputSchema: { properties: { alpha: {}, beta: {} } } });
document.modelContext.registerTool({ name: "equivalent", inputSchema: { properties: { beta: {}, alpha: {}, alpha: {} } } });
document.modelContext.registerTool({ name: "empty-status", inputSchema: { properties: {} } });
document.modelContext.registerTool({ name: "empty-status", inputSchema: { properties: dynamicProperties } });
document.modelContext.registerTool({ description: "unnamed" });
document.modelContext.registerTool({ name: "", description: "unnamed" });
`),
@@ -1702,10 +2012,16 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen
expect(
result.observations.webmcp_declarations.map(
({ name, description, schema_property_names }) => ({
({
name,
description,
schema_property_names,
schema_property_names_coverage,
}) => ({
name,
description,
schema_property_names,
schema_property_names_coverage,
}),
),
).toEqual([
@@ -1713,19 +2029,44 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen
name: "lookup",
description: "first",
schema_property_names: ["alpha"],
schema_property_names_coverage: "complete",
},
{
name: "lookup",
description: "second",
schema_property_names: ["alpha"],
schema_property_names_coverage: "complete",
},
{
name: "equivalent",
description: null,
schema_property_names: ["alpha", "beta"],
schema_property_names_coverage: "complete",
},
{
name: "empty-status",
description: null,
schema_property_names: [],
schema_property_names_coverage: "complete",
},
{
name: "empty-status",
description: null,
schema_property_names: [],
schema_property_names_coverage: "unknown",
},
{
name: null,
description: "unnamed",
schema_property_names: [],
schema_property_names_coverage: "not_present",
},
{
name: "",
description: "unnamed",
schema_property_names: [],
schema_property_names_coverage: "not_present",
},
{ name: null, description: "unnamed", schema_property_names: [] },
{ name: "", description: "unnamed", schema_property_names: [] },
]);
expect(result.observations.webmcp_declarations[2]?.location).toEqual({
script_key: `scr_${"1".repeat(64)}`,
+5 -15
View File
@@ -2,28 +2,18 @@ import * as t from "@babel/types";
import { sanitizeBrowserUrl } from "./browserObservation.js";
import {
isNullishJavaScriptSyntax,
readExactJavaScriptLiteral,
semanticStaticPropertyName,
staticObjectProperty,
} from "./javascript/javascriptAstValues.js";
/** Pick the endpoint argument for common network-call patterns. */
/** Read a statically known own property value from an object expression. */
export const objectValue = (
object: t.ObjectExpression,
name: string,
): t.ObjectProperty["value"] | undefined => {
let value: t.ObjectProperty["value"] | undefined;
for (const property of object.properties) {
if (t.isSpreadElement(property)) {
value = undefined;
continue;
}
const key = semanticStaticPropertyName(property.key, property.computed);
if (key === "" && !t.isStringLiteral(property.key)) value = undefined;
else if (key === name)
value = t.isObjectProperty(property) ? property.value : undefined;
}
return value;
const result = staticObjectProperty(object, name, isNullishJavaScriptSyntax);
return result.status === "explicit" ? result.property.value : undefined;
};
export const objectString = (
+53 -14
View File
@@ -12,6 +12,9 @@ import { isIntrinsicMapLookup } from "./javascript/javascriptSemanticMapOperatio
import {
semanticStaticPropertyKey,
semanticStaticPropertyName,
isNullishJavaScriptSyntax,
staticObjectProperty,
staticObjectPropertyNames,
unwrapJavaScriptExpression,
} from "./javascript/javascriptAstValues.js";
import {
@@ -23,7 +26,6 @@ import {
location,
locationFields,
objectString,
objectValue,
resolveSpecifier,
isUrlLikeModuleSpecifier,
} from "./webBundleAnalyzerAst.js";
@@ -365,23 +367,60 @@ const addWebMcpDeclaration = (
if (!t.isObjectExpression(declaration)) return;
const name = objectString(declaration, "name");
const description = objectString(declaration, "description");
const schema =
objectValue(declaration, "inputSchema") ??
objectValue(declaration, "input_schema");
const properties = t.isObjectExpression(schema)
? objectValue(schema, "properties")
: undefined;
const schemaPropertyNames = t.isObjectExpression(properties)
? properties.properties.flatMap((property) => {
if (!t.isObjectProperty(property)) return [];
const key = semanticStaticPropertyKey(property.key, property.computed);
return key === null ? [] : [key];
})
: [];
const camelSchema = staticObjectProperty(
declaration,
"inputSchema",
isNullishJavaScriptSyntax,
);
const schemaProperty =
camelSchema.status === "missing" ||
(camelSchema.status === "explicit" &&
isNullishJavaScriptSyntax(camelSchema.property.value))
? staticObjectProperty(
declaration,
"input_schema",
isNullishJavaScriptSyntax,
)
: camelSchema;
let schemaPropertyNames: readonly string[] = [];
let schemaPropertyNamesCoverage: NonNullable<
WebMcpDeclaration["schema_property_names_coverage"]
> = "unknown";
if (schemaProperty.status === "missing") {
schemaPropertyNamesCoverage = "not_present";
} else if (schemaProperty.status === "explicit") {
const schema = schemaProperty.property.value;
if (t.isObjectExpression(schema)) {
const propertiesProperty = staticObjectProperty(
schema,
"properties",
isNullishJavaScriptSyntax,
);
if (propertiesProperty.status === "missing") {
schemaPropertyNamesCoverage = "not_present";
} else if (propertiesProperty.status === "explicit") {
const properties = propertiesProperty.property.value;
if (t.isObjectExpression(properties)) {
const projection = staticObjectPropertyNames(
properties,
isNullishJavaScriptSyntax,
);
if (
projection.status === "complete" ||
projection.status === "partial"
) {
schemaPropertyNames = projection.names;
schemaPropertyNamesCoverage = projection.status;
}
}
}
}
}
const metadata = {
name: name ?? null,
description: description ?? null,
schema_property_names: [...new Set(schemaPropertyNames)].sort(),
schema_property_names_coverage: schemaPropertyNamesCoverage,
};
// Deduplicate the metadata we return, preserving omission and descriptions.
// Property names are a variable-length source-derived list: NUL-joins
@@ -0,0 +1,62 @@
import { mkdir, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { toolContract } from "../../src/contracts/toolContracts.js";
import { parseEvidence } from "../../src/domain/evidence.js";
import { analysisViewResultSchema } from "../../src/domain/analysisView/analysisView.js";
import {
analysisViewJavaScriptEvidence,
analysisViewLayoutEvidence,
} from "../fixtures/analysisView.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
cliTest
.skipIf(process.platform === "win32")
.for(["relative", "absolute"] as const)(
"reads selected analysis-view Evidence through a %s caller path containing a symlink parent",
async (pathKind, { cli }) => {
const root = await createTestTempDirectory(
"rea-analysis-view-caller-path-",
);
await mkdir(join(root, "real", "inner"), { recursive: true });
await symlink("real/inner", join(root, "link"));
const selected = analysisViewJavaScriptEvidence();
const decoy = analysisViewLayoutEvidence();
expect(selected.evidence_id).not.toBe(decoy.evidence_id);
await writeFile(
join(root, "real", "evidence.json"),
JSON.stringify(selected),
);
await writeFile(join(root, "evidence.json"), JSON.stringify(decoy));
const sourcePath =
pathKind === "relative"
? "link/../evidence.json"
: `${root}/link/../evidence.json`;
const queryPath = join(root, "query.json");
await writeFile(
queryPath,
JSON.stringify({
source: { kind: "evidence-file", path: sourcePath },
view: { kind: "summary" },
}),
);
const result = await cli.run({
arguments: ["inspect-analysis-view", queryPath, "--json"],
cwd: root,
});
expect(result.exitCode, result.stdout + result.stderr).toBe(0);
const contractResult = toolContract(
"inspect_analysis_view",
).outputSchema.parse(result.json);
const output = parseEvidence(contractResult);
const view = analysisViewResultSchema.parse(output.normalized_result);
expect(view.kind).toBe("summary");
expect(view.parent_evidence_id).toBe(selected.evidence_id);
expect(output.evidence_links).toEqual([selected.evidence_id]);
},
);
@@ -104,12 +104,12 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
};
let traceEvidence: unknown;
if (surface === "CLI") {
// The application Evidence document exceeds the runner argument limit
// when it is placed on argv. The command reads the same JSON from a file.
const traceInput = join(root, "trace-input.json");
await writeFile(traceInput, JSON.stringify(traceArguments));
const response = await cli.run({
arguments: [
"trace-application-feature",
JSON.stringify(traceArguments),
"--json",
],
arguments: ["trace-application-feature", traceInput, "--json"],
});
expect(response.exitCode, response.stderr).toBe(0);
traceEvidence = response.json;
@@ -0,0 +1,401 @@
import { expect } from "vitest";
import { aliasSelectionBudgetSource } from "../../fixtures/javascriptAliasSelectionBudget.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import type { TestCli } from "../../support/cli/cliFixture.js";
import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js";
import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js";
const fixtureImports = (loader: string) => {
switch (loader) {
case "pattern alias":
return { "#window/*": "./proxy/*.cjs" };
case "recursive alias":
return { "#entry": "#window", "#window": "./proxy/window.cjs" };
case "external-package alias":
return { "#entry": "fixture-window" };
case "builtin alias":
return { "#entry": "fs" };
case "electron alias":
return { "#entry": "electron" };
case "electron main alias":
return { "#entry": "electron/main" };
default:
return { "#entry": null };
}
};
const unprovenLoaderSources: Readonly<Record<string, string>> = {
"custom iterator assignment module loader":
'const values = { *[Symbol.iterator]() { yield module; } }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");',
"replaced iterator assignment module loader":
'const values = [{}]; values[Symbol.iterator] = function* () { yield module; }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");',
"custom iterator rest declaration module loader":
'const values = { *[Symbol.iterator]() { yield module; } }; const [...holder] = values; holder[0].require("../shared/window.cjs");',
"native array assignment local loader":
'const holder = { loader: module }; [holder.loader] = [{ require() {} }]; holder.loader.require("../shared/window.cjs");',
"logical rewrite assignment module loader":
'const holder = {}; holder.loader ??= module; holder.loader.require = replacement; require("../shared/window.cjs");',
"object rest declaration module loader":
'const { ...holder } = { loader: module }; holder.loader.require("../shared/window.cjs");',
"array rest declaration module loader":
'const [...holder] = [module]; holder[0].require("../shared/window.cjs");',
"object assignment module loader":
'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require("../shared/window.cjs");',
"array assignment module loader":
'const holder = {}; [holder.loader] = [module]; holder.loader.require("../shared/window.cjs");',
"nested assignment module loader":
'const holder = {}; ({ inner: { loader: holder.loader } } = { inner: { loader: module } }); holder.loader.require("../shared/window.cjs");',
"default assignment module loader":
'const holder = {}; ({ loader: holder.loader = module } = {}); holder.loader.require("../shared/window.cjs");',
"rest array assignment module loader":
'const holder = {}; let ignored; [ignored, ...holder.loaders] = [{}, module]; holder.loaders[0].require("../shared/window.cjs");',
"delegate assignment module loader":
'const holder = {}; ({ load: holder.load } = { load: module.require }); holder.load.call(module, "../shared/window.cjs");',
"rewrite assignment module loader":
'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require = replacement; require("../shared/window.cjs");',
"exports assignment local loader":
'const holder = {}; ({ exports: holder.loader } = module); holder.loader.require("../shared/window.cjs");',
"rest assignment local loader":
'const holder = {}; ({ ...holder.loader } = module); holder.loader.require("../shared/window.cjs");',
"excluded assignment local loader":
'const holder = {}; let ignored; ({ loader: ignored, ...holder.copy } = { loader: module }); holder.copy.loader.require("../shared/window.cjs");',
"inactive default assignment local loader":
'const holder = {}; ({ loader: holder.loader = module } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");',
"overwritten assignment local loader":
'const holder = { loader: module }; ({ loader: holder.loader } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");',
"bounded alias selection": aliasSelectionBudgetSource(),
"captured module delegate":
'const load = module.require; load.call(module, "../shared/window.cjs");',
"destructured module delegate":
'const { require: load } = module; load.apply(module, ["../shared/window.cjs"]);',
"selected local delegate slots":
'const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement; require("../shared/window.cjs");',
"local assigned loader":
'const holder = {}; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");',
"overwritten local loader":
'const holder = { loader: module }; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");',
"normalized absolute ESM": 'import "/discard%2F/../shared/window.cjs";',
"normalized absolute dynamic import":
'import("/discard%5C/../shared/window.cjs");',
"rewritten CommonJS delegate":
'module.require = replacement; require("../shared/window.cjs");',
"nested rewritten CommonJS delegate":
'const holder = { loader: module }; holder.loader.require = replacement; require("../shared/window.cjs");',
"assigned rewritten CommonJS delegate":
'const holder = {}; holder.loader = module; holder.loader.require = replacement; require("../shared/window.cjs");',
"assigned module loader":
'const holder = {}; holder.loader = module; holder.loader.require("../shared/window.cjs");',
"nested module loader":
'const holder = { loader: module }; holder.loader.require("../shared/window.cjs");',
"destructured module loader":
'const holder = { loader: module }; const { loader } = holder; loader.require("../shared/window.cjs");',
"array module loader":
'const holder = [module]; holder[0].require("../shared/window.cjs");',
"bound module loader":
'const load = module.require.bind(module); load("../shared/window.cjs");',
"bound module alias loader":
'const current = module; const load = current.require.bind(current); load("../shared/window.cjs");',
"computed bound module loader":
'const load = module["require"].bind(module); load("../shared/window.cjs");',
"fallback module alias":
'const { loader = module } = options; loader.require("../shared/window.cjs");',
"inactive fallback":
'const { loader = module } = { loader: {} }; loader.require("../shared/window.cjs");',
"absolute ESM": 'import "/shared/window.cjs";',
"invalid absolute ESM": 'import "/shared%2Fwindow.cjs";',
"invalid absolute dynamic import": 'import("/shared%5Cwindow.cjs");',
"absolute dynamic import": 'import("/shared/window.cjs");',
"absolute file URL": 'import "file:///shared/window.cjs";',
"dynamic file URL": 'import("file:///shared/window.cjs");',
"createRequire base":
'import { createRequire } from "node:module"; const require = createRequire(new URL("../other/nested/main.cjs", import.meta.url)); require("../shared/window.cjs");',
};
const uncheckedModuleLoaders = new Set([
"custom iterator assignment module loader",
"replaced iterator assignment module loader",
"custom iterator rest declaration module loader",
"logical rewrite assignment module loader",
"object rest declaration module loader",
"array rest declaration module loader",
"object assignment module loader",
"array assignment module loader",
"nested assignment module loader",
"default assignment module loader",
"rest array assignment module loader",
"delegate assignment module loader",
"rewrite assignment module loader",
"bounded alias selection",
"captured module delegate",
"destructured module delegate",
"rewritten CommonJS delegate",
"nested rewritten CommonJS delegate",
"assigned rewritten CommonJS delegate",
"assigned module loader",
"module.require",
"computed module.require",
"module alias",
"later reassigned module alias",
"conditional module alias",
"fallback module alias",
"bound module loader",
"bound module alias loader",
"computed bound module loader",
"nested module loader",
"destructured module loader",
"array module loader",
]);
const fixtureMainPath = (loader: string): string => {
switch (loader) {
case "absolute ESM":
case "normalized absolute ESM":
case "normalized absolute dynamic import":
case "invalid absolute ESM":
case "invalid absolute dynamic import":
case "absolute dynamic import":
case "absolute file URL":
case "dynamic file URL":
case "createRequire base":
return "main.mjs";
case "absolute main":
return "/opt/app/main.cjs";
case "Windows relative main":
return "sub\\main.cjs";
case "Windows drive-relative main":
return "C:main.cjs";
case "percent main":
case "single percent main":
return "main%2Fentry.cjs";
default:
return "main.cjs";
}
};
const fixtureMainInventoryPath = (loader: string, main: string): string => {
if (loader === "absolute main") return "opt/app/main.cjs";
if (loader === "Windows relative main") return "b/sub/main.cjs";
if (loader === "Windows drive-relative main") return "b/main.cjs";
return `b/${main}`;
};
const fixtureLoaderSource = (loader: string): string =>
unprovenLoaderSources[loader] ??
(loader === "resolve-only"
? 'require.resolve("../shared/window.cjs");'
: loader === "missing wildcard alias"
? 'require("#missing/*");'
: loader === "absolute require"
? 'require("/shared/window.cjs");'
: loader === "projected module"
? 'const { exports: api } = module; api.require = () => {}; api.require("../shared/window.cjs");'
: loader === "absolute main"
? 'require("../../shared/window.cjs");'
: loader === "later reassigned module alias"
? 'let current = module; current.require("../shared/window.cjs"); current = other;'
: loader === "earlier reassigned module alias"
? 'let current = module; current = { require() {} }; current.require("../shared/window.cjs");'
: loader === "conditional module alias"
? 'let current = module; if (flag) current = other; current.require("../shared/window.cjs");'
: loader === "module.require"
? 'module.require("../shared/window.cjs");'
: loader === "computed module.require"
? 'module["require"]("../shared/window.cjs");'
: loader === "module alias"
? 'const current = module; current.require("../shared/window.cjs");'
: loader === "local module"
? 'function local(module) { module.require("../shared/window.cjs"); }'
: loader === "pattern alias"
? 'require("#window/window");'
: loader.endsWith("alias")
? 'require("#entry");'
: 'require("../shared/window.cjs");');
const explicitCompetingLoaders = [
"module.require",
"computed module.require",
"module alias",
"later reassigned module alias",
"earlier reassigned module alias",
"conditional module alias",
"absolute main",
"Windows relative main",
"Windows drive-relative main",
"percent main",
"pattern alias",
"recursive alias",
"external-package alias",
"builtin alias",
"electron alias",
"electron main alias",
"blocked alias",
"local module",
"projected module",
"resolve-only",
"missing wildcard alias",
"absolute require",
] as const;
/** Every competing-loader case, in the order the original matrix ran. */
export const competingLoaderNames = [
...explicitCompetingLoaders,
...Object.keys(unprovenLoaderSources),
"single percent main",
];
const groupSize = Math.ceil(competingLoaderNames.length / 3);
/** Three disjoint groups so Vitest can run the matrix on separate workers. */
export const competingLoaderGroups = [
competingLoaderNames.slice(0, groupSize),
competingLoaderNames.slice(groupSize, groupSize * 2),
competingLoaderNames.slice(groupSize * 2),
] as const;
/** One public CLI or MCP observation for a competing loader fixture. */
export const assertCompetingLoaderOwnership = async (
surface: "CLI" | "stdio MCP",
loader: string,
cli: TestCli,
): Promise<void> => {
const root = await createTestTempDirectory("rea-loader-owner-");
const single = loader === "single percent main";
const inert = [
"native array assignment local loader",
"exports assignment local loader",
"rest assignment local loader",
"excluded assignment local loader",
"inactive default assignment local loader",
"overwritten assignment local loader",
"local assigned loader",
"overwritten local loader",
"invalid absolute ESM",
"invalid absolute dynamic import",
"builtin alias",
"electron alias",
"electron main alias",
"inactive fallback",
"blocked alias",
"local module",
"projected module",
"resolve-only",
"missing wildcard alias",
"earlier reassigned module alias",
].includes(loader);
const absolute = loader === "absolute main";
const main = fixtureMainPath(loader);
const imports = fixtureImports(loader);
const load = fixtureLoaderSource(loader);
await writeFixtureFiles(root, {
"a/package.json": JSON.stringify({ main: "main.cjs" }),
"a/main.cjs":
single || absolute
? "module.exports = 1;"
: 'require("../shared/window.cjs");',
"b/package.json": JSON.stringify({ main, imports }),
[fixtureMainInventoryPath(loader, main)]: load,
"b/proxy/window.cjs": 'require("../../shared/window.cjs");',
"b/node_modules/fixture-window/package.json": JSON.stringify({
main: "index.cjs",
}),
"b/node_modules/fixture-window/index.cjs":
'require("../../../shared/window.cjs");',
"shared/window.cjs":
'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");',
"a/index.html": "first application page",
"b/index.html": "second application page",
"shared/index.html": "source-relative decoy",
"index.html": "artifact-root decoy",
});
const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);
if (loader.startsWith("Windows "))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "loads",
properties: expect.objectContaining({
declared_path: main,
resolution_status: "unavailable",
resolved_path: null,
}),
}),
]),
);
if (
["absolute require", "absolute ESM", "absolute dynamic import"].includes(
loader,
)
)
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
specifier: "/shared/window.cjs",
resolution_status: "unavailable",
resolved_path: null,
}),
}),
]),
);
if (loader === "createRequire base")
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
kind: "require",
specifier: null,
resolved_path: null,
}),
}),
]),
);
if (loader.endsWith("alias") && !inert && !loader.includes("module alias"))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
specifier: loader === "pattern alias" ? "#window/window" : "#entry",
resolution_status: "unavailable",
}),
}),
]),
);
if (uncheckedModuleLoaders.has(loader))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
kind: "require",
specifier: null,
resolved_path: null,
}),
}),
]),
);
const roles = graph.nodes.filter(({ kind }) => kind === "electron-renderer");
expect(roles).toHaveLength(1);
const role = roles[0];
const target = single ? "b/index.html" : inert ? "a/index.html" : null;
expect(role?.observations).toEqual(
expect.arrayContaining([
expect.objectContaining({
properties: expect.objectContaining({
resolved_path: target,
resolution_status: target === null ? "unavailable" : "resolved",
}),
}),
]),
);
expect(
graph.edges
.filter(
({ source_node_id, relation }) =>
source_node_id === role?.node_id && relation === "maps_to",
)
.map(({ properties }) => properties.resolved_path),
).toEqual(target === null ? [] : [target]);
};
@@ -0,0 +1,29 @@
import { expect, it } from "vitest";
import { cliTest } from "../../support/cli/cliFixture.js";
import {
assertCompetingLoaderOwnership,
competingLoaderGroups,
competingLoaderNames,
} from "./javascriptLiteralLoaderOwnership.js";
const loaders = competingLoaderGroups[0];
it("partitions every competing loader across the three ownership files", () => {
const assigned = competingLoaderGroups.flat();
expect(assigned).toEqual(competingLoaderNames);
expect(new Set(assigned).size).toBe(competingLoaderNames.length);
for (const group of competingLoaderGroups)
expect(group.length).toBeGreaterThan(0);
});
cliTest.for(
(["CLI", "stdio MCP"] as const).flatMap((surface) =>
loaders.map((loader) => ({ surface, loader })),
),
)(
"preserves competing loader ownership for $loader through $surface",
async ({ surface, loader }, { cli }) => {
await assertCompetingLoaderOwnership(surface, loader, cli);
},
);
@@ -0,0 +1,18 @@
import { cliTest } from "../../support/cli/cliFixture.js";
import {
assertCompetingLoaderOwnership,
competingLoaderGroups,
} from "./javascriptLiteralLoaderOwnership.js";
const loaders = competingLoaderGroups[2];
cliTest.for(
(["CLI", "stdio MCP"] as const).flatMap((surface) =>
loaders.map((loader) => ({ surface, loader })),
),
)(
"preserves competing loader ownership for $loader through $surface",
async ({ surface, loader }, { cli }) => {
await assertCompetingLoaderOwnership(surface, loader, cli);
},
);
@@ -0,0 +1,18 @@
import { cliTest } from "../../support/cli/cliFixture.js";
import {
assertCompetingLoaderOwnership,
competingLoaderGroups,
} from "./javascriptLiteralLoaderOwnership.js";
const loaders = competingLoaderGroups[1];
cliTest.for(
(["CLI", "stdio MCP"] as const).flatMap((surface) =>
loaders.map((loader) => ({ surface, loader })),
),
)(
"preserves competing loader ownership for $loader through $surface",
async ({ surface, loader }, { cli }) => {
await assertCompetingLoaderOwnership(surface, loader, cli);
},
);
@@ -4,30 +4,10 @@ import { join } from "node:path";
import { createPackageWithOptions } from "@electron/asar";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { aliasSelectionBudgetSource } from "../../fixtures/javascriptAliasSelectionBudget.js";
import { cliTest } from "../../support/cli/cliFixture.js";
import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js";
import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js";
const fixtureImports = (loader: string) => {
switch (loader) {
case "pattern alias":
return { "#window/*": "./proxy/*.cjs" };
case "recursive alias":
return { "#entry": "#window", "#window": "./proxy/window.cjs" };
case "external-package alias":
return { "#entry": "fixture-window" };
case "builtin alias":
return { "#entry": "fs" };
case "electron alias":
return { "#entry": "electron" };
case "electron main alias":
return { "#entry": "electron/main" };
default:
return { "#entry": null };
}
};
cliTest.for(
(["CLI", "stdio MCP"] as const).flatMap((surface) =>
[
@@ -96,371 +76,6 @@ cliTest.for(
},
);
const unprovenLoaderSources: Readonly<Record<string, string>> = {
"custom iterator assignment module loader":
'const values = { *[Symbol.iterator]() { yield module; } }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");',
"replaced iterator assignment module loader":
'const values = [{}]; values[Symbol.iterator] = function* () { yield module; }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");',
"custom iterator rest declaration module loader":
'const values = { *[Symbol.iterator]() { yield module; } }; const [...holder] = values; holder[0].require("../shared/window.cjs");',
"native array assignment local loader":
'const holder = { loader: module }; [holder.loader] = [{ require() {} }]; holder.loader.require("../shared/window.cjs");',
"logical rewrite assignment module loader":
'const holder = {}; holder.loader ??= module; holder.loader.require = replacement; require("../shared/window.cjs");',
"object rest declaration module loader":
'const { ...holder } = { loader: module }; holder.loader.require("../shared/window.cjs");',
"array rest declaration module loader":
'const [...holder] = [module]; holder[0].require("../shared/window.cjs");',
"object assignment module loader":
'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require("../shared/window.cjs");',
"array assignment module loader":
'const holder = {}; [holder.loader] = [module]; holder.loader.require("../shared/window.cjs");',
"nested assignment module loader":
'const holder = {}; ({ inner: { loader: holder.loader } } = { inner: { loader: module } }); holder.loader.require("../shared/window.cjs");',
"default assignment module loader":
'const holder = {}; ({ loader: holder.loader = module } = {}); holder.loader.require("../shared/window.cjs");',
"rest array assignment module loader":
'const holder = {}; let ignored; [ignored, ...holder.loaders] = [{}, module]; holder.loaders[0].require("../shared/window.cjs");',
"delegate assignment module loader":
'const holder = {}; ({ load: holder.load } = { load: module.require }); holder.load.call(module, "../shared/window.cjs");',
"rewrite assignment module loader":
'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require = replacement; require("../shared/window.cjs");',
"exports assignment local loader":
'const holder = {}; ({ exports: holder.loader } = module); holder.loader.require("../shared/window.cjs");',
"rest assignment local loader":
'const holder = {}; ({ ...holder.loader } = module); holder.loader.require("../shared/window.cjs");',
"excluded assignment local loader":
'const holder = {}; let ignored; ({ loader: ignored, ...holder.copy } = { loader: module }); holder.copy.loader.require("../shared/window.cjs");',
"inactive default assignment local loader":
'const holder = {}; ({ loader: holder.loader = module } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");',
"overwritten assignment local loader":
'const holder = { loader: module }; ({ loader: holder.loader } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");',
"bounded alias selection": aliasSelectionBudgetSource(),
"captured module delegate":
'const load = module.require; load.call(module, "../shared/window.cjs");',
"destructured module delegate":
'const { require: load } = module; load.apply(module, ["../shared/window.cjs"]);',
"selected local delegate slots":
'const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement; require("../shared/window.cjs");',
"local assigned loader":
'const holder = {}; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");',
"overwritten local loader":
'const holder = { loader: module }; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");',
"normalized absolute ESM": 'import "/discard%2F/../shared/window.cjs";',
"normalized absolute dynamic import":
'import("/discard%5C/../shared/window.cjs");',
"rewritten CommonJS delegate":
'module.require = replacement; require("../shared/window.cjs");',
"nested rewritten CommonJS delegate":
'const holder = { loader: module }; holder.loader.require = replacement; require("../shared/window.cjs");',
"assigned rewritten CommonJS delegate":
'const holder = {}; holder.loader = module; holder.loader.require = replacement; require("../shared/window.cjs");',
"assigned module loader":
'const holder = {}; holder.loader = module; holder.loader.require("../shared/window.cjs");',
"nested module loader":
'const holder = { loader: module }; holder.loader.require("../shared/window.cjs");',
"destructured module loader":
'const holder = { loader: module }; const { loader } = holder; loader.require("../shared/window.cjs");',
"array module loader":
'const holder = [module]; holder[0].require("../shared/window.cjs");',
"bound module loader":
'const load = module.require.bind(module); load("../shared/window.cjs");',
"bound module alias loader":
'const current = module; const load = current.require.bind(current); load("../shared/window.cjs");',
"computed bound module loader":
'const load = module["require"].bind(module); load("../shared/window.cjs");',
"fallback module alias":
'const { loader = module } = options; loader.require("../shared/window.cjs");',
"inactive fallback":
'const { loader = module } = { loader: {} }; loader.require("../shared/window.cjs");',
"absolute ESM": 'import "/shared/window.cjs";',
"invalid absolute ESM": 'import "/shared%2Fwindow.cjs";',
"invalid absolute dynamic import": 'import("/shared%5Cwindow.cjs");',
"absolute dynamic import": 'import("/shared/window.cjs");',
"absolute file URL": 'import "file:///shared/window.cjs";',
"dynamic file URL": 'import("file:///shared/window.cjs");',
"createRequire base":
'import { createRequire } from "node:module"; const require = createRequire(new URL("../other/nested/main.cjs", import.meta.url)); require("../shared/window.cjs");',
};
const uncheckedModuleLoaders = new Set([
"custom iterator assignment module loader",
"replaced iterator assignment module loader",
"custom iterator rest declaration module loader",
"logical rewrite assignment module loader",
"object rest declaration module loader",
"array rest declaration module loader",
"object assignment module loader",
"array assignment module loader",
"nested assignment module loader",
"default assignment module loader",
"rest array assignment module loader",
"delegate assignment module loader",
"rewrite assignment module loader",
"bounded alias selection",
"captured module delegate",
"destructured module delegate",
"rewritten CommonJS delegate",
"nested rewritten CommonJS delegate",
"assigned rewritten CommonJS delegate",
"assigned module loader",
"module.require",
"computed module.require",
"module alias",
"later reassigned module alias",
"conditional module alias",
"fallback module alias",
"bound module loader",
"bound module alias loader",
"computed bound module loader",
"nested module loader",
"destructured module loader",
"array module loader",
]);
const fixtureMainPath = (loader: string): string => {
switch (loader) {
case "absolute ESM":
case "normalized absolute ESM":
case "normalized absolute dynamic import":
case "invalid absolute ESM":
case "invalid absolute dynamic import":
case "absolute dynamic import":
case "absolute file URL":
case "dynamic file URL":
case "createRequire base":
return "main.mjs";
case "absolute main":
return "/opt/app/main.cjs";
case "Windows relative main":
return "sub\\main.cjs";
case "Windows drive-relative main":
return "C:main.cjs";
case "percent main":
case "single percent main":
return "main%2Fentry.cjs";
default:
return "main.cjs";
}
};
const fixtureMainInventoryPath = (loader: string, main: string): string => {
if (loader === "absolute main") return "opt/app/main.cjs";
if (loader === "Windows relative main") return "b/sub/main.cjs";
if (loader === "Windows drive-relative main") return "b/main.cjs";
return `b/${main}`;
};
const fixtureLoaderSource = (loader: string): string =>
unprovenLoaderSources[loader] ??
(loader === "resolve-only"
? 'require.resolve("../shared/window.cjs");'
: loader === "missing wildcard alias"
? 'require("#missing/*");'
: loader === "absolute require"
? 'require("/shared/window.cjs");'
: loader === "projected module"
? 'const { exports: api } = module; api.require = () => {}; api.require("../shared/window.cjs");'
: loader === "absolute main"
? 'require("../../shared/window.cjs");'
: loader === "later reassigned module alias"
? 'let current = module; current.require("../shared/window.cjs"); current = other;'
: loader === "earlier reassigned module alias"
? 'let current = module; current = { require() {} }; current.require("../shared/window.cjs");'
: loader === "conditional module alias"
? 'let current = module; if (flag) current = other; current.require("../shared/window.cjs");'
: loader === "module.require"
? 'module.require("../shared/window.cjs");'
: loader === "computed module.require"
? 'module["require"]("../shared/window.cjs");'
: loader === "module alias"
? 'const current = module; current.require("../shared/window.cjs");'
: loader === "local module"
? 'function local(module) { module.require("../shared/window.cjs"); }'
: loader === "pattern alias"
? 'require("#window/window");'
: loader.endsWith("alias")
? 'require("#entry");'
: 'require("../shared/window.cjs");');
cliTest.for(
(["CLI", "stdio MCP"] as const).flatMap((surface) =>
[
"module.require",
"computed module.require",
"module alias",
"later reassigned module alias",
"earlier reassigned module alias",
"conditional module alias",
"absolute main",
"Windows relative main",
"Windows drive-relative main",
"percent main",
"pattern alias",
"recursive alias",
"external-package alias",
"builtin alias",
"electron alias",
"electron main alias",
"blocked alias",
"local module",
"projected module",
"resolve-only",
"missing wildcard alias",
"absolute require",
...Object.keys(unprovenLoaderSources),
"single percent main",
].map((loader) => ({ surface, loader })),
),
)(
"preserves competing loader ownership for $loader through $surface",
async ({ surface, loader }, { cli }) => {
const root = await createTestTempDirectory("rea-loader-owner-");
const single = loader === "single percent main";
const inert = [
"native array assignment local loader",
"exports assignment local loader",
"rest assignment local loader",
"excluded assignment local loader",
"inactive default assignment local loader",
"overwritten assignment local loader",
"local assigned loader",
"overwritten local loader",
"invalid absolute ESM",
"invalid absolute dynamic import",
"builtin alias",
"electron alias",
"electron main alias",
"inactive fallback",
"blocked alias",
"local module",
"projected module",
"resolve-only",
"missing wildcard alias",
"earlier reassigned module alias",
].includes(loader);
const absolute = loader === "absolute main";
const main = fixtureMainPath(loader);
const imports = fixtureImports(loader);
const load = fixtureLoaderSource(loader);
await writeFixtureFiles(root, {
"a/package.json": JSON.stringify({ main: "main.cjs" }),
"a/main.cjs":
single || absolute
? "module.exports = 1;"
: 'require("../shared/window.cjs");',
"b/package.json": JSON.stringify({ main, imports }),
[fixtureMainInventoryPath(loader, main)]: load,
"b/proxy/window.cjs": 'require("../../shared/window.cjs");',
"b/node_modules/fixture-window/package.json": JSON.stringify({
main: "index.cjs",
}),
"b/node_modules/fixture-window/index.cjs":
'require("../../../shared/window.cjs");',
"shared/window.cjs":
'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");',
"a/index.html": "first application page",
"b/index.html": "second application page",
"shared/index.html": "source-relative decoy",
"index.html": "artifact-root decoy",
});
const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);
if (loader.startsWith("Windows "))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "loads",
properties: expect.objectContaining({
declared_path: main,
resolution_status: "unavailable",
resolved_path: null,
}),
}),
]),
);
if (
["absolute require", "absolute ESM", "absolute dynamic import"].includes(
loader,
)
)
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
specifier: "/shared/window.cjs",
resolution_status: "unavailable",
resolved_path: null,
}),
}),
]),
);
if (loader === "createRequire base")
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
kind: "require",
specifier: null,
resolved_path: null,
}),
}),
]),
);
if (loader.endsWith("alias") && !inert && !loader.includes("module alias"))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
specifier:
loader === "pattern alias" ? "#window/window" : "#entry",
resolution_status: "unavailable",
}),
}),
]),
);
if (uncheckedModuleLoaders.has(loader))
expect(graph.edges).toEqual(
expect.arrayContaining([
expect.objectContaining({
relation: "imports",
properties: expect.objectContaining({
kind: "require",
specifier: null,
resolved_path: null,
}),
}),
]),
);
const roles = graph.nodes.filter(
({ kind }) => kind === "electron-renderer",
);
expect(roles).toHaveLength(1);
const role = roles[0];
const target = single ? "b/index.html" : inert ? "a/index.html" : null;
expect(role?.observations).toEqual(
expect.arrayContaining([
expect.objectContaining({
properties: expect.objectContaining({
resolved_path: target,
resolution_status: target === null ? "unavailable" : "resolved",
}),
}),
]),
);
expect(
graph.edges
.filter(
({ source_node_id, relation }) =>
source_node_id === role?.node_id && relation === "maps_to",
)
.map(({ properties }) => properties.resolved_path),
).toEqual(target === null ? [] : [target]);
},
);
cliTest.for(["CLI", "stdio MCP"] as const)(
"propagates the application owner into an inventoried ASAR through %s",
async (surface, { cli }) => {
@@ -0,0 +1,133 @@
import { expect } from "vitest";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js";
import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
import { cliTest } from "../../support/cli/cliFixture.js";
const capturedSource = `
const dynamic = globalThis.options;
document.modelContext.registerTool({
name: "spread",
inputSchema: { properties: { ...{ query: {} }, direct: {} } }
});
document.modelContext.registerTool({
name: "partial",
inputSchema: { properties: { known: {}, ...dynamic } }
});
document.modelContext.registerTool({
name: "empty-status", description: "same",
inputSchema: { properties: dynamic }
});
document.modelContext.registerTool({
name: "empty-status", description: "same",
inputSchema: { properties: {} }
});
document.modelContext.registerTool({
name: "own-after-prototype-setter",
inputSchema: {
properties: { own: {} },
__proto__: { properties: { inherited: {} } }
}
});
`;
const inspectDeclarations = (document: unknown) => {
const parsed =
toolContract("analyze_web_bundle").outputSchema.parse(document);
const parsedEvidence = parseEvidence(parsed);
return {
evidence: parsedEvidence,
declarations: parsed.normalized_result.observations.webmcp_declarations,
};
};
const expectCoverageCases = (
declarations: ReturnType<typeof inspectDeclarations>["declarations"],
) => {
expect(declarations).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "spread",
schema_property_names: ["direct", "query"],
schema_property_names_coverage: "complete",
}),
expect.objectContaining({
name: "partial",
schema_property_names: ["known"],
schema_property_names_coverage: "partial",
}),
expect.objectContaining({
name: "empty-status",
description: "same",
schema_property_names: [],
schema_property_names_coverage: "unknown",
}),
expect.objectContaining({
name: "empty-status",
description: "same",
schema_property_names: [],
schema_property_names_coverage: "complete",
}),
expect.objectContaining({
name: "own-after-prototype-setter",
schema_property_names: ["own"],
schema_property_names_coverage: "complete",
}),
]),
);
expect(
declarations.filter(({ name }) => name === "empty-status"),
).toHaveLength(2);
};
cliTest(
"preserves WebMCP schema-name coverage through compiled CLI and stdio MCP",
async ({ cli }) => {
const browser = await startFakeCdpBrowser({
commandResult: ({ method }) =>
method === "Debugger.getScriptSource"
? { scriptSource: capturedSource }
: undefined,
});
try {
const contract = toolContract("analyze_web_bundle");
const cliReply = await cli.run({
arguments: [
"analyze-web-bundle",
browser.endpoint,
"allowed-page",
"--allowed-origins",
browser.allowedOrigin,
"--observation-ms",
"0",
"--json",
],
});
expect(cliReply.exitCode, cliReply.stdout + cliReply.stderr).toBe(0);
const cliAnalysis = inspectDeclarations(cliReply.json);
expectCoverageCases(cliAnalysis.declarations);
const { call } = await connectLocalToolsMcp();
const input = {
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
include_script_sources: true,
fetch_source_maps: false,
};
const response = await call(contract.name, input);
expect(response.isError, JSON.stringify(response)).not.toBe(true);
const mcpAnalysis = inspectDeclarations(response.structuredContent);
expectCoverageCases(mcpAnalysis.declarations);
expect(mcpAnalysis.evidence.operation).toBe("analyze_web_bundle");
expect(mcpAnalysis.evidence.authority).toBe("external-service");
expect(mcpAnalysis.declarations).toEqual(cliAnalysis.declarations);
} finally {
await browser.close();
}
},
120_000,
);
@@ -0,0 +1,232 @@
import {
chmod,
mkdir,
mkdtemp,
readFile,
realpath,
rm,
symlink,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Cli } from "incur";
import pino from "pino";
import { expect, it } from "vitest";
import type { BrowserScenarioCapturePort } from "../../../src/application/BrowserScenarioCapturePort.js";
import type { ElectronActiveObservationPort } from "../../../src/application/javascript/ElectronActiveObservationPort.js";
import { registerBrowserScenarioCommands } from "../../../src/cliBrowserScenarioCommands.js";
import { registerAnalysisViewCommands } from "../../../src/cli/analysisViewCommands.js";
import { registerElectronCommands } from "../../../src/cli/electronCommands.js";
import { AnalysisCapabilityUnavailableError } from "../../../src/domain/analysisErrorCore.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { err } from "../../../src/domain/result.js";
import {
analysisViewJavaScriptEvidence,
analysisViewLayoutEvidence,
} from "../../fixtures/analysisView.js";
const providerIdentity = {
id: "test.path-boundary",
name: "Path boundary test provider",
version: null,
} as const;
const serveJson = async (
cli: ReturnType<typeof Cli.create>,
command: string,
input: unknown,
): Promise<unknown> => {
let stdout = "";
await cli.serve([command, JSON.stringify(input), "--json"], {
env: {},
stdout: (text) => {
stdout += text;
},
exit: () => undefined,
});
return JSON.parse(stdout) as unknown;
};
it.skipIf(process.platform === "win32")(
"keeps caller-selected JSON paths through the public CLI and provider boundaries",
async () => {
const root = await mkdtemp(join(tmpdir(), "rea-cli-json-path-"));
const originalCwd = process.cwd();
const originalExitCode = process.exitCode;
try {
await mkdir(join(root, "real/inner"), { recursive: true });
await mkdir(join(root, "real/app"), { recursive: true });
await mkdir(join(root, "real/root"), { recursive: true });
await mkdir(join(root, "app"), { recursive: true });
await mkdir(join(root, "root"), { recursive: true });
await writeFile(join(root, "real/tool"), "selected executable");
await writeFile(join(root, "tool"), "lexical sibling");
await chmod(join(root, "real/tool"), 0o700);
await chmod(join(root, "tool"), 0o700);
await symlink(join(root, "real/inner"), join(root, "link"));
process.chdir(root);
const electronInputs: string[][] = [];
const electronProvider: ElectronActiveObservationPort = {
identity: () => providerIdentity,
capture: async (input) => {
electronInputs.push([
input.executable_path,
input.application_path,
input.application_root,
]);
return err(
new AnalysisCapabilityUnavailableError(
providerIdentity.id,
"capture_electron_scenario",
"intentional inert test provider",
),
);
},
close: async () => undefined,
};
const electronCli = Cli.create("rea-path-electron-test", {
sync: false,
});
registerElectronCommands(
electronCli,
pino({ level: "silent" }),
{},
{
createActiveProvider: () => electronProvider,
},
);
for (const path of ["link/../", `${root}/link/../`]) {
const output = await serveJson(
electronCli,
"capture-electron-scenario",
{
executable_path: `${path}tool`,
application_path: `${path}app`,
application_root: `${path}root`,
actions: [],
},
);
expect(output).toMatchObject({ code: "capability_unavailable" });
}
const expectedElectronPaths = [
await realpath(join(root, "real/tool")),
await realpath(join(root, "real/app")),
await realpath(join(root, "real/root")),
];
expect(electronInputs).toEqual([
expectedElectronPaths,
expectedElectronPaths,
]);
const browserInputs: string[] = [];
const browserProvider: BrowserScenarioCapturePort = {
identity: () => providerIdentity,
captureScenario: async (scenario) => {
if (scenario.browser.mode === "launch")
browserInputs.push(scenario.browser.executable_path);
return err(
new AnalysisCapabilityUnavailableError(
providerIdentity.id,
"capture_browser_scenario",
"intentional inert test provider",
),
);
},
close: async () => undefined,
};
const browserCli = Cli.create("rea-path-browser-test", {
sync: false,
});
registerBrowserScenarioCommands(
browserCli,
pino({ level: "silent" }),
{},
{ createProvider: () => browserProvider },
);
for (const executablePath of [
"link/../browser",
`${root}/link/../browser`,
]) {
const output = await serveJson(browserCli, "capture-browser-scenario", {
browser: { mode: "launch", executable_path: executablePath },
start_url: { url: "https://example.test" },
actions: [
{
step_id: "settle",
action: "wait_for_timeout",
duration_ms: 1,
},
],
});
expect(output).toMatchObject({ code: "capability_unavailable" });
}
expect(browserInputs).toEqual([
`${root}/link/../browser`,
`${root}/link/../browser`,
]);
} finally {
process.chdir(originalCwd);
process.exitCode = originalExitCode;
await rm(root, { recursive: true, force: true });
}
},
);
it.skipIf(process.platform === "win32")(
"reads the caller-selected Evidence file through the registered CLI",
async () => {
const root = await mkdtemp(join(tmpdir(), "rea-cli-evidence-path-"));
const originalCwd = process.cwd();
const originalExitCode = process.exitCode;
try {
await mkdir(join(root, "real/inner"), { recursive: true });
await symlink(join(root, "real/inner"), join(root, "link"));
process.chdir(root);
const selectedEvidence = analysisViewLayoutEvidence();
const siblingEvidence = analysisViewJavaScriptEvidence();
await mkdir(join(root, "real"), { recursive: true });
await writeFile(
join(root, "real/evidence.json"),
JSON.stringify(selectedEvidence),
);
await writeFile(
join(root, "evidence.json"),
JSON.stringify(siblingEvidence),
);
expect(
parseEvidence(
JSON.parse(await readFile(`${root}/link/../evidence.json`, "utf8")),
).evidence_id,
).toBe(selectedEvidence.evidence_id);
const analysisViewCli = Cli.create("rea-path-evidence-test", {
sync: false,
});
registerAnalysisViewCommands(analysisViewCli, pino({ level: "silent" }));
for (const path of [
"link/../evidence.json",
`${root}/link/../evidence.json`,
]) {
const output = parseEvidence(
await serveJson(analysisViewCli, "inspect-analysis-view", {
source: { kind: "evidence-file", path },
view: { kind: "summary" },
}),
);
expect(output.parameters.parent_evidence_id, path).toBe(
selectedEvidence.evidence_id,
);
expect(output.evidence_links, path).toEqual([
selectedEvidence.evidence_id,
]);
}
} finally {
process.chdir(originalCwd);
process.exitCode = originalExitCode;
await rm(root, { recursive: true, force: true });
}
},
);
+174 -7
View File
@@ -13,6 +13,12 @@ const actionPath = join(repository, ".github/actions/npm-cache/action.yml");
const readAction = async () =>
z
.object({
inputs: z
.record(
z.string(),
z.object({ default: z.string().optional() }).passthrough(),
)
.optional(),
outputs: z.record(z.string(), z.object({ value: z.string() })),
runs: z.object({
steps: z.array(
@@ -42,9 +48,11 @@ const readWorkflows = async () => {
jobs: z
.record(
z.string(),
z.object({
steps: z.array(z.record(z.string(), z.unknown())).optional(),
}),
z
.object({
steps: z.array(z.record(z.string(), z.unknown())).optional(),
})
.passthrough(),
)
.optional(),
})
@@ -107,6 +115,9 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
expect(output).toContain(
`key=npm-download-Windows-${process.arch}-lock-hash\n`,
);
expect(output).toContain(
`restore-key=npm-download-Windows-${process.arch}-\n`,
);
}
});
@@ -120,6 +131,7 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
step.uses?.startsWith("actions/cache/restore@"),
);
expect(action.inputs?.save?.default).toBe("false");
expect(action.outputs["can-save"]?.value).toBe(
"${{ steps.context.outputs.can-save }}",
);
@@ -138,19 +150,27 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
expect(restoreAndSave?.uses).toBe(
"actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830",
);
expect(restoreAndSave?.if).toBe("steps.context.outputs.can-save == 'true'");
expect(restoreAndSave?.if).toBe(
"steps.context.outputs.can-save == 'true' && inputs.save == 'true'",
);
expect(restoreOnly?.uses).toBe(
"actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830",
);
expect(restoreOnly?.if).toBe("steps.context.outputs.can-save != 'true'");
expect(restoreOnly?.if).toBe(
"steps.context.outputs.can-save != 'true' || inputs.save != 'true'",
);
expect(restoreAndSave?.with?.path).toBe(
"${{ steps.context.outputs.path }}",
);
expect(restoreOnly?.with?.path).toBe(restoreAndSave?.with?.path);
expect(restoreAndSave?.with?.key).toBe("${{ steps.context.outputs.key }}");
expect(restoreOnly?.with?.key).toBe(restoreAndSave?.with?.key);
expect(restoreAndSave?.with).not.toHaveProperty("restore-keys");
expect(restoreOnly?.with).not.toHaveProperty("restore-keys");
expect(restoreAndSave?.with?.["restore-keys"]).toBe(
"${{ steps.context.outputs.restore-key }}",
);
expect(restoreOnly?.with?.["restore-keys"]).toBe(
restoreAndSave?.with?.["restore-keys"],
);
});
it("replaces every inline setup-node npm cache immediately and preserves conditions", async () => {
@@ -258,6 +278,153 @@ it("shares static task snapshots only after validation using the npm cache owner
path: "node_modules/.cache/knip",
key: "${{ steps.knip-cache.outputs.cache-primary-key }}",
});
expect(
steps.find((step) => step.run === "node scripts/ci/cache-retention.mjs"),
).toBeUndefined();
});
it("deletes superseded caches only from the trusted follow-up workflow", async () => {
const workflow = z
.object({
on: z.object({
workflow_run: z.object({
workflows: z.array(z.literal("CI")),
types: z.array(z.literal("completed")),
}),
workflow_call: z.unknown().optional(),
pull_request: z.unknown().optional(),
}),
permissions: z.record(z.string(), z.string()),
jobs: z.record(
z.string(),
z.object({
if: z.string(),
"runs-on": z.literal("ubuntu-latest"),
steps: z.array(z.record(z.string(), z.unknown())),
}),
),
})
.parse(
parse(
await readFile(
join(repository, ".github/workflows/cache-retention.yml"),
"utf8",
),
),
);
expect(workflow.on.workflow_call).toBeUndefined();
expect(workflow.on.pull_request).toBeUndefined();
expect(workflow.permissions).toEqual({
contents: "read",
actions: "write",
});
const job = workflow.jobs.retain;
expect(job?.if).toContain(
"github.event.workflow_run.conclusion == 'success'",
);
expect(job?.if).toContain(
"github.event.workflow_run.conclusion == 'failure'",
);
expect(job?.if).toContain(
"github.event.workflow_run.conclusion == 'timed_out'",
);
expect(job?.if).toContain(
"github.event.workflow_run.conclusion == 'cancelled'",
);
expect(job?.if).toContain(
"github.event.workflow_run.head_repository.full_name == github.repository",
);
expect(job?.if).toContain(
"github.event.workflow_run.head_branch == github.event.repository.default_branch",
);
expect(job?.if).toContain("github.event.workflow_run.event == 'push'");
expect(job?.if).toContain("github.event.workflow_run.event == 'schedule'");
expect(job?.if).toContain(
"github.event.workflow_run.event == 'workflow_dispatch'",
);
expect(job?.if).not.toContain("pull_request");
const steps = job?.steps ?? [];
expect(steps.some((step) => step.run === "npm ci")).toBe(false);
const checkout = steps[0];
expect(checkout?.uses).toBe(
"actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
);
expect(checkout?.with).toMatchObject({
ref: "${{ github.event.workflow_run.head_sha }}",
"persist-credentials": false,
});
const retention = steps.find(
(step) => step.run === "node scripts/ci/cache-retention.mjs",
);
const retentionEnv = z.record(z.string(), z.string()).parse(retention?.env);
expect(retentionEnv.CACHE_PREFIX).toBe(
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-",
);
expect(retentionEnv.CACHE_KEEP_KEY).toBe(
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}",
);
expect(retentionEnv.CACHE_PORTABLE_PREFIX).toBe(
"rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-",
);
expect(retentionEnv.DEDUPE_NPM_DOWNLOADS).toBe("true");
expect(retentionEnv.GH_TOKEN).toBe("${{ github.token }}");
const ci = (await readWorkflows()).find(({ file }) => file === "ci.yml");
expect(JSON.stringify(ci?.workflow)).not.toContain('"actions":"write"');
const deepSteps = ci?.workflow.jobs?.["static-deep"]?.steps ?? [];
const publishedKey = (id: string) =>
z
.string()
.parse(
z
.record(z.string(), z.unknown())
.parse(deepSteps.find((step) => step.id === id)?.with).key,
);
const savedOnThisRun = "${{ github.event.workflow_run.head_sha }}";
const deepPrefix = "rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-";
const knipPrefix = "knip-meta-v1-${{ runner.os }}-";
expect(retentionEnv.CACHE_DEEP_PREFIX).toBe(deepPrefix);
expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toBe(
publishedKey("static-cache").replaceAll(
"${{ github.sha }}",
savedOnThisRun,
),
);
expect(publishedKey("static-cache").startsWith(deepPrefix)).toBe(true);
expect(retentionEnv.CACHE_KNIP_PREFIX).toBe(knipPrefix);
expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe(
publishedKey("knip-cache").replaceAll("${{ github.sha }}", savedOnThisRun),
);
expect(publishedKey("knip-cache").startsWith(knipPrefix)).toBe(true);
});
it("publishes each hosted npm download cache from one writer", async () => {
const workflows = await readWorkflows();
const writers: { file: string; job: string; save: string }[] = [];
for (const { file, workflow } of workflows)
for (const [job, definition] of Object.entries(workflow.jobs ?? {}))
for (const step of definition.steps ?? []) {
if (step.uses !== "./.github/actions/npm-cache") continue;
const save = z
.record(z.string(), z.unknown())
.parse(step.with ?? {}).save;
if (save === undefined) continue;
writers.push({ file, job, save: z.string().parse(save) });
}
expect(writers).toEqual([
{ file: "ci.yml", job: "static", save: "true" },
{
file: "ci.yml",
job: "package-e2e",
save: "${{ matrix.os != 'ubuntu-latest' }}",
},
{ file: "ci.yml", job: "windows-curated", save: "true" },
]);
const ci = workflows.find(({ file }) => file === "ci.yml");
expect(ci?.workflow.jobs?.static?.permissions).toBeUndefined();
expect(ci?.workflow.jobs?.build?.permissions).toEqual({
contents: "read",
actions: "read",
});
});
it("passes the npm cache owner's selected path to package installation", async () => {
@@ -0,0 +1,464 @@
import { execFileSync } from "node:child_process";
import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { describe, expect, it } from "vitest";
import {
listAllCaches,
retainCaches,
selectNpmDownloadDuplicateDeletions,
selectPrefixDeletions,
} from "../../../scripts/ci/cache-retention.mjs";
const cache = (
id: number,
key: string,
created: string,
accessed = created,
size = 100,
version = "a".repeat(64),
ref = "refs/heads/main",
) => ({
id,
key,
version,
ref,
created_at: created,
last_accessed_at: accessed,
size_in_bytes: size,
});
describe("Turbo snapshot retention", () => {
const prefix = "rea-static-v1-Linux-X64-";
it("keeps the two newest snapshots and the saved key", () => {
const keepKey = `${prefix}saved`;
const deletions = selectPrefixDeletions(
[
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 10),
cache(2, `${prefix}older`, "2026-10-20T02:00:00Z", undefined, 20),
cache(3, `${prefix}recent`, "2026-10-20T03:00:00Z", undefined, 30),
cache(4, keepKey, "2026-10-20T00:00:00Z", undefined, 40),
],
{ prefix, keepKey },
);
expect(deletions.map((entry) => entry.id)).toEqual([1]);
expect(deletions.reduce((sum, entry) => sum + entry.size, 0)).toBe(10);
});
it("keeps an in-flight generation when the saved key is already newest", () => {
const keepKey = `${prefix}newest`;
const deletions = selectPrefixDeletions(
[
cache(1, `${prefix}a`, "2026-10-20T01:00:00Z"),
cache(2, `${prefix}b`, "2026-10-20T02:00:00Z"),
cache(3, `${prefix}c`, "2026-10-20T03:00:00Z"),
cache(4, keepKey, "2026-10-20T04:00:00Z"),
],
{ prefix, keepKey },
);
expect(
deletions.map((entry) => entry.id).sort((left, right) => left - right),
).toEqual([1, 2]);
});
it("keeps two generations of the deep and Knip snapshot families", () => {
const deep = "rea-static-deep-v1-Linux-X64-";
const knip = "knip-meta-v1-Linux-";
expect(
selectPrefixDeletions(
[
cache(1, `${deep}old`, "2026-10-20T01:00:00Z"),
cache(2, `${deep}mid`, "2026-10-20T02:00:00Z"),
cache(3, `${deep}new`, "2026-10-20T03:00:00Z"),
],
{ prefix: deep, keepKey: `${deep}new` },
).map((entry) => entry.id),
).toEqual([1]);
expect(
selectPrefixDeletions(
[
cache(4, `${knip}old`, "2026-10-20T01:00:00Z"),
cache(5, `${knip}mid`, "2026-10-20T02:00:00Z"),
cache(6, `${knip}new`, "2026-10-20T03:00:00Z"),
],
{ prefix: knip, keepKey: `${knip}new` },
).map((entry) => entry.id),
).toEqual([4]);
});
it("retains the newest portable snapshots without a caller keep key", () => {
const portable = "rea-portable-build-v2-Linux-X64-";
const deletions = selectPrefixDeletions(
[
cache(7, `${portable}one`, "2026-10-20T01:00:00Z"),
cache(8, `${portable}two`, "2026-10-20T02:00:00Z"),
cache(9, `${portable}three`, "2026-10-20T03:00:00Z"),
],
{ prefix: portable },
);
expect(deletions.map((entry) => entry.id)).toEqual([7]);
});
it("refuses prefixes and lists that can reach download caches", () => {
expect(() =>
selectPrefixDeletions([], { prefix: "npm-download-Linux-x64-" }),
).toThrow(/one Turbo family/u);
expect(() =>
selectPrefixDeletions([], { prefix: "rea-static-v1-" }),
).toThrow(/one Turbo family/u);
expect(() =>
selectPrefixDeletions([], { prefix: "knip-meta-v1-" }),
).toThrow(/one Turbo family/u);
expect(() =>
selectPrefixDeletions(
[cache(1, "npm-download-Linux-x64-abc", "2026-10-20T01:00:00Z")],
{
prefix,
keepKey: `${prefix}saved`,
},
),
).toThrow(/outside/u);
expect(() =>
selectPrefixDeletions([], {
prefix,
keepKey: "rea-static-v1-Windows-X64-other",
}),
).toThrow(/keep key/u);
});
});
describe("npm download duplicate retention", () => {
it("deletes only an older copy of the same key, version, and ref", () => {
const key = "npm-download-Linux-x64-lock";
const version = "a".repeat(64);
const ref = "refs/heads/main";
const deletions = selectNpmDownloadDuplicateDeletions([
cache(
1,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T02:00:00Z",
179,
version,
ref,
),
cache(
2,
key,
"2026-10-20T01:05:00Z",
"2026-10-20T03:00:00Z",
199,
version,
ref,
),
cache(
3,
"npm-download-Windows-x64-lock",
"2026-10-20T01:00:00Z",
undefined,
186,
),
]);
expect(deletions.map((entry) => entry.id)).toEqual([1]);
});
it("keeps records that share a key but differ by version or ref", () => {
const key = "npm-download-Linux-x64-lock";
const version = "a".repeat(64);
const otherVersion = "b".repeat(64);
const deletions = selectNpmDownloadDuplicateDeletions([
cache(
1,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T04:00:00Z",
199,
version,
"refs/heads/main",
),
cache(
2,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T03:00:00Z",
188,
otherVersion,
"refs/heads/main",
),
cache(
3,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T02:00:00Z",
180,
version,
"refs/heads/feature",
),
]);
expect(deletions).toEqual([]);
});
it("refuses a download record without a version or ref", () => {
const key = "npm-download-Linux-x64-lock";
const { version: _version, ...withoutVersion } = cache(
1,
key,
"2026-10-20T01:00:00Z",
);
expect(() => selectNpmDownloadDuplicateDeletions([withoutVersion])).toThrow(
/version/u,
);
const { ref: _ref, ...withoutRef } = cache(2, key, "2026-10-20T01:00:00Z");
expect(() => selectNpmDownloadDuplicateDeletions([withoutRef])).toThrow(
/ref/u,
);
});
it("does not delete the only copy when pagination repeats its id", () => {
const key = "npm-download-Linux-x64-lock";
const version = "a".repeat(64);
const ref = "refs/heads/main";
const deletions = selectNpmDownloadDuplicateDeletions([
cache(
5,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T02:00:00Z",
179,
version,
ref,
),
cache(
5,
key,
"2026-10-20T01:00:00Z",
"2026-10-20T04:00:00Z",
179,
version,
ref,
),
]);
expect(deletions).toEqual([]);
});
it("aborts when the same id is listed with a different version", () => {
const key = "npm-download-Linux-x64-lock";
expect(() =>
selectNpmDownloadDuplicateDeletions([
cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "a".repeat(64)),
cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "b".repeat(64)),
]),
).toThrow(/inconsistent/u);
});
it("refuses a Turbo snapshot in the download list", () => {
expect(() =>
selectNpmDownloadDuplicateDeletions([
cache(1, "rea-static-v1-Linux-X64-sha", "2026-10-20T01:00:00Z"),
]),
).toThrow(/non-download/u);
});
});
describe("retention orchestration", () => {
const repository = "morluto/rea";
const prefix = "rea-static-v1-Linux-X64-";
it("deletes nothing when a later list is incomplete", async () => {
const deleted: number[] = [];
const page = Array.from({ length: 100 }, (_, index) =>
cache(index + 1, `${prefix}${String(index)}`, "2026-10-20T01:00:00Z"),
);
await expect(
retainCaches({
repository,
request: async (_method, endpoint) => {
if (endpoint.includes("npm-download-"))
throw new Error("list failed after the first family");
return { actions_caches: page };
},
deleteCache: async (id) => {
deleted.push(id);
},
families: [{ prefix, keepKey: `${prefix}0` }],
dedupeNpmDownloads: true,
}),
).rejects.toThrow(/list failed|exceeded/u);
expect(deleted).toEqual([]);
});
it("stops listing once a short page proves the inventory is complete", async () => {
const requests: string[] = [];
const caches = await listAllCaches(
async (_method, endpoint) => {
requests.push(endpoint);
const page = new URL(endpoint, "https://example.test").searchParams.get(
"page",
);
return {
actions_caches:
page === "1"
? Array.from({ length: 100 }, (_, index) =>
cache(
index + 1,
`${prefix}${String(index)}`,
"2026-10-20T01:00:00Z",
),
)
: [cache(101, `${prefix}last`, "2026-10-20T02:00:00Z")],
};
},
repository,
prefix,
);
expect(caches).toHaveLength(101);
expect(requests).toHaveLength(2);
});
it("applies every family only after both lists succeed", async () => {
const deleted: number[] = [];
const keepKey = `${prefix}saved`;
const portable = "rea-portable-build-v2-Linux-X64-";
const result = await retainCaches({
repository,
request: async (_method, endpoint) => {
const key = new URL(endpoint, "https://example.test").searchParams.get(
"key",
);
if (key === prefix)
return {
actions_caches: [
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
cache(3, keepKey, "2026-10-20T03:00:00Z"),
],
};
if (key === portable)
return {
actions_caches: [
cache(4, `${portable}old`, "2026-10-20T01:00:00Z"),
cache(5, `${portable}new`, "2026-10-20T02:00:00Z"),
],
};
return {
actions_caches: [
cache(
6,
"npm-download-Linux-x64-lock",
"2026-10-20T01:00:00Z",
"2026-10-20T01:00:00Z",
),
cache(
7,
"npm-download-Linux-x64-lock",
"2026-10-20T02:00:00Z",
"2026-10-20T03:00:00Z",
),
],
};
},
deleteCache: async (id) => {
deleted.push(id);
},
families: [{ prefix, keepKey }, { prefix: portable }],
dedupeNpmDownloads: true,
});
expect(deleted).toEqual([1, 6]);
expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]);
expect(result.plan[1]?.deletions).toEqual([]);
expect(result.npmDeletions.map((entry) => entry.id)).toEqual([6]);
});
it("dry-run reports the same deletions without calling delete", async () => {
const result = await retainCaches({
repository,
request: async () => ({
actions_caches: [
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 50),
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z", undefined, 60),
cache(3, `${prefix}new`, "2026-10-20T03:00:00Z", undefined, 70),
],
}),
deleteCache: async () => {
throw new Error("dry-run must not delete");
},
families: [{ prefix, keepKey: `${prefix}new` }],
dryRun: true,
});
expect(result.dryRun).toBe(true);
expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]);
});
});
describe("repeated cache ids", () => {
const repository = "morluto/rea";
const prefix = "rea-static-v1-Linux-X64-";
it("treats one cache id on two pages as a single snapshot", () => {
const keepKey = `${prefix}saved`;
const deletions = selectPrefixDeletions(
[
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T03:00:00Z"),
cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T04:00:00Z"),
],
{ prefix, keepKey },
);
expect(deletions.map((entry) => entry.id)).toEqual([1]);
});
it("does not delete when one cache id is listed with inconsistent fields", async () => {
const deleted: number[] = [];
const repeated = cache(3, `${prefix}new`, "2026-10-20T03:00:00Z");
await expect(
retainCaches({
repository,
request: async () => ({
actions_caches: [
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
repeated,
{ ...repeated, version: "b".repeat(64) },
],
}),
deleteCache: async (id) => {
deleted.push(id);
},
families: [{ prefix, keepKey: `${prefix}new` }],
}),
).rejects.toThrow(/inconsistent/u);
expect(deleted).toEqual([]);
});
});
it("runs retention from the command line without deleting on an unsafe prefix", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-cache-retention-"));
const log = join(root, "gh.log");
const gh = join(root, "gh");
await writeFile(
gh,
`#!/bin/sh
printf '%s\\n' "$@" >> "${log}"
exit 1
`,
);
await chmod(gh, 0o755);
expect(() =>
execFileSync("node", ["scripts/ci/cache-retention.mjs"], {
cwd: process.cwd(),
env: {
...process.env,
PATH: `${root}${delimiter}${process.env.PATH ?? ""}`,
CI_REPOSITORY: "morluto/rea",
CACHE_PREFIX: "npm-download-Linux-x64-",
CACHE_KEEP_KEY: "npm-download-Linux-x64-lock",
},
encoding: "utf8",
}),
).toThrow(/one Turbo family/u);
await expect(readFile(log, "utf8")).rejects.toMatchObject({ code: "ENOENT" });
});
+106 -23
View File
@@ -8,10 +8,10 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
allocateByDuration,
estimatedShardCost,
parseTestDurations,
readTestDurations,
weightFileDurations,
scheduleWall,
seedSplitLoaderDurations,
} from "../../ciTestSequencer.js";
describe("CI test history", () => {
@@ -86,28 +86,43 @@ describe("CI test history", () => {
});
});
describe("split loader history", () => {
const source =
"tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts";
const parts = [
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts",
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts",
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts",
];
it("divides a pre-split duration across the four current files", () => {
const seeded = seedSplitLoaderDurations(
new Map([
[source, 192_000],
["tests/other.test.ts", 10],
]),
);
expect(seeded.get(source)).toBe(54_000);
for (const path of parts) expect(seeded.get(path)).toBe(46_000);
expect(seeded.get("tests/other.test.ts")).toBe(10);
expect(
[...seeded.values()].reduce((sum, duration) => sum + duration, 0),
).toBe(192_000 + 10);
});
it("keeps published durations once any split file is in history", () => {
const recorded = new Map([
[source, 20_000],
[parts[0]!, 30_000],
]);
expect(seedSplitLoaderDurations(recorded)).toEqual(recorded);
expect(
seedSplitLoaderDurations(new Map([["tests/other.test.ts", 10]])),
).toEqual(new Map([["tests/other.test.ts", 10]]));
});
});
describe("duration shard allocation", () => {
it("charges serial process-boundary files their full duration and other projects one worker share", () => {
expect(estimatedShardCost("process-boundary", 400)).toBe(400);
expect(estimatedShardCost("domain", 400)).toBe(200);
expect(estimatedShardCost("adapters", 400)).toBe(200);
});
it("fills unknown files from the raw median before serial and shared weighting", () => {
expect(
weightFileDurations([
{ project: "domain", duration: 400 },
{ project: "process-boundary", duration: undefined },
{ project: "adapters", duration: undefined },
]),
).toEqual([200, 400, 200]);
expect(
weightFileDurations([
{ project: "process-boundary", duration: undefined },
]),
).toEqual([undefined]);
});
it.prop([
fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), {
minLength: 1,
@@ -188,4 +203,72 @@ describe("duration shard allocation", () => {
expect(shards!.flat()).toHaveLength(items.length);
expect(new Set(shards!.flat()).size).toBe(items.length);
});
it("keeps each file whole and runs the serial group after the parallel group", () => {
const parallel = (id: string, duration: number) => ({
item: id,
key: id,
duration,
laneWidth: 2,
groupOrder: 0,
});
const serial = (id: string, duration: number) => ({
item: id,
key: id,
duration,
laneWidth: 1,
groupOrder: 1,
});
const paired = [
parallel("wide", 300),
parallel("peer", 300),
parallel("third", 300),
];
expect(scheduleWall(paired.slice(0, 1), 1)).toBe(300);
expect(scheduleWall(paired.slice(0, 2), 1)).toBe(300);
expect(scheduleWall(paired, 1)).toBe(600);
expect(
scheduleWall([parallel("ordinary", 100), serial("tail", 60)], 1),
).toBe(160);
expect(() =>
scheduleWall(
[parallel("wide", 10), { ...parallel("wide", 10), laneWidth: 1 }],
1,
),
).toThrow(RangeError);
const packed = [
parallel("ordinary", 100),
serial("tail-a", 60),
serial("tail-b", 60),
];
expect(allocateByDuration(packed, 2)).toEqual([
["ordinary"],
["tail-a", "tail-b"],
]);
const spread = [
parallel("wide", 300),
serial("tail-a", 100),
serial("tail-b", 100),
parallel("small", 40),
];
const shards = allocateByDuration(spread, 2);
expect(shards).toEqual([["wide"], ["tail-a", "tail-b", "small"]]);
expect(
scheduleWall(
spread.filter((entry) => entry.item === "wide"),
1,
),
).toBe(300);
expect(
Math.max(
...shards!.map((ids) =>
scheduleWall(
spread.filter((entry) => ids.includes(entry.item)),
1,
),
),
),
).toBe(300);
});
});
+159 -40
View File
@@ -60,32 +60,138 @@ export async function readTestDurations(
}
}
type TimedItem<T> = { item: T; key: string; duration: number | undefined };
/**
* Case counts from the competing-loader file before it was split: 54 cases
* stayed in the original file and 46 moved into each of the three new files.
* History that still names only the original path is that combined cost.
*/
const LOADER_HISTORY_SOURCE =
"tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts";
const LOADER_HISTORY_PARTS = [
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts",
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts",
"tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts",
] as const;
const LOADER_HISTORY_REMAINDER_CASES = 54;
const LOADER_HISTORY_PART_CASES = 46;
const LOADER_HISTORY_CASES =
LOADER_HISTORY_REMAINDER_CASES +
LOADER_HISTORY_PART_CASES * LOADER_HISTORY_PARTS.length;
// process-boundary runs its files one at a time. Other projects overlap up to
// the runner worker cap, so equal file-duration sums are not equal wall time.
const PARALLEL_TEST_WORKERS = 2;
export function estimatedShardCost(
project: string,
durationMs: number,
): number {
if (project === "process-boundary") return durationMs;
return durationMs / PARALLEL_TEST_WORKERS;
/**
* Give the split loader files a share of the pre-split duration. Once history
* records any of the new paths, their published durations stand.
*/
export function seedSplitLoaderDurations(
durations: ReadonlyMap<string, number>,
): Map<string, number> {
const combined = durations.get(LOADER_HISTORY_SOURCE);
if (
combined === undefined ||
LOADER_HISTORY_PARTS.some((path) => durations.has(path))
)
return new Map(durations);
const seeded = new Map(durations);
seeded.set(
LOADER_HISTORY_SOURCE,
(combined * LOADER_HISTORY_REMAINDER_CASES) / LOADER_HISTORY_CASES,
);
for (const path of LOADER_HISTORY_PARTS)
seeded.set(
path,
(combined * LOADER_HISTORY_PART_CASES) / LOADER_HISTORY_CASES,
);
return seeded;
}
/** Weight raw durations, filling gaps from the raw median before worker sharing. */
export function weightFileDurations(
entries: readonly { project: string; duration: number | undefined }[],
): (number | undefined)[] {
const known = entries
.flatMap(({ duration }) => (duration === undefined ? [] : [duration]))
.sort((left, right) => left - right);
if (!known.length) return entries.map(() => undefined);
const lower = known[Math.floor((known.length - 1) / 2)]!;
const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2;
return entries.map(({ project, duration }) =>
estimatedShardCost(project, duration ?? median),
type TimedItem<T> = {
item: T;
key: string;
duration: number | undefined;
/**
* Workers for this file's sequence group. The file still occupies one worker
* for its full duration; the other lanes only overlap its peers.
*/
laneWidth?: number;
/**
* Vitest sequence group. Groups run from lowest to highest, so a later group
* adds its own makespan instead of overlapping an earlier one.
*/
groupOrder?: number;
};
type Phase = { laneWidth: number; lanes: number[] };
const laneWidthOf = <T>(entry: TimedItem<T>): number => {
const laneWidth = entry.laneWidth ?? 1;
if (!Number.isInteger(laneWidth) || laneWidth < 1)
throw new RangeError("laneWidth must be a positive integer");
return laneWidth;
};
const groupOrderOf = <T>(entry: TimedItem<T>): number => {
const groupOrder = entry.groupOrder ?? 0;
if (!Number.isInteger(groupOrder) || groupOrder < 0)
throw new RangeError("groupOrder must be a non-negative integer");
return groupOrder;
};
const durationOf = <T>(entry: TimedItem<T>, unknownMs: number): number => {
const duration = entry.duration ?? unknownMs;
if (!Number.isFinite(duration) || duration <= 0)
throw new RangeError("duration must be positive");
return duration;
};
const wallOf = (phases: ReadonlyMap<number, Phase>): number => {
let wall = 0;
for (const phase of phases.values())
wall += phase.lanes.reduce((max, lane) => Math.max(max, lane), 0);
return wall;
};
/** Place one whole file on the least-loaded lane of its sequence group. */
const place = <T>(
phases: ReadonlyMap<number, Phase>,
entry: TimedItem<T>,
duration: number,
): Map<number, Phase> => {
const groupOrder = groupOrderOf(entry);
const laneWidth = laneWidthOf(entry);
const existing = phases.get(groupOrder);
if (existing && existing.laneWidth !== laneWidth)
throw new RangeError("one sequence group must keep a single lane width");
const lanes = existing
? existing.lanes.slice()
: Array.from({ length: laneWidth }, () => 0);
let shortest = 0;
for (let index = 1; index < lanes.length; index += 1)
if (lanes[index]! < lanes[shortest]!) shortest = index;
lanes[shortest] = lanes[shortest]! + duration;
const next = new Map(phases);
next.set(groupOrder, { laneWidth, lanes });
return next;
};
/**
* Wall time of one shard. Each file keeps its full duration on one worker.
* Files overlap only inside a sequence group, up to that group's lane width,
* and later groups run after earlier ones.
*/
export function scheduleWall<T>(
items: readonly TimedItem<T>[],
unknownMs: number,
): number {
const ordered = items.toSorted(
(left, right) =>
durationOf(right, unknownMs) - durationOf(left, unknownMs) ||
(left.key < right.key ? -1 : left.key > right.key ? 1 : 0),
);
return wallOf(
ordered.reduce(
(phases, entry) => place(phases, entry, durationOf(entry, unknownMs)),
new Map<number, Phase>(),
),
);
}
@@ -104,19 +210,29 @@ export function allocateByDuration<T>(
const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2;
const ordered = items.toSorted(
(left, right) =>
(right.duration ?? median) - (left.duration ?? median) ||
durationOf(right, median) - durationOf(left, median) ||
(left.key < right.key ? -1 : left.key > right.key ? 1 : 0),
);
const shards = Array.from({ length: shardCount }, () => ({
duration: 0,
phases: new Map<number, Phase>(),
items: [] as T[],
}));
for (const entry of ordered) {
const lightest = shards.reduce((left, right) =>
right.duration < left.duration ? right : left,
);
lightest.items.push(entry.item);
lightest.duration += entry.duration ?? median;
const duration = durationOf(entry, median);
let chosen = shards[0]!;
let chosenPhases = place(chosen.phases, entry, duration);
let chosenWall = wallOf(chosenPhases);
for (const candidate of shards.slice(1)) {
const phases = place(candidate.phases, entry, duration);
const wall = wallOf(phases);
if (wall < chosenWall) {
chosen = candidate;
chosenPhases = phases;
chosenWall = wall;
}
}
chosen.phases = chosenPhases;
chosen.items.push(entry.item);
}
return shards.map(({ items }) => items);
}
@@ -129,26 +245,29 @@ export class CiTestSequencer extends BaseSequencer {
const shard = this.ctx.config.shard;
if (!historyPath || !shard) return super.shard(files);
const root = this.ctx.config.root;
const durations = await readTestDurations(historyPath, root);
if (!durations) return super.shard(files);
const recorded = await readTestDurations(historyPath, root);
if (!recorded) return super.shard(files);
const durations = seedSplitLoaderDurations(recorded);
const timed = files.map((item) => {
const weighted = files.map((item) => {
const path = relative(root, resolve(root, item.moduleId))
.split(sep)
.join("/");
const projectWorkers = item.project.config.maxWorkers;
const laneWidth =
typeof projectWorkers === "number" &&
Number.isInteger(projectWorkers) &&
projectWorkers >= 1
? projectWorkers
: 1;
return {
item,
key: path + "\0" + item.project.name + "\0" + item.pool,
project: item.project.name,
duration: durations.get(path),
laneWidth,
groupOrder: item.project.config.sequence.groupOrder,
};
});
const costs = weightFileDurations(timed);
const weighted = timed.map((entry, index) => ({
item: entry.item,
key: entry.key,
duration: costs[index],
}));
const shards = allocateByDuration(weighted, shard.count);
return shards ? shards[shard.index - 1]! : super.shard(files);
}