Compare commits

...
Author SHA1 Message Date
Kaoru0822andClaude Opus 5.5 ee58671be4 fix(ci): close storage verifier sockets and trace browser verifier stages
Since 8b4c3de3b added the storage end-to-end check, the real-browser lane
has hit its 10-minute job limit on main and on every PR. The verifier
printed nothing before cancellation, so the stalled step was unknown.
Locally, on macOS with Chrome for Testing 155, it passes in about 70 s.

The storage site was the only verifier site closed without
`closeAllConnections()`. The external browser outlives the site, so a
socket it keeps open can hold `server.close()` and its `close` event
indefinitely. The popup, network and metadata sites already close
connections first.

Write per-stage markers to stderr so a cancelled job shows which step
stalled.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 03:09:42 +00:00
2 changed files with 38 additions and 0 deletions

No files matched your search

@@ -99,6 +99,12 @@ const scenarioFor = (browser, origin) =>
capture: { after_each_step: ["dom"], at_end: ["dom"] }, capture: { after_each_step: ["dom"], at_end: ["dom"] },
}); });
const storageStart = Date.now();
const stage = (name) =>
process.stderr.write(
`[verify:browser:storage] ${name} +${String(Date.now() - storageStart)}ms\n`,
);
const raceWithTimeout = (operation, timeoutMs, onTimeout = () => undefined) => { const raceWithTimeout = (operation, timeoutMs, onTimeout = () => undefined) => {
const controller = new AbortController(); const controller = new AbortController();
return Promise.race([ return Promise.race([
@@ -175,6 +181,8 @@ const startStorageSite = async () => {
return { return {
origin: `http://127.0.0.1:${port}`, origin: `http://127.0.0.1:${port}`,
close: async () => { close: async () => {
// The external browser outlives this site and can keep sockets open.
server.closeAllConnections();
server.close(); server.close();
await once(server, "close"); await once(server, "close");
}, },
@@ -200,8 +208,11 @@ export async function verifyScenarioStorage({
}) { }) {
const site = await startStorageSite(); const site = await startStorageSite();
try { try {
stage("launch");
await verifyLaunchStorage(executable, site.origin); await verifyLaunchStorage(executable, site.origin);
stage("connect");
await verifyConnectedStorage(endpoint, targetId, site.origin); await verifyConnectedStorage(endpoint, targetId, site.origin);
stage("close");
return { return {
mocked: false, mocked: false,
launch: true, launch: true,
@@ -224,6 +235,7 @@ const verifyLaunchStorage = async (executable, origin) => {
origin, origin,
); );
const profilesBefore = await scenarioProfiles(); const profilesBefore = await scenarioProfiles();
stage("launch: scenario CLI");
const result = await runScenarioCli(scenario); const result = await runScenarioCli(scenario);
const capture = result.normalized_result; const capture = result.normalized_result;
assert.ok(capture, "Launch-mode storage scenario returned no result"); assert.ok(capture, "Launch-mode storage scenario returned no result");
@@ -236,6 +248,7 @@ const verifyLaunchStorage = async (executable, origin) => {
...scenario, ...scenario,
actions: [scenario.actions[0]], actions: [scenario.actions[0]],
}); });
stage("launch: fresh in-process scenario");
const fresh = await createBrowserScenarioProvider( const fresh = await createBrowserScenarioProvider(
process.env, process.env,
).captureScenario(freshScenario); ).captureScenario(freshScenario);
@@ -297,7 +310,9 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
"capture_browser_scenario", "capture_browser_scenario",
); );
try { try {
stage("connect: storage partition");
await assertStorageKeyPartition(observer, origin); await assertStorageKeyPartition(observer, origin);
stage("connect: scenario CLI");
const result = await runScenarioCli(scenario); const result = await runScenarioCli(scenario);
const capture = result.normalized_result; const capture = result.normalized_result;
assert.ok(capture, "Connect-mode storage scenario returned no result"); assert.ok(capture, "Connect-mode storage scenario returned no result");
@@ -308,9 +323,12 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
empty: "main-empty-seed", empty: "main-empty-seed",
}); });
assert.equal(capture.browser.cleanup, "disconnected-external"); assert.equal(capture.browser.cleanup, "disconnected-external");
stage("connect: caller debugger pause");
await verifyCallerDebuggerPause(observer, origin, scenario); await verifyCallerDebuggerPause(observer, origin, scenario);
stage("connect: cancellation");
await verifyCancelledStorage(observer, endpoint, targetId, scenario); await verifyCancelledStorage(observer, endpoint, targetId, scenario);
} finally { } finally {
stage("connect: restore target");
await raceWithTimeout(restoreTarget(observer, target.url), 1_000).catch( await raceWithTimeout(restoreTarget(observer, target.url), 1_000).catch(
() => undefined, () => undefined,
); );
+20
View File
@@ -53,6 +53,12 @@ const SCENARIO_URL_SECRET_VALUE = "rea-browser-url-verifier-secret";
process.env.REA_BROWSER_VERIFIER_SECRET = SCENARIO_SECRET_VALUE; process.env.REA_BROWSER_VERIFIER_SECRET = SCENARIO_SECRET_VALUE;
process.env.REA_BROWSER_VERIFIER_URL = SCENARIO_URL_SECRET_VALUE; process.env.REA_BROWSER_VERIFIER_URL = SCENARIO_URL_SECRET_VALUE;
const verifierRun = createVerifierRun(); const verifierRun = createVerifierRun();
const verifierStart = Date.now();
// Stage markers locate a stalled step when CI cancels the job at its limit.
const stage = (name) =>
process.stderr.write(
`[verify:browser] ${name} +${String(Date.now() - verifierStart)}ms\n`,
);
const executable = await browserExecutable(); const executable = await browserExecutable();
const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-")); const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
@@ -109,6 +115,7 @@ try {
); );
if (!observed.ok) throw observed.error; if (!observed.ok) throw observed.error;
assertObservation(observed.value, site.origin); assertObservation(observed.value, site.origin);
stage("verifyBrowserDomDestinations");
const domDestinations = await verifyBrowserDomDestinations({ const domDestinations = await verifyBrowserDomDestinations({
cdp_endpoint: endpoint, cdp_endpoint: endpoint,
target_id: target, target_id: target,
@@ -168,6 +175,7 @@ try {
); );
assertSensitiveShapes(withSource.value); assertSensitiveShapes(withSource.value);
stage("verifyPublicBundle");
const bundle = await verifyPublicBundle(endpoint, target, site.origin); const bundle = await verifyPublicBundle(endpoint, target, site.origin);
const captureDiff = await provider.compareCaptures( const captureDiff = await provider.compareCaptures(
@@ -231,6 +239,7 @@ try {
throw new Error("Real Chrome same-origin SPA timeline was missing"); throw new Error("Real Chrome same-origin SPA timeline was missing");
pageProxy = await startPageCdpProxy(endpoint); pageProxy = await startPageCdpProxy(endpoint);
stage("verifyPageScopedTransport");
await verifyPageScopedTransport(provider, pageProxy, site.origin); await verifyPageScopedTransport(provider, pageProxy, site.origin);
const attachedScenarioInput = browserScenario( const attachedScenarioInput = browserScenario(
{ {
@@ -277,16 +286,19 @@ try {
throw new Error("Scenario attachment terminated its external browser"); throw new Error("Scenario attachment terminated its external browser");
const profilesBefore = await scenarioProfiles(); const profilesBefore = await scenarioProfiles();
stage("verifyScenarioEnvironment");
const scenarioEnvironment = await verifyScenarioEnvironment( const scenarioEnvironment = await verifyScenarioEnvironment(
endpoint, endpoint,
target, target,
site.origin, site.origin,
); );
stage("verifyScenarioStorage");
const scenarioStorage = await verifyScenarioStorage({ const scenarioStorage = await verifyScenarioStorage({
executable, executable,
endpoint, endpoint,
targetId: target, targetId: target,
}); });
stage("launchedScenario");
const launchedScenario = await createBrowserScenarioProvider( const launchedScenario = await createBrowserScenarioProvider(
process.env, process.env,
).captureScenario( ).captureScenario(
@@ -308,6 +320,7 @@ try {
if ([...profilesAfter].some((entry) => !profilesBefore.has(entry))) if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
throw new Error("Scenario launch retained a temporary browser profile"); throw new Error("Scenario launch retained a temporary browser profile");
assertScenarioCapture(launchedScenario.value); assertScenarioCapture(launchedScenario.value);
stage("verifyScenarioFailureEvidence");
const scenarioFailure = await verifyScenarioFailureEvidence( const scenarioFailure = await verifyScenarioFailureEvidence(
executable, executable,
site.origin, site.origin,
@@ -339,18 +352,24 @@ try {
`Browser scenario included unselected structured value: ${unselectedValue}`, `Browser scenario included unselected structured value: ${unselectedValue}`,
); );
stage("verifyLargeScreenshotE2e");
const largeScreenshot = await verifyLargeScreenshotE2e(endpoint, site.origin); const largeScreenshot = await verifyLargeScreenshotE2e(endpoint, site.origin);
stage("verifyPopupEventCoverage");
const popupEvents = await verifyPopupEventCoverage(executable); const popupEvents = await verifyPopupEventCoverage(executable);
stage("verifyBrowserNetworkEvidence");
const networkEvidence = await verifyBrowserNetworkEvidence( const networkEvidence = await verifyBrowserNetworkEvidence(
executable, executable,
undefined, undefined,
{ cdp_endpoint: endpoint, target_id: target }, { cdp_endpoint: endpoint, target_id: target },
); );
stage("verifyBrowserScriptExport");
const scriptExport = await verifyBrowserScriptExport(executable, undefined, { const scriptExport = await verifyBrowserScriptExport(executable, undefined, {
cdp_endpoint: endpoint, cdp_endpoint: endpoint,
target_id: target, target_id: target,
}); });
stage("verifyBrowserModules");
const moduleTrace = await verifyBrowserModules(executable); const moduleTrace = await verifyBrowserModules(executable);
stage("verifyBrowserCaptureMetadataBudget");
const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint); const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint);
report = { report = {
browser: observed.value.browser.product, browser: observed.value.browser.product,
@@ -393,6 +412,7 @@ try {
process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n"); process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n");
if (browser !== undefined) await stopProcess(browser); if (browser !== undefined) await stopProcess(browser);
process.stderr.write("Browser verifier cleanup: closing fixture site\n"); process.stderr.write("Browser verifier cleanup: closing fixture site\n");
stage("cleanup");
await site.close(); await site.close();
await rm(profile, { await rm(profile, {
recursive: true, recursive: true,