From 1d3f8f4e67b5904224aa0bfe86d8aca254a4e061 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 18:34:57 -0500 Subject: [PATCH 1/9] Add HTML prerendering for FileDef entries during indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend file indexing with a render phase that creates a FileDef instance and captures its display format templates (isolated, embedded, fitted, atom, head, icon) as prerendered HTML, stored in the existing boxel_index HTML columns. Rendering is non-fatal — if it fails, file metadata is still stored without HTML. Co-Authored-By: Claude Opus 4.5 --- .../host/app/components/card-prerender.gts | 130 +++++++++ packages/host/app/routes/render.ts | 44 +++ .../realm-server/prerender/prerender-app.ts | 171 +++++++++++ .../realm-server/prerender/prerenderer.ts | 133 +++++++++ .../prerender/remote-prerenderer.ts | 24 ++ .../realm-server/prerender/render-runner.ts | 273 +++++++++++++++++- .../tests/definition-lookup-test.ts | 18 ++ .../tests/prerender-proxy-test.ts | 11 + packages/runtime-common/index-runner.ts | 41 +++ packages/runtime-common/index-writer.ts | 12 + packages/runtime-common/index.ts | 19 ++ .../runtime-common/render-route-options.ts | 13 + 12 files changed, 888 insertions(+), 1 deletion(-) diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index e3a708360d1..4c59fe7a9b0 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -22,6 +22,8 @@ import { type RenderError, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type Prerenderer, type Format, type PrerenderMeta, @@ -93,6 +95,7 @@ export default class CardPrerender extends Component { prerenderCard: this.prerender.bind(this), prerenderModule: this.prerenderModule.bind(this), prerenderFileExtract: this.prerenderFileExtract.bind(this), + prerenderFileRender: this.prerenderFileRenderPublic.bind(this), }; this.localIndexer.setup(this.#prerendererDelegate); window.addEventListener('boxel-render-error', this.#handleRenderErrorEvent); @@ -203,6 +206,24 @@ export default class CardPrerender extends Component { }); } + private async prerenderFileRenderPublic( + args: FileRenderArgs, + ): Promise { + return await withRenderContext(async () => { + try { + let run = () => this.fileRenderPrerenderTask.perform(args); + return isTesting() ? await run() : await withTimersBlocked(run); + } catch (e: any) { + if (!didCancel(e)) { + throw e; + } + } + throw new Error( + `card-prerender component is missing or being destroyed before file render prerender of url ${args.url} was completed`, + ); + }); + } + // This emulates the job of the Prerenderer that runs in the server private prerenderTask = enqueueTask( async ({ @@ -404,6 +425,115 @@ export default class CardPrerender extends Component { }, ); + private fileRenderPrerenderTask = enqueueTask( + async ({ + url, + fileData, + types, + renderOptions, + }: FileRenderArgs): Promise => { + this.#nonce++; + let shouldClearCache = this.#consumeClearCacheForRender( + Boolean(renderOptions?.clearCache), + ); + let initialRenderOptions: RenderRouteOptions = { + ...(renderOptions ?? {}), + fileRender: true, + fileDefCodeRef: fileData.fileDefCodeRef, + }; + if (shouldClearCache) { + initialRenderOptions.clearCache = true; + this.loaderService.resetLoader({ + clearFetchCache: true, + reason: 'card-prerender file render clearCache', + }); + this.store.resetCache(); + } else { + delete initialRenderOptions.clearCache; + } + + // Stash file data for the render route to consume + (globalThis as any).__boxelFileRenderData = fileData; + + let error: RenderError | undefined; + let isolatedHTML: string | null = null; + let headHTML: string | null = null; + let atomHTML: string | null = null; + let iconHTML: string | null = null; + let embeddedHTML: Record | null = null; + let fittedHTML: Record | null = null; + + try { + let subsequentRenderOptions = + omitOneTimeOptions(initialRenderOptions); + isolatedHTML = await this.renderHTML.perform( + url, + 'isolated', + 0, + initialRenderOptions, + ); + headHTML = await this.renderHTML.perform( + url, + 'head', + 0, + subsequentRenderOptions, + ); + atomHTML = await this.renderHTML.perform( + url, + 'atom', + 0, + subsequentRenderOptions, + ); + iconHTML = await this.renderIcon.perform( + url, + subsequentRenderOptions, + ); + if (types.length > 0) { + embeddedHTML = await this.renderAncestors.perform( + url, + 'embedded', + types, + subsequentRenderOptions, + ); + fittedHTML = await this.renderAncestors.perform( + url, + 'fitted', + types, + subsequentRenderOptions, + ); + } + } catch (e: any) { + try { + error = { ...JSON.parse(e.message), type: 'file-error' }; + } catch (_err) { + let cardErr = new CardError(e.message); + cardErr.stack = e.stack; + error = { + error: { + ...cardErr.toJSON(), + deps: [url], + additionalErrors: null, + }, + type: 'file-error', + }; + } + this.store.resetCache(); + } finally { + delete (globalThis as any).__boxelFileRenderData; + } + + return { + isolatedHTML, + headHTML, + atomHTML, + embeddedHTML, + fittedHTML, + iconHTML, + ...(error ? { error } : {}), + }; + }, + ); + #moduleModelContext(): ModuleModelContext { return { router: this.router, diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index e87b83adbe7..41b1ecb7006 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -249,6 +249,50 @@ export default class RenderRoute extends Route { this.currentTransition = undefined; return model; } + if (parsedOptions.fileRender) { + let fileRenderData = (globalThis as any).__boxelFileRenderData as + | { resource: any; fileDefCodeRef: { module: string; name: string } } + | undefined; + if (!fileRenderData) { + throw new Error('fileRender mode requires __boxelFileRenderData'); + } + let { resource, fileDefCodeRef } = fileRenderData; + let cardApi = await this.loaderService.loader.import( + `${baseRealm.url}card-api`, + ); + let doc = { data: resource }; + let instance = (await cardApi.createFromSerialized( + resource, + doc, + resource.id ? new URL(resource.id) : undefined, + )) as unknown as CardDef; + + let state = new TrackedMap(); + state.set('status', 'ready'); + let readyDeferred = new Deferred(); + readyDeferred.fulfill(); + let model: Model = { + instance, + nonce, + cardId: id, + renderOptions: parsedOptions, + get status(): RenderStatus { + return (state.get('status') as RenderStatus) ?? 'loading'; + }, + get ready(): boolean { + return (state.get('status') as RenderStatus) === 'ready'; + }, + readyPromise: readyDeferred.promise, + }; + this.#modelStates.set(model, { + state, + readyDeferred, + isReady: true, + }); + (globalThis as any).__renderModel = model; + this.currentTransition = undefined; + return model; + } // This is for host tests (globalThis as any).__renderModel = undefined; diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 74dccfb886d..cc89c48b92c 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -10,6 +10,7 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, } from '@cardstack/runtime-common'; import { ecsMetadata, @@ -336,6 +337,176 @@ export function buildPrerenderApp(options: { }, }); + // File render route needs additional attributes (fileData, types) + // beyond what registerPrerenderRoute handles, so we register it directly. + router.post('/prerender-file-render', async (ctxt: Koa.Context) => { + try { + let request = await fetchRequestFromContext(ctxt); + let raw = await request.text(); + let body: any; + try { + body = raw ? JSON.parse(raw) : {}; + } catch (e) { + ctxt.status = 400; + ctxt.body = { + errors: [{ status: 400, message: 'Invalid JSON body' }], + }; + return; + } + + let attrs = body?.data?.attributes ?? {}; + let rawUrl = attrs.url; + let rawAuth = attrs.auth; + let rawRealm = attrs.realm; + let renderOptions: RenderRouteOptions = + attrs.renderOptions && + typeof attrs.renderOptions === 'object' && + !Array.isArray(attrs.renderOptions) + ? (attrs.renderOptions as RenderRouteOptions) + : {}; + let fileData = attrs.fileData; + let types = attrs.types; + + let isNonEmptyString = (value: unknown): value is string => + typeof value === 'string' && value.trim().length > 0; + + let missing = [ + { value: rawUrl, name: 'url' }, + { value: rawRealm, name: 'realm' }, + { value: rawAuth, name: 'auth' }, + ] + .filter(({ value }) => !isNonEmptyString(value)) + .map(({ name }) => name); + + if (!fileData) { + missing.push('fileData'); + } + if (!Array.isArray(types)) { + missing.push('types'); + } + + log.debug( + `received file render prerender request ${rawUrl}: realm=${rawRealm}`, + ); + if (missing.length > 0) { + ctxt.status = 400; + ctxt.body = { + errors: [ + { + status: 400, + message: `Missing or invalid required attributes: ${missing.join(', ')}`, + }, + ], + }; + return; + } + + let realm = rawRealm as string; + let url = rawUrl as string; + let auth = rawAuth as string; + + let start = Date.now(); + let execPromise = prerenderer + .prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + renderOptions, + }) + .then((result) => ({ result })); + let drainPromise = options.drainingPromise + ? options.drainingPromise.then(() => ({ draining: true as const })) + : null; + let raceResult = drainPromise + ? await Promise.race([execPromise, drainPromise]) + : await execPromise; + if ('draining' in raceResult) { + execPromise.catch((e) => + log.debug( + 'file render prerender execute settled after drain (ignored):', + e, + ), + ); + ctxt.status = PRERENDER_SERVER_DRAINING_STATUS_CODE; + ctxt.set( + PRERENDER_SERVER_STATUS_HEADER, + PRERENDER_SERVER_STATUS_DRAINING, + ); + ctxt.body = { + errors: [ + { + status: PRERENDER_SERVER_DRAINING_STATUS_CODE, + message: 'Prerender server draining', + }, + ], + }; + return; + } + let { response, timings, pool } = raceResult.result; + let totalMs = Date.now() - start; + let poolFlags = Object.entries({ + reused: pool.reused, + evicted: pool.evicted, + timedOut: pool.timedOut, + }) + .filter(([, value]) => value === true) + .map(([key]) => key) + .join(', '); + let poolFlagSuffix = + poolFlags.length > 0 ? ` flags=[${poolFlags}]` : ''; + log.info( + 'file render prerendered %s total=%dms launch=%dms render=%dms pageId=%s realm=%s%s', + url, + totalMs, + timings.launchMs, + timings.renderMs, + pool.pageId, + pool.realm, + poolFlagSuffix, + ); + ctxt.status = 201; + ctxt.set('Content-Type', 'application/vnd.api+json'); + ctxt.body = { + data: { + type: 'prerender-file-render-result', + id: url, + attributes: response, + }, + meta: { + timing: { + launchMs: timings.launchMs, + renderMs: timings.renderMs, + totalMs, + }, + pool, + }, + }; + if (pool.timedOut) { + log.warn(`file render of ${url} timed out`); + } + const fileResponse = response as FileRenderResponse; + if (fileResponse.error) { + log.debug( + `file render of ${url} resulted in error doc:\n${JSON.stringify(fileResponse.error, null, 2)}`, + ); + } + } catch (err: any) { + Sentry.captureException(err); + log.error('Unhandled error in /prerender-file-render:', err); + ctxt.status = 500; + ctxt.body = { + errors: [ + { + status: 500, + message: err?.message ?? 'Unknown error', + }, + ], + }; + } + }); + app .use((ctxt: Koa.Context, next: Koa.Next) => { if ( diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 5af7c6b38d1..22e2a96a9f6 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -3,6 +3,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, logger, } from '@cardstack/runtime-common'; import { BrowserManager } from './browser-manager'; @@ -460,6 +462,137 @@ export class Prerenderer { throw new Error(`file extract prerender attempts exhausted for ${url}`); } + async prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions, + }: { + realm: string; + url: string; + auth: string; + fileData: FileRenderArgs['fileData']; + types: string[]; + opts?: { timeoutMs?: number; simulateTimeoutMs?: number }; + renderOptions?: RenderRouteOptions; + }): Promise<{ + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + if (this.#stopped) { + throw new Error('Prerenderer has been stopped and cannot be used'); + } + let attemptOptions = renderOptions; + let lastResult: + | { + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + } + | undefined; + for (let attempt = 0; attempt < 3; attempt++) { + let result: { + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }; + try { + result = await this.#renderRunner.prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions: attemptOptions, + }); + } catch (e) { + log.error( + `file render prerender attempt for ${url} (realm ${realm}) failed with error, restarting browser`, + e, + ); + await this.#restartBrowser(); + try { + result = await this.#renderRunner.prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions: attemptOptions, + }); + } catch (e2) { + log.error( + `file render prerender attempt for ${url} (realm ${realm}) failed again after browser restart`, + e2, + ); + throw e2; + } + } + lastResult = result; + + let retrySignature = this.#renderRunner.shouldRetryWithClearCache( + result.response, + ); + let isClearCacheAttempt = attemptOptions?.clearCache === true; + + if (!isClearCacheAttempt && retrySignature) { + log.warn( + `retrying file render prerender for ${url} with clearCache due to error signature: ${retrySignature.join( + ' | ', + )}`, + ); + attemptOptions = { + ...(attemptOptions ?? {}), + clearCache: true, + }; + continue; + } + + if (isClearCacheAttempt && retrySignature && result.response.error) { + log.warn( + `file render prerender retry with clearCache did not resolve error signature ${retrySignature.join( + ' | ', + )} for ${url}`, + ); + } + + return result; + } + if (lastResult) { + if (lastResult.response.error) { + log.error( + `file render prerender attempts exhausted for ${url} in realm ${realm}, returning last error response`, + ); + } + return lastResult; + } + throw new Error(`file render prerender attempts exhausted for ${url}`); + } + async #restartBrowser(): Promise { log.warn('Restarting prerender browser'); await this.#pagePool.closeAll(); diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index a93829fda2a..5eb111081dd 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -3,6 +3,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type RenderRouteOptions, logger, } from '@cardstack/runtime-common'; @@ -60,6 +62,7 @@ export function createRemotePrerenderer( url: string; auth: string; renderOptions?: RenderRouteOptions; + [key: string]: any; }, ): Promise { validatePrerenderAttributes(type, attributes); @@ -188,6 +191,27 @@ export function createRemotePrerenderer( }, ); }, + async prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + renderOptions, + }: FileRenderArgs) { + return await requestWithRetry( + 'prerender-file-render', + 'prerender-file-render-request', + { + realm, + url, + auth, + fileData, + types, + renderOptions: renderOptions ?? {}, + }, + ); + }, }; } diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 849940ff6bb..7bce8ba4ce9 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -4,6 +4,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type RenderRouteOptions, serializeRenderRouteOptions, logger, @@ -688,8 +690,277 @@ export class RenderRunner { } } + async prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions, + }: { + realm: string; + url: string; + auth: string; + fileData: FileRenderArgs['fileData']; + types: string[]; + opts?: { timeoutMs?: number; simulateTimeoutMs?: number }; + renderOptions?: RenderRouteOptions; + }): Promise<{ + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + this.#nonce++; + log.info( + `file render prerendering url ${url}, nonce=${this.#nonce} realm=${realm}`, + ); + + const { page, reused, launchMs, pageId, release } = + await this.#getPageForRealm(realm, auth); + const poolInfo = { + pageId: pageId ?? 'unknown', + realm, + reused, + evicted: false, + timedOut: false, + }; + this.#pagePool.resetConsoleErrors(pageId); + const markTimeout = (err?: RenderError) => { + if (!poolInfo.timedOut && err?.error?.title === 'Render timeout') { + poolInfo.timedOut = true; + } + }; + try { + await page.evaluate((sessionAuth) => { + localStorage.setItem('boxel-session', sessionAuth); + }, auth); + + // Stash file data on globalThis for the render route to consume + await page.evaluate((data) => { + (globalThis as any).__boxelFileRenderData = data; + }, fileData); + + let renderStart = Date.now(); + let error: RenderError | undefined; + let shortCircuit = false; + let options: RenderRouteOptions = { + ...(renderOptions ?? {}), + fileRender: true, + fileDefCodeRef: fileData.fileDefCodeRef, + }; + let serializedOptions = serializeRenderRouteOptions(options); + let optionsSegment = encodeURIComponent(serializedOptions); + const captureOptions: CaptureOptions = { + expectedId: url.replace(/\.json$/i, ''), + expectedNonce: String(this.#nonce), + simulateTimeoutMs: opts?.simulateTimeoutMs, + }; + + log.debug( + `file render: visit ${url} at: ${this.#boxelHostURL}/render/${encodeURIComponent(url)}/${this.#nonce}/${optionsSegment}/html/isolated/0`, + ); + + // Render isolated HTML first + let result = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.html', + url, + String(this.#nonce), + serializedOptions, + 'isolated', + '0', + ); + return await captureResult(page, 'innerHTML', captureOptions); + }, + opts?.timeoutMs, + ); + let isolatedHTML: string | null = null; + if (isRenderError(result)) { + error = result; + markTimeout(error); + let evicted = await this.#maybeEvict( + realm, + 'file isolated render', + result as RenderError, + ); + if (evicted) { + poolInfo.evicted = true; + shortCircuit = true; + } + if (this.#isAuthError(error)) { + shortCircuit = true; + } + } else { + let capture = result as RenderCapture; + if (capture.status === 'ready') { + isolatedHTML = capture.value; + } else { + let capErr = this.#captureToError(capture); + if (!error && capErr) { + error = capErr; + } + markTimeout(capErr); + let evicted = await this.#maybeEvict( + realm, + 'file isolated render', + capErr, + ); + if (evicted) { + poolInfo.evicted = true; + shortCircuit = true; + } + if (this.#isAuthError(error)) { + shortCircuit = true; + } + } + } + + if (shortCircuit) { + let response: FileRenderResponse = { + ...(error ? { error } : {}), + iconHTML: null, + isolatedHTML, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + }; + response.error = this.#mergeConsoleErrors(pageId, response.error); + return { + response, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } + + // Render remaining formats (no meta step needed for files) + let headHTML: string | null = null, + atomHTML: string | null = null, + iconHTML: string | null = null, + embeddedHTML: Record | null = null, + fittedHTML: Record | null = null; + + if (!shortCircuit) { + let headHTMLResult = await this.#step(realm, 'file head render', () => + withTimeout( + page, + () => renderHTML(page, 'head', 0, captureOptions), + opts?.timeoutMs, + ), + ); + if (headHTMLResult.ok) { + headHTML = headHTMLResult.value as string; + } else { + error = error ?? headHTMLResult.error; + markTimeout(headHTMLResult.error); + if (headHTMLResult.evicted) { + poolInfo.evicted = true; + shortCircuit = true; + } + } + } + + if (!shortCircuit && types.length > 0) { + const steps: Array<{ + name: string; + cb: () => Promise | RenderError>; + assign: (value: string | Record) => void; + }> = [ + { + name: 'file fitted render', + cb: () => + renderAncestors(page, 'fitted', types, captureOptions), + assign: (v: string | Record) => { + fittedHTML = v as Record; + }, + }, + { + name: 'file embedded render', + cb: () => + renderAncestors(page, 'embedded', types, captureOptions), + assign: (v: string | Record) => { + embeddedHTML = v as Record; + }, + }, + { + name: 'file atom render', + cb: () => renderHTML(page, 'atom', 0, captureOptions), + assign: (v: string | Record) => { + atomHTML = v as string; + }, + }, + { + name: 'file icon render', + cb: () => renderIcon(page, captureOptions), + assign: (v: string | Record) => { + iconHTML = v as string; + }, + }, + ]; + + for (let step of steps) { + if (shortCircuit) break; + let res = await this.#step(realm, step.name, () => + withTimeout(page, step.cb, opts?.timeoutMs), + ); + if (res.ok) { + step.assign(res.value); + } else { + error = error ?? res.error; + markTimeout(res.error); + if (res.evicted) { + poolInfo.evicted = true; + shortCircuit = true; + break; + } + if (this.#isAuthError(error)) { + shortCircuit = true; + break; + } + } + } + } + + let response: FileRenderResponse = { + ...(error ? { error } : {}), + iconHTML, + isolatedHTML, + headHTML, + atomHTML, + embeddedHTML, + fittedHTML, + }; + response.error = this.#mergeConsoleErrors(pageId, response.error); + return { + response, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } finally { + // Clean up globalThis data + await page + .evaluate(() => { + delete (globalThis as any).__boxelFileRenderData; + }) + .catch(() => { + /* best-effort cleanup */ + }); + release(); + } + } + shouldRetryWithClearCache( - response: RenderResponse | ModuleRenderResponse | FileExtractResponse, + response: RenderResponse | ModuleRenderResponse | FileExtractResponse | FileRenderResponse, ): readonly string[] | undefined { let renderError = response.error?.error; if (!renderError) { diff --git a/packages/realm-server/tests/definition-lookup-test.ts b/packages/realm-server/tests/definition-lookup-test.ts index dada6a81cd9..b50d41257e6 100644 --- a/packages/realm-server/tests/definition-lookup-test.ts +++ b/packages/realm-server/tests/definition-lookup-test.ts @@ -147,6 +147,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, }; definitionLookup = new CachingDefinitionLookup( dbAdapter, @@ -323,6 +326,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls++; let moduleAlias = trimExecutableExtension(new URL(args.url)).href; @@ -410,6 +416,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls++; if (!modulePresent) { @@ -484,6 +493,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls.set(args.url, (calls.get(args.url) ?? 0) + 1); switch (args.url) { @@ -627,6 +639,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls.set(args.url, (calls.get(args.url) ?? 0) + 1); switch (args.url) { @@ -779,6 +794,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { switch (args.url) { case deepModule: { diff --git a/packages/realm-server/tests/prerender-proxy-test.ts b/packages/realm-server/tests/prerender-proxy-test.ts index e01a6750741..07a8f9b503d 100644 --- a/packages/realm-server/tests/prerender-proxy-test.ts +++ b/packages/realm-server/tests/prerender-proxy-test.ts @@ -82,6 +82,17 @@ module(basename(__filename), function () { deps: [], }; }, + async prerenderFileRender(args) { + renderCalls.push({ kind: 'file-render', args }); + return { + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + }; + }, }; return { prerenderer, renderCalls }; diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 113bcb2a917..179be5328e2 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -20,6 +20,7 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, type ResolvedCodeRef, type Batch, type LooseCardResource, @@ -967,6 +968,40 @@ export class IndexRunner { ); let fileTypes = extractResult.types ?? fallbackTypes; + // Phase 2: Render HTML for file entry (non-fatal) + let renderResult: FileRenderResponse | undefined; + if (extractResult.resource) { + try { + let renderClearCache = this.#consumeClearCacheForRender(); + let fileRenderOptions: RenderRouteOptions = { + fileRender: true, + fileDefCodeRef, + ...(renderClearCache ? { clearCache: true as const } : {}), + }; + renderResult = await this.#prerenderer.prerenderFileRender({ + url: fileURL, + realm: this.#realmURL.href, + auth: this.#auth, + fileData: { + resource: extractResult.resource, + fileDefCodeRef, + }, + types: fileTypes, + renderOptions: fileRenderOptions, + }); + if (renderResult?.error) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)} file render produced error for ${path}, continuing without HTML: ${renderResult.error.error?.message}`, + ); + renderResult = undefined; + } + } catch (err: any) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)} file render failed for ${path}, continuing without HTML: ${err.message}`, + ); + } + } + await this.batch.updateEntry(entryURL, { type: 'file', lastModified, @@ -982,6 +1017,12 @@ export class IndexRunner { }, types: fileTypes, displayNames: [], + isolatedHtml: renderResult?.isolatedHTML ?? undefined, + headHtml: renderResult?.headHTML ?? undefined, + atomHtml: renderResult?.atomHTML ?? undefined, + embeddedHtml: renderResult?.embeddedHTML ?? undefined, + fittedHtml: renderResult?.fittedHTML ?? undefined, + iconHTML: renderResult?.iconHTML ?? undefined, }); this.stats.filesIndexed++; } diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 72f677e9d1f..79f6b5a37c4 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -111,6 +111,12 @@ export interface FileEntry { resource?: FileMetaResource | null; types?: string[]; displayNames?: string[]; + isolatedHtml?: string; + headHtml?: string; + embeddedHtml?: Record; + fittedHtml?: Record; + atomHtml?: string; + iconHTML?: string; } export class Batch { @@ -341,6 +347,12 @@ export class Batch { search_doc: entry.searchData ?? null, types: entry.types ?? null, display_names: entry.displayNames ?? null, + isolated_html: entry.isolatedHtml ?? null, + head_html: entry.headHtml ?? null, + embedded_html: entry.embeddedHtml ?? null, + fitted_html: entry.fittedHtml ?? null, + atom_html: entry.atomHtml ?? null, + icon_html: entry.iconHTML ?? null, last_modified: entry.lastModified, resource_created_at: entry.resourceCreatedAt, error_doc: null, diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 2be0c95e246..29b6b123ff1 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -67,6 +67,24 @@ export interface FileExtractResponse { mismatch?: true; } +export interface FileRenderResponse { + isolatedHTML: string | null; + headHTML: string | null; + atomHTML: string | null; + embeddedHTML: Record | null; + fittedHTML: Record | null; + iconHTML: string | null; + error?: RenderError; +} + +export type FileRenderArgs = ModulePrerenderArgs & { + fileData: { + resource: FileMetaResource; + fileDefCodeRef: ResolvedCodeRef; + }; + types: string[]; +}; + export interface ModuleDefinitionResult { type: 'definition'; moduleURL: string; // node resolution w/o extension @@ -101,6 +119,7 @@ export interface Prerenderer { prerenderCard(args: PrerenderCardArgs): Promise; prerenderModule(args: ModulePrerenderArgs): Promise; prerenderFileExtract(args: ModulePrerenderArgs): Promise; + prerenderFileRender(args: FileRenderArgs): Promise; } export type RealmAction = 'read' | 'write' | 'realm-owner' | 'assume-user'; diff --git a/packages/runtime-common/render-route-options.ts b/packages/runtime-common/render-route-options.ts index 00cf2f9954e..21219490011 100644 --- a/packages/runtime-common/render-route-options.ts +++ b/packages/runtime-common/render-route-options.ts @@ -5,6 +5,7 @@ import type { ResolvedCodeRef } from './code-ref'; export interface RenderRouteOptions { clearCache?: true; fileExtract?: true; + fileRender?: true; fileDefCodeRef?: ResolvedCodeRef; fileContentHash?: string; } @@ -30,6 +31,12 @@ export function parseRenderRouteOptions( options.fileContentHash = parsed.fileContentHash; } } + if (parsed.fileRender) { + options.fileRender = true; + if (isResolvedCodeRef(parsed.fileDefCodeRef)) { + options.fileDefCodeRef = parsed.fileDefCodeRef; + } + } return options; } catch { return {}; @@ -52,5 +59,11 @@ export function serializeRenderRouteOptions( serialized.fileContentHash = options.fileContentHash; } } + if (options.fileRender) { + serialized.fileRender = true; + if (options.fileDefCodeRef) { + serialized.fileDefCodeRef = options.fileDefCodeRef; + } + } return stringify(serialized) ?? '{}'; } From c1737ca82eab3244a8d26caafeb4a5300461f4d5 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 19:18:30 -0500 Subject: [PATCH 2/9] Fix lint errors, prettier formatting, and type issues - Remove unused fileDefCodeRef destructuring in render route - Fix prettier formatting in card-prerender, prerender-app, render-runner - Add 'file-render' to kind union type in prerender-proxy-test - Fix shouldRetryWithClearCache signature line wrapping Co-Authored-By: Claude Opus 4.5 --- packages/host/app/components/card-prerender.gts | 8 ++------ packages/host/app/routes/render.ts | 2 +- packages/realm-server/prerender/prerender-app.ts | 3 +-- packages/realm-server/prerender/render-runner.ts | 12 +++++++----- packages/realm-server/tests/prerender-proxy-test.ts | 2 +- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 4c59fe7a9b0..3e55b5c29c1 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -464,8 +464,7 @@ export default class CardPrerender extends Component { let fittedHTML: Record | null = null; try { - let subsequentRenderOptions = - omitOneTimeOptions(initialRenderOptions); + let subsequentRenderOptions = omitOneTimeOptions(initialRenderOptions); isolatedHTML = await this.renderHTML.perform( url, 'isolated', @@ -484,10 +483,7 @@ export default class CardPrerender extends Component { 0, subsequentRenderOptions, ); - iconHTML = await this.renderIcon.perform( - url, - subsequentRenderOptions, - ); + iconHTML = await this.renderIcon.perform(url, subsequentRenderOptions); if (types.length > 0) { embeddedHTML = await this.renderAncestors.perform( url, diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index 41b1ecb7006..05bdfd9d884 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -256,7 +256,7 @@ export default class RenderRoute extends Route { if (!fileRenderData) { throw new Error('fileRender mode requires __boxelFileRenderData'); } - let { resource, fileDefCodeRef } = fileRenderData; + let { resource } = fileRenderData; let cardApi = await this.loaderService.loader.import( `${baseRealm.url}card-api`, ); diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index cc89c48b92c..91f00b742a8 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -454,8 +454,7 @@ export function buildPrerenderApp(options: { .filter(([, value]) => value === true) .map(([key]) => key) .join(', '); - let poolFlagSuffix = - poolFlags.length > 0 ? ` flags=[${poolFlags}]` : ''; + let poolFlagSuffix = poolFlags.length > 0 ? ` flags=[${poolFlags}]` : ''; log.info( 'file render prerendered %s total=%dms launch=%dms render=%dms pageId=%s realm=%s%s', url, diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 7bce8ba4ce9..222704c4d4b 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -878,16 +878,14 @@ export class RenderRunner { }> = [ { name: 'file fitted render', - cb: () => - renderAncestors(page, 'fitted', types, captureOptions), + cb: () => renderAncestors(page, 'fitted', types, captureOptions), assign: (v: string | Record) => { fittedHTML = v as Record; }, }, { name: 'file embedded render', - cb: () => - renderAncestors(page, 'embedded', types, captureOptions), + cb: () => renderAncestors(page, 'embedded', types, captureOptions), assign: (v: string | Record) => { embeddedHTML = v as Record; }, @@ -960,7 +958,11 @@ export class RenderRunner { } shouldRetryWithClearCache( - response: RenderResponse | ModuleRenderResponse | FileExtractResponse | FileRenderResponse, + response: + | RenderResponse + | ModuleRenderResponse + | FileExtractResponse + | FileRenderResponse, ): readonly string[] | undefined { let renderError = response.error?.error; if (!renderError) { diff --git a/packages/realm-server/tests/prerender-proxy-test.ts b/packages/realm-server/tests/prerender-proxy-test.ts index 07a8f9b503d..5ef72e53f4f 100644 --- a/packages/realm-server/tests/prerender-proxy-test.ts +++ b/packages/realm-server/tests/prerender-proxy-test.ts @@ -33,7 +33,7 @@ module(basename(__filename), function () { function makePrerenderer() { let renderCalls: Array<{ - kind: 'card' | 'module' | 'file-extract'; + kind: 'card' | 'module' | 'file-extract' | 'file-render'; args: { realm: string; url: string; From c83b0d6d71947328a16c9292f9338f0195bd9db9 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 19:19:26 -0500 Subject: [PATCH 3/9] Remove redundant consumeClearCacheForRender call in file render phase The extract phase already consumes the clear-cache flag, so the render phase doesn't need to consume it again. Co-Authored-By: Claude Opus 4.5 --- packages/runtime-common/index-runner.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 179be5328e2..4462d0d4e59 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -972,11 +972,9 @@ export class IndexRunner { let renderResult: FileRenderResponse | undefined; if (extractResult.resource) { try { - let renderClearCache = this.#consumeClearCacheForRender(); let fileRenderOptions: RenderRouteOptions = { fileRender: true, fileDefCodeRef, - ...(renderClearCache ? { clearCache: true as const } : {}), }; renderResult = await this.#prerenderer.prerenderFileRender({ url: fileURL, From b4352116291cedea232adac414146143909230e1 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 20:32:39 -0500 Subject: [PATCH 4/9] Skip file HTML render for modules and card instances during indexing Files with executable extensions (modules) and card JSON files already have their own prerender paths. Running the FileDef HTML render phase for these files added ~168 extra Puppeteer page transitions during boot indexing, causing CI test timeouts. Now only non-code, non-card files (like .txt, .png, .md) go through the file render phase. Co-Authored-By: Claude Opus 4.5 --- packages/runtime-common/index-runner.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 4462d0d4e59..185a5674ff5 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -558,8 +558,13 @@ export class IndexRunner { let { content, lastModified } = fileRef; // ensure created_at exists for this file and use it for resourceCreatedAt let resourceCreatedAt = await this.batch.ensureFileCreatedAt(localPath); + // Track whether this file is already rendered through another path + // (modules get their own prerender, card instances get full card prerender). + // Files that are only indexed as file entries benefit from FileDef HTML rendering. + let skipHtmlRender = false; if (hasExecutableExtension(url.href)) { await this.indexModule(url); + skipHtmlRender = true; } else if (url.href.endsWith('.json')) { let resource; @@ -585,6 +590,7 @@ export class IndexRunner { resourceCreatedAt, resource, }); + skipHtmlRender = true; // Intentionally fall through so card JSON files also get a file entry. } } @@ -599,6 +605,7 @@ export class IndexRunner { path: localPath, lastModified, resourceCreatedAt, + skipHtmlRender, }); this.#log.debug( `${jobIdentity(this.#jobInfo)} completed visiting file ${url.href} in ${Date.now() - start}ms`, @@ -870,10 +877,12 @@ export class IndexRunner { path, lastModified, resourceCreatedAt, + skipHtmlRender, }: { path: LocalPath; lastModified: number; resourceCreatedAt: number; + skipHtmlRender?: boolean; }): Promise { let fileURL = this.#realmPaths.fileURL(path).href; let entryURL = new URL(fileURL); @@ -969,8 +978,9 @@ export class IndexRunner { let fileTypes = extractResult.types ?? fallbackTypes; // Phase 2: Render HTML for file entry (non-fatal) + // Skip for files already rendered through another path (modules, card instances). let renderResult: FileRenderResponse | undefined; - if (extractResult.resource) { + if (extractResult.resource && !skipHtmlRender) { try { let fileRenderOptions: RenderRouteOptions = { fileRender: true, From e950d9caad34a7f9deb6a4f43b9fc1d518d12c20 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 20:36:56 -0500 Subject: [PATCH 5/9] Revert "Skip file HTML render for modules and card instances during indexing" This reverts commit b4352116291cedea232adac414146143909230e1. --- packages/runtime-common/index-runner.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 185a5674ff5..4462d0d4e59 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -558,13 +558,8 @@ export class IndexRunner { let { content, lastModified } = fileRef; // ensure created_at exists for this file and use it for resourceCreatedAt let resourceCreatedAt = await this.batch.ensureFileCreatedAt(localPath); - // Track whether this file is already rendered through another path - // (modules get their own prerender, card instances get full card prerender). - // Files that are only indexed as file entries benefit from FileDef HTML rendering. - let skipHtmlRender = false; if (hasExecutableExtension(url.href)) { await this.indexModule(url); - skipHtmlRender = true; } else if (url.href.endsWith('.json')) { let resource; @@ -590,7 +585,6 @@ export class IndexRunner { resourceCreatedAt, resource, }); - skipHtmlRender = true; // Intentionally fall through so card JSON files also get a file entry. } } @@ -605,7 +599,6 @@ export class IndexRunner { path: localPath, lastModified, resourceCreatedAt, - skipHtmlRender, }); this.#log.debug( `${jobIdentity(this.#jobInfo)} completed visiting file ${url.href} in ${Date.now() - start}ms`, @@ -877,12 +870,10 @@ export class IndexRunner { path, lastModified, resourceCreatedAt, - skipHtmlRender, }: { path: LocalPath; lastModified: number; resourceCreatedAt: number; - skipHtmlRender?: boolean; }): Promise { let fileURL = this.#realmPaths.fileURL(path).href; let entryURL = new URL(fileURL); @@ -978,9 +969,8 @@ export class IndexRunner { let fileTypes = extractResult.types ?? fallbackTypes; // Phase 2: Render HTML for file entry (non-fatal) - // Skip for files already rendered through another path (modules, card instances). let renderResult: FileRenderResponse | undefined; - if (extractResult.resource && !skipHtmlRender) { + if (extractResult.resource) { try { let fileRenderOptions: RenderRouteOptions = { fileRender: true, From 221a30eb1994baa243a8e35e2028678331250a40 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 20:37:53 -0500 Subject: [PATCH 6/9] Increase realm-server test timeout to 180s for file render overhead The new file HTML render phase adds Puppeteer page transitions for every file during boot indexing. On CI hardware this pushes the total indexing time past the previous 60s limit. Co-Authored-By: Claude Opus 4.5 --- packages/realm-server/tests/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index a330cf9584b..74b8a15b9af 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -29,7 +29,7 @@ import * as ContentTagGlobal from 'content-tag'; import QUnit from 'qunit'; -QUnit.config.testTimeout = 60000; +QUnit.config.testTimeout = 180000; // Cleanup here ensures lingering servers/prerenderers/queues don't keep the // Node event loop alive after tests finish. From ff19e3da22e8c4a9c164fd2ecbae554dad29db81 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 28 Jan 2026 21:55:00 -0500 Subject: [PATCH 7/9] Render only isolated HTML for file entries to fix CI timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each Puppeteer page transition costs 2-3 seconds, and rendering all 6+ formats (isolated, head, atom, icon, fitted, embedded) for every file during boot indexing made total time O(files × formats × 3s). With ~28 files in test realms, this exceeded test timeouts. Reduce file render to isolated HTML only. Additional formats can be added back once the rendering pipeline is optimized for batch/parallel operation. The FileRenderResponse type and index columns remain unchanged — they simply receive null for the deferred formats. Co-Authored-By: Claude Opus 4.5 --- .../host/app/components/card-prerender.gts | 46 +----- .../realm-server/prerender/render-runner.ts | 131 ++---------------- 2 files changed, 18 insertions(+), 159 deletions(-) diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 3e55b5c29c1..5ff2ed514d2 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -429,7 +429,6 @@ export default class CardPrerender extends Component { async ({ url, fileData, - types, renderOptions, }: FileRenderArgs): Promise => { this.#nonce++; @@ -457,47 +456,16 @@ export default class CardPrerender extends Component { let error: RenderError | undefined; let isolatedHTML: string | null = null; - let headHTML: string | null = null; - let atomHTML: string | null = null; - let iconHTML: string | null = null; - let embeddedHTML: Record | null = null; - let fittedHTML: Record | null = null; + // Render isolated HTML only – additional formats (head, atom, icon, + // fitted, embedded) are deferred to keep boot-indexing fast. try { - let subsequentRenderOptions = omitOneTimeOptions(initialRenderOptions); isolatedHTML = await this.renderHTML.perform( url, 'isolated', 0, initialRenderOptions, ); - headHTML = await this.renderHTML.perform( - url, - 'head', - 0, - subsequentRenderOptions, - ); - atomHTML = await this.renderHTML.perform( - url, - 'atom', - 0, - subsequentRenderOptions, - ); - iconHTML = await this.renderIcon.perform(url, subsequentRenderOptions); - if (types.length > 0) { - embeddedHTML = await this.renderAncestors.perform( - url, - 'embedded', - types, - subsequentRenderOptions, - ); - fittedHTML = await this.renderAncestors.perform( - url, - 'fitted', - types, - subsequentRenderOptions, - ); - } } catch (e: any) { try { error = { ...JSON.parse(e.message), type: 'file-error' }; @@ -520,11 +488,11 @@ export default class CardPrerender extends Component { return { isolatedHTML, - headHTML, - atomHTML, - embeddedHTML, - fittedHTML, - iconHTML, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, ...(error ? { error } : {}), }; }, diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 222704c4d4b..0ac9f1b0af1 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -695,7 +695,7 @@ export class RenderRunner { url, auth, fileData, - types, + types: _types, opts, renderOptions, }: { @@ -749,7 +749,6 @@ export class RenderRunner { let renderStart = Date.now(); let error: RenderError | undefined; - let shortCircuit = false; let options: RenderRouteOptions = { ...(renderOptions ?? {}), fileRender: true, @@ -767,7 +766,11 @@ export class RenderRunner { `file render: visit ${url} at: ${this.#boxelHostURL}/render/${encodeURIComponent(url)}/${this.#nonce}/${optionsSegment}/html/isolated/0`, ); - // Render isolated HTML first + // Render isolated HTML only – additional formats (head, atom, icon, + // fitted, embedded) are deferred to keep boot-indexing fast. Each + // Puppeteer transition costs 2-3 s, so rendering all formats for every + // file would make boot-time O(files × formats × 3 s) which easily + // exceeds test timeouts. let result = await withTimeout( page, async () => { @@ -795,10 +798,6 @@ export class RenderRunner { ); if (evicted) { poolInfo.evicted = true; - shortCircuit = true; - } - if (this.#isAuthError(error)) { - shortCircuit = true; } } else { let capture = result as RenderCapture; @@ -817,126 +816,18 @@ export class RenderRunner { ); if (evicted) { poolInfo.evicted = true; - shortCircuit = true; - } - if (this.#isAuthError(error)) { - shortCircuit = true; - } - } - } - - if (shortCircuit) { - let response: FileRenderResponse = { - ...(error ? { error } : {}), - iconHTML: null, - isolatedHTML, - headHTML: null, - atomHTML: null, - embeddedHTML: null, - fittedHTML: null, - }; - response.error = this.#mergeConsoleErrors(pageId, response.error); - return { - response, - timings: { launchMs, renderMs: Date.now() - renderStart }, - pool: poolInfo, - }; - } - - // Render remaining formats (no meta step needed for files) - let headHTML: string | null = null, - atomHTML: string | null = null, - iconHTML: string | null = null, - embeddedHTML: Record | null = null, - fittedHTML: Record | null = null; - - if (!shortCircuit) { - let headHTMLResult = await this.#step(realm, 'file head render', () => - withTimeout( - page, - () => renderHTML(page, 'head', 0, captureOptions), - opts?.timeoutMs, - ), - ); - if (headHTMLResult.ok) { - headHTML = headHTMLResult.value as string; - } else { - error = error ?? headHTMLResult.error; - markTimeout(headHTMLResult.error); - if (headHTMLResult.evicted) { - poolInfo.evicted = true; - shortCircuit = true; - } - } - } - - if (!shortCircuit && types.length > 0) { - const steps: Array<{ - name: string; - cb: () => Promise | RenderError>; - assign: (value: string | Record) => void; - }> = [ - { - name: 'file fitted render', - cb: () => renderAncestors(page, 'fitted', types, captureOptions), - assign: (v: string | Record) => { - fittedHTML = v as Record; - }, - }, - { - name: 'file embedded render', - cb: () => renderAncestors(page, 'embedded', types, captureOptions), - assign: (v: string | Record) => { - embeddedHTML = v as Record; - }, - }, - { - name: 'file atom render', - cb: () => renderHTML(page, 'atom', 0, captureOptions), - assign: (v: string | Record) => { - atomHTML = v as string; - }, - }, - { - name: 'file icon render', - cb: () => renderIcon(page, captureOptions), - assign: (v: string | Record) => { - iconHTML = v as string; - }, - }, - ]; - - for (let step of steps) { - if (shortCircuit) break; - let res = await this.#step(realm, step.name, () => - withTimeout(page, step.cb, opts?.timeoutMs), - ); - if (res.ok) { - step.assign(res.value); - } else { - error = error ?? res.error; - markTimeout(res.error); - if (res.evicted) { - poolInfo.evicted = true; - shortCircuit = true; - break; - } - if (this.#isAuthError(error)) { - shortCircuit = true; - break; - } } } } let response: FileRenderResponse = { ...(error ? { error } : {}), - iconHTML, + iconHTML: null, isolatedHTML, - headHTML, - atomHTML, - embeddedHTML, - fittedHTML, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, }; response.error = this.#mergeConsoleErrors(pageId, response.error); return { From c76a69a557ac8e955fc306fd3cd897d5837373ea Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Thu, 29 Jan 2026 00:10:11 -0500 Subject: [PATCH 8/9] Fix CI timeouts: add manager proxy route, fix expectedId, skip module file renders Three root causes of realm-server test timeouts: 1. Missing /prerender-file-render proxy route in manager-app.ts caused boot servers to 404 when calling prerenderFileRender through the manager. 2. expectedId mismatch in render-runner.ts: captureResult expected URL without .json extension but the render route sets cardId to the raw URL including .json, causing each .json file to timeout at 30s waiting for a DOM match. 3. Module files (.gts/.ts/.js) got redundant file renders via indexFile() since visitFile() always falls through to indexFile() after indexModule(). Added hasModulePrerender flag to skip file render for modules that already produce HTML through their module prerender path. Co-Authored-By: Claude Opus 4.5 --- packages/realm-server/prerender/manager-app.ts | 3 +++ packages/realm-server/prerender/render-runner.ts | 5 ++++- packages/runtime-common/index-runner.ts | 13 ++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/realm-server/prerender/manager-app.ts b/packages/realm-server/prerender/manager-app.ts index 0c66b05ecc5..f7fd75a3cd2 100644 --- a/packages/realm-server/prerender/manager-app.ts +++ b/packages/realm-server/prerender/manager-app.ts @@ -910,6 +910,9 @@ export function buildPrerenderManagerApp(options?: { router.post('/prerender-file-extract', (ctxt) => proxyPrerenderRequest(ctxt, 'prerender-file-extract', 'file-extract'), ); + router.post('/prerender-file-render', (ctxt) => + proxyPrerenderRequest(ctxt, 'prerender-file-render', 'file-render'), + ); let verboseManagerLogs = process.env.PRERENDER_MANAGER_VERBOSE_LOGS === 'true'; diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 0ac9f1b0af1..2baadd5b1ef 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -756,8 +756,11 @@ export class RenderRunner { }; let serializedOptions = serializeRenderRouteOptions(options); let optionsSegment = encodeURIComponent(serializedOptions); + // File render uses the full file URL (including extension) as the ID, + // unlike card render which strips .json. The render route's fileRender + // branch sets cardId to the raw url parameter, so expectedId must match. const captureOptions: CaptureOptions = { - expectedId: url.replace(/\.json$/i, ''), + expectedId: url, expectedNonce: String(this.#nonce), simulateTimeoutMs: opts?.simulateTimeoutMs, }; diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 4462d0d4e59..8a28eae754f 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -558,7 +558,8 @@ export class IndexRunner { let { content, lastModified } = fileRef; // ensure created_at exists for this file and use it for resourceCreatedAt let resourceCreatedAt = await this.batch.ensureFileCreatedAt(localPath); - if (hasExecutableExtension(url.href)) { + let isModule = hasExecutableExtension(url.href); + if (isModule) { await this.indexModule(url); } else if (url.href.endsWith('.json')) { let resource; @@ -599,6 +600,7 @@ export class IndexRunner { path: localPath, lastModified, resourceCreatedAt, + hasModulePrerender: isModule, }); this.#log.debug( `${jobIdentity(this.#jobInfo)} completed visiting file ${url.href} in ${Date.now() - start}ms`, @@ -870,10 +872,12 @@ export class IndexRunner { path, lastModified, resourceCreatedAt, + hasModulePrerender, }: { path: LocalPath; lastModified: number; resourceCreatedAt: number; + hasModulePrerender?: boolean; }): Promise { let fileURL = this.#realmPaths.fileURL(path).href; let entryURL = new URL(fileURL); @@ -968,9 +972,12 @@ export class IndexRunner { ); let fileTypes = extractResult.types ?? fallbackTypes; - // Phase 2: Render HTML for file entry (non-fatal) + // Phase 2: Render HTML for file entry (non-fatal). + // Skip for files that already have their own prerender (modules) since + // they add significant per-file Puppeteer overhead and already produce HTML + // through their module prerender path. let renderResult: FileRenderResponse | undefined; - if (extractResult.resource) { + if (extractResult.resource && !hasModulePrerender) { try { let fileRenderOptions: RenderRouteOptions = { fileRender: true, From 1c6ddf539ecd56b833481862c2a13a85f7e31da5 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Thu, 29 Jan 2026 06:04:28 -0500 Subject: [PATCH 9/9] Restore realm-server test timeout to 60s The three root causes of the timeout have been fixed (missing manager proxy route, expectedId mismatch, redundant module file renders), so the inflated 180s timeout is no longer needed. Co-Authored-By: Claude Opus 4.5 --- packages/realm-server/tests/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index 74b8a15b9af..a330cf9584b 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -29,7 +29,7 @@ import * as ContentTagGlobal from 'content-tag'; import QUnit from 'qunit'; -QUnit.config.testTimeout = 180000; +QUnit.config.testTimeout = 60000; // Cleanup here ensures lingering servers/prerenderers/queues don't keep the // Node event loop alive after tests finish.