diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index e3a708360d..5ff2ed514d 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -22,6 +22,8 @@ import { type RenderError, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type Prerenderer, type Format, type PrerenderMeta, @@ -93,6 +95,7 @@ export default class CardPrerender extends Component { prerenderCard: this.prerender.bind(this), prerenderModule: this.prerenderModule.bind(this), prerenderFileExtract: this.prerenderFileExtract.bind(this), + prerenderFileRender: this.prerenderFileRenderPublic.bind(this), }; this.localIndexer.setup(this.#prerendererDelegate); window.addEventListener('boxel-render-error', this.#handleRenderErrorEvent); @@ -203,6 +206,24 @@ export default class CardPrerender extends Component { }); } + private async prerenderFileRenderPublic( + args: FileRenderArgs, + ): Promise { + return await withRenderContext(async () => { + try { + let run = () => this.fileRenderPrerenderTask.perform(args); + return isTesting() ? await run() : await withTimersBlocked(run); + } catch (e: any) { + if (!didCancel(e)) { + throw e; + } + } + throw new Error( + `card-prerender component is missing or being destroyed before file render prerender of url ${args.url} was completed`, + ); + }); + } + // This emulates the job of the Prerenderer that runs in the server private prerenderTask = enqueueTask( async ({ @@ -404,6 +425,79 @@ export default class CardPrerender extends Component { }, ); + private fileRenderPrerenderTask = enqueueTask( + async ({ + url, + fileData, + renderOptions, + }: FileRenderArgs): Promise => { + this.#nonce++; + let shouldClearCache = this.#consumeClearCacheForRender( + Boolean(renderOptions?.clearCache), + ); + let initialRenderOptions: RenderRouteOptions = { + ...(renderOptions ?? {}), + fileRender: true, + fileDefCodeRef: fileData.fileDefCodeRef, + }; + if (shouldClearCache) { + initialRenderOptions.clearCache = true; + this.loaderService.resetLoader({ + clearFetchCache: true, + reason: 'card-prerender file render clearCache', + }); + this.store.resetCache(); + } else { + delete initialRenderOptions.clearCache; + } + + // Stash file data for the render route to consume + (globalThis as any).__boxelFileRenderData = fileData; + + let error: RenderError | undefined; + let isolatedHTML: string | null = null; + + // Render isolated HTML only – additional formats (head, atom, icon, + // fitted, embedded) are deferred to keep boot-indexing fast. + try { + isolatedHTML = await this.renderHTML.perform( + url, + 'isolated', + 0, + initialRenderOptions, + ); + } catch (e: any) { + try { + error = { ...JSON.parse(e.message), type: 'file-error' }; + } catch (_err) { + let cardErr = new CardError(e.message); + cardErr.stack = e.stack; + error = { + error: { + ...cardErr.toJSON(), + deps: [url], + additionalErrors: null, + }, + type: 'file-error', + }; + } + this.store.resetCache(); + } finally { + delete (globalThis as any).__boxelFileRenderData; + } + + return { + isolatedHTML, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + ...(error ? { error } : {}), + }; + }, + ); + #moduleModelContext(): ModuleModelContext { return { router: this.router, diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index e87b83adbe..05bdfd9d88 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -249,6 +249,50 @@ export default class RenderRoute extends Route { this.currentTransition = undefined; return model; } + if (parsedOptions.fileRender) { + let fileRenderData = (globalThis as any).__boxelFileRenderData as + | { resource: any; fileDefCodeRef: { module: string; name: string } } + | undefined; + if (!fileRenderData) { + throw new Error('fileRender mode requires __boxelFileRenderData'); + } + let { resource } = fileRenderData; + let cardApi = await this.loaderService.loader.import( + `${baseRealm.url}card-api`, + ); + let doc = { data: resource }; + let instance = (await cardApi.createFromSerialized( + resource, + doc, + resource.id ? new URL(resource.id) : undefined, + )) as unknown as CardDef; + + let state = new TrackedMap(); + state.set('status', 'ready'); + let readyDeferred = new Deferred(); + readyDeferred.fulfill(); + let model: Model = { + instance, + nonce, + cardId: id, + renderOptions: parsedOptions, + get status(): RenderStatus { + return (state.get('status') as RenderStatus) ?? 'loading'; + }, + get ready(): boolean { + return (state.get('status') as RenderStatus) === 'ready'; + }, + readyPromise: readyDeferred.promise, + }; + this.#modelStates.set(model, { + state, + readyDeferred, + isReady: true, + }); + (globalThis as any).__renderModel = model; + this.currentTransition = undefined; + return model; + } // This is for host tests (globalThis as any).__renderModel = undefined; diff --git a/packages/realm-server/prerender/manager-app.ts b/packages/realm-server/prerender/manager-app.ts index 0c66b05ecc..f7fd75a3cd 100644 --- a/packages/realm-server/prerender/manager-app.ts +++ b/packages/realm-server/prerender/manager-app.ts @@ -910,6 +910,9 @@ export function buildPrerenderManagerApp(options?: { router.post('/prerender-file-extract', (ctxt) => proxyPrerenderRequest(ctxt, 'prerender-file-extract', 'file-extract'), ); + router.post('/prerender-file-render', (ctxt) => + proxyPrerenderRequest(ctxt, 'prerender-file-render', 'file-render'), + ); let verboseManagerLogs = process.env.PRERENDER_MANAGER_VERBOSE_LOGS === 'true'; diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 74dccfb886..91f00b742a 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -10,6 +10,7 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, } from '@cardstack/runtime-common'; import { ecsMetadata, @@ -336,6 +337,175 @@ export function buildPrerenderApp(options: { }, }); + // File render route needs additional attributes (fileData, types) + // beyond what registerPrerenderRoute handles, so we register it directly. + router.post('/prerender-file-render', async (ctxt: Koa.Context) => { + try { + let request = await fetchRequestFromContext(ctxt); + let raw = await request.text(); + let body: any; + try { + body = raw ? JSON.parse(raw) : {}; + } catch (e) { + ctxt.status = 400; + ctxt.body = { + errors: [{ status: 400, message: 'Invalid JSON body' }], + }; + return; + } + + let attrs = body?.data?.attributes ?? {}; + let rawUrl = attrs.url; + let rawAuth = attrs.auth; + let rawRealm = attrs.realm; + let renderOptions: RenderRouteOptions = + attrs.renderOptions && + typeof attrs.renderOptions === 'object' && + !Array.isArray(attrs.renderOptions) + ? (attrs.renderOptions as RenderRouteOptions) + : {}; + let fileData = attrs.fileData; + let types = attrs.types; + + let isNonEmptyString = (value: unknown): value is string => + typeof value === 'string' && value.trim().length > 0; + + let missing = [ + { value: rawUrl, name: 'url' }, + { value: rawRealm, name: 'realm' }, + { value: rawAuth, name: 'auth' }, + ] + .filter(({ value }) => !isNonEmptyString(value)) + .map(({ name }) => name); + + if (!fileData) { + missing.push('fileData'); + } + if (!Array.isArray(types)) { + missing.push('types'); + } + + log.debug( + `received file render prerender request ${rawUrl}: realm=${rawRealm}`, + ); + if (missing.length > 0) { + ctxt.status = 400; + ctxt.body = { + errors: [ + { + status: 400, + message: `Missing or invalid required attributes: ${missing.join(', ')}`, + }, + ], + }; + return; + } + + let realm = rawRealm as string; + let url = rawUrl as string; + let auth = rawAuth as string; + + let start = Date.now(); + let execPromise = prerenderer + .prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + renderOptions, + }) + .then((result) => ({ result })); + let drainPromise = options.drainingPromise + ? options.drainingPromise.then(() => ({ draining: true as const })) + : null; + let raceResult = drainPromise + ? await Promise.race([execPromise, drainPromise]) + : await execPromise; + if ('draining' in raceResult) { + execPromise.catch((e) => + log.debug( + 'file render prerender execute settled after drain (ignored):', + e, + ), + ); + ctxt.status = PRERENDER_SERVER_DRAINING_STATUS_CODE; + ctxt.set( + PRERENDER_SERVER_STATUS_HEADER, + PRERENDER_SERVER_STATUS_DRAINING, + ); + ctxt.body = { + errors: [ + { + status: PRERENDER_SERVER_DRAINING_STATUS_CODE, + message: 'Prerender server draining', + }, + ], + }; + return; + } + let { response, timings, pool } = raceResult.result; + let totalMs = Date.now() - start; + let poolFlags = Object.entries({ + reused: pool.reused, + evicted: pool.evicted, + timedOut: pool.timedOut, + }) + .filter(([, value]) => value === true) + .map(([key]) => key) + .join(', '); + let poolFlagSuffix = poolFlags.length > 0 ? ` flags=[${poolFlags}]` : ''; + log.info( + 'file render prerendered %s total=%dms launch=%dms render=%dms pageId=%s realm=%s%s', + url, + totalMs, + timings.launchMs, + timings.renderMs, + pool.pageId, + pool.realm, + poolFlagSuffix, + ); + ctxt.status = 201; + ctxt.set('Content-Type', 'application/vnd.api+json'); + ctxt.body = { + data: { + type: 'prerender-file-render-result', + id: url, + attributes: response, + }, + meta: { + timing: { + launchMs: timings.launchMs, + renderMs: timings.renderMs, + totalMs, + }, + pool, + }, + }; + if (pool.timedOut) { + log.warn(`file render of ${url} timed out`); + } + const fileResponse = response as FileRenderResponse; + if (fileResponse.error) { + log.debug( + `file render of ${url} resulted in error doc:\n${JSON.stringify(fileResponse.error, null, 2)}`, + ); + } + } catch (err: any) { + Sentry.captureException(err); + log.error('Unhandled error in /prerender-file-render:', err); + ctxt.status = 500; + ctxt.body = { + errors: [ + { + status: 500, + message: err?.message ?? 'Unknown error', + }, + ], + }; + } + }); + app .use((ctxt: Koa.Context, next: Koa.Next) => { if ( diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 5af7c6b38d..22e2a96a9f 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -3,6 +3,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, logger, } from '@cardstack/runtime-common'; import { BrowserManager } from './browser-manager'; @@ -460,6 +462,137 @@ export class Prerenderer { throw new Error(`file extract prerender attempts exhausted for ${url}`); } + async prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions, + }: { + realm: string; + url: string; + auth: string; + fileData: FileRenderArgs['fileData']; + types: string[]; + opts?: { timeoutMs?: number; simulateTimeoutMs?: number }; + renderOptions?: RenderRouteOptions; + }): Promise<{ + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + if (this.#stopped) { + throw new Error('Prerenderer has been stopped and cannot be used'); + } + let attemptOptions = renderOptions; + let lastResult: + | { + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + } + | undefined; + for (let attempt = 0; attempt < 3; attempt++) { + let result: { + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }; + try { + result = await this.#renderRunner.prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions: attemptOptions, + }); + } catch (e) { + log.error( + `file render prerender attempt for ${url} (realm ${realm}) failed with error, restarting browser`, + e, + ); + await this.#restartBrowser(); + try { + result = await this.#renderRunner.prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types, + opts, + renderOptions: attemptOptions, + }); + } catch (e2) { + log.error( + `file render prerender attempt for ${url} (realm ${realm}) failed again after browser restart`, + e2, + ); + throw e2; + } + } + lastResult = result; + + let retrySignature = this.#renderRunner.shouldRetryWithClearCache( + result.response, + ); + let isClearCacheAttempt = attemptOptions?.clearCache === true; + + if (!isClearCacheAttempt && retrySignature) { + log.warn( + `retrying file render prerender for ${url} with clearCache due to error signature: ${retrySignature.join( + ' | ', + )}`, + ); + attemptOptions = { + ...(attemptOptions ?? {}), + clearCache: true, + }; + continue; + } + + if (isClearCacheAttempt && retrySignature && result.response.error) { + log.warn( + `file render prerender retry with clearCache did not resolve error signature ${retrySignature.join( + ' | ', + )} for ${url}`, + ); + } + + return result; + } + if (lastResult) { + if (lastResult.response.error) { + log.error( + `file render prerender attempts exhausted for ${url} in realm ${realm}, returning last error response`, + ); + } + return lastResult; + } + throw new Error(`file render prerender attempts exhausted for ${url}`); + } + async #restartBrowser(): Promise { log.warn('Restarting prerender browser'); await this.#pagePool.closeAll(); diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index a93829fda2..5eb111081d 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -3,6 +3,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type RenderRouteOptions, logger, } from '@cardstack/runtime-common'; @@ -60,6 +62,7 @@ export function createRemotePrerenderer( url: string; auth: string; renderOptions?: RenderRouteOptions; + [key: string]: any; }, ): Promise { validatePrerenderAttributes(type, attributes); @@ -188,6 +191,27 @@ export function createRemotePrerenderer( }, ); }, + async prerenderFileRender({ + realm, + url, + auth, + fileData, + types, + renderOptions, + }: FileRenderArgs) { + return await requestWithRetry( + 'prerender-file-render', + 'prerender-file-render-request', + { + realm, + url, + auth, + fileData, + types, + renderOptions: renderOptions ?? {}, + }, + ); + }, }; } diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 849940ff6b..2baadd5b1e 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -4,6 +4,8 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, + type FileRenderArgs, type RenderRouteOptions, serializeRenderRouteOptions, logger, @@ -688,8 +690,173 @@ export class RenderRunner { } } + async prerenderFileRenderAttempt({ + realm, + url, + auth, + fileData, + types: _types, + opts, + renderOptions, + }: { + realm: string; + url: string; + auth: string; + fileData: FileRenderArgs['fileData']; + types: string[]; + opts?: { timeoutMs?: number; simulateTimeoutMs?: number }; + renderOptions?: RenderRouteOptions; + }): Promise<{ + response: FileRenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + this.#nonce++; + log.info( + `file render prerendering url ${url}, nonce=${this.#nonce} realm=${realm}`, + ); + + const { page, reused, launchMs, pageId, release } = + await this.#getPageForRealm(realm, auth); + const poolInfo = { + pageId: pageId ?? 'unknown', + realm, + reused, + evicted: false, + timedOut: false, + }; + this.#pagePool.resetConsoleErrors(pageId); + const markTimeout = (err?: RenderError) => { + if (!poolInfo.timedOut && err?.error?.title === 'Render timeout') { + poolInfo.timedOut = true; + } + }; + try { + await page.evaluate((sessionAuth) => { + localStorage.setItem('boxel-session', sessionAuth); + }, auth); + + // Stash file data on globalThis for the render route to consume + await page.evaluate((data) => { + (globalThis as any).__boxelFileRenderData = data; + }, fileData); + + let renderStart = Date.now(); + let error: RenderError | undefined; + let options: RenderRouteOptions = { + ...(renderOptions ?? {}), + fileRender: true, + fileDefCodeRef: fileData.fileDefCodeRef, + }; + let serializedOptions = serializeRenderRouteOptions(options); + let optionsSegment = encodeURIComponent(serializedOptions); + // File render uses the full file URL (including extension) as the ID, + // unlike card render which strips .json. The render route's fileRender + // branch sets cardId to the raw url parameter, so expectedId must match. + const captureOptions: CaptureOptions = { + expectedId: url, + expectedNonce: String(this.#nonce), + simulateTimeoutMs: opts?.simulateTimeoutMs, + }; + + log.debug( + `file render: visit ${url} at: ${this.#boxelHostURL}/render/${encodeURIComponent(url)}/${this.#nonce}/${optionsSegment}/html/isolated/0`, + ); + + // Render isolated HTML only – additional formats (head, atom, icon, + // fitted, embedded) are deferred to keep boot-indexing fast. Each + // Puppeteer transition costs 2-3 s, so rendering all formats for every + // file would make boot-time O(files × formats × 3 s) which easily + // exceeds test timeouts. + let result = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.html', + url, + String(this.#nonce), + serializedOptions, + 'isolated', + '0', + ); + return await captureResult(page, 'innerHTML', captureOptions); + }, + opts?.timeoutMs, + ); + let isolatedHTML: string | null = null; + if (isRenderError(result)) { + error = result; + markTimeout(error); + let evicted = await this.#maybeEvict( + realm, + 'file isolated render', + result as RenderError, + ); + if (evicted) { + poolInfo.evicted = true; + } + } else { + let capture = result as RenderCapture; + if (capture.status === 'ready') { + isolatedHTML = capture.value; + } else { + let capErr = this.#captureToError(capture); + if (!error && capErr) { + error = capErr; + } + markTimeout(capErr); + let evicted = await this.#maybeEvict( + realm, + 'file isolated render', + capErr, + ); + if (evicted) { + poolInfo.evicted = true; + } + } + } + + let response: FileRenderResponse = { + ...(error ? { error } : {}), + iconHTML: null, + isolatedHTML, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + }; + response.error = this.#mergeConsoleErrors(pageId, response.error); + return { + response, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } finally { + // Clean up globalThis data + await page + .evaluate(() => { + delete (globalThis as any).__boxelFileRenderData; + }) + .catch(() => { + /* best-effort cleanup */ + }); + release(); + } + } + shouldRetryWithClearCache( - response: RenderResponse | ModuleRenderResponse | FileExtractResponse, + response: + | RenderResponse + | ModuleRenderResponse + | FileExtractResponse + | FileRenderResponse, ): readonly string[] | undefined { let renderError = response.error?.error; if (!renderError) { diff --git a/packages/realm-server/tests/definition-lookup-test.ts b/packages/realm-server/tests/definition-lookup-test.ts index dada6a81cd..b50d41257e 100644 --- a/packages/realm-server/tests/definition-lookup-test.ts +++ b/packages/realm-server/tests/definition-lookup-test.ts @@ -147,6 +147,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, }; definitionLookup = new CachingDefinitionLookup( dbAdapter, @@ -323,6 +326,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls++; let moduleAlias = trimExecutableExtension(new URL(args.url)).href; @@ -410,6 +416,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls++; if (!modulePresent) { @@ -484,6 +493,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls.set(args.url, (calls.get(args.url) ?? 0) + 1); switch (args.url) { @@ -627,6 +639,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { calls.set(args.url, (calls.get(args.url) ?? 0) + 1); switch (args.url) { @@ -779,6 +794,9 @@ module(basename(__filename), function () { async prerenderFileExtract() { throw new Error('Not implemented in mock'); }, + async prerenderFileRender() { + throw new Error('Not implemented in mock'); + }, async prerenderModule(args: ModulePrerenderArgs) { switch (args.url) { case deepModule: { diff --git a/packages/realm-server/tests/prerender-proxy-test.ts b/packages/realm-server/tests/prerender-proxy-test.ts index e01a675074..5ef72e53f4 100644 --- a/packages/realm-server/tests/prerender-proxy-test.ts +++ b/packages/realm-server/tests/prerender-proxy-test.ts @@ -33,7 +33,7 @@ module(basename(__filename), function () { function makePrerenderer() { let renderCalls: Array<{ - kind: 'card' | 'module' | 'file-extract'; + kind: 'card' | 'module' | 'file-extract' | 'file-render'; args: { realm: string; url: string; @@ -82,6 +82,17 @@ module(basename(__filename), function () { deps: [], }; }, + async prerenderFileRender(args) { + renderCalls.push({ kind: 'file-render', args }); + return { + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + }; + }, }; return { prerenderer, renderCalls }; diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 113bcb2a91..8a28eae754 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -20,6 +20,7 @@ import { type RenderResponse, type ModuleRenderResponse, type FileExtractResponse, + type FileRenderResponse, type ResolvedCodeRef, type Batch, type LooseCardResource, @@ -557,7 +558,8 @@ export class IndexRunner { let { content, lastModified } = fileRef; // ensure created_at exists for this file and use it for resourceCreatedAt let resourceCreatedAt = await this.batch.ensureFileCreatedAt(localPath); - if (hasExecutableExtension(url.href)) { + let isModule = hasExecutableExtension(url.href); + if (isModule) { await this.indexModule(url); } else if (url.href.endsWith('.json')) { let resource; @@ -598,6 +600,7 @@ export class IndexRunner { path: localPath, lastModified, resourceCreatedAt, + hasModulePrerender: isModule, }); this.#log.debug( `${jobIdentity(this.#jobInfo)} completed visiting file ${url.href} in ${Date.now() - start}ms`, @@ -869,10 +872,12 @@ export class IndexRunner { path, lastModified, resourceCreatedAt, + hasModulePrerender, }: { path: LocalPath; lastModified: number; resourceCreatedAt: number; + hasModulePrerender?: boolean; }): Promise { let fileURL = this.#realmPaths.fileURL(path).href; let entryURL = new URL(fileURL); @@ -967,6 +972,41 @@ export class IndexRunner { ); let fileTypes = extractResult.types ?? fallbackTypes; + // Phase 2: Render HTML for file entry (non-fatal). + // Skip for files that already have their own prerender (modules) since + // they add significant per-file Puppeteer overhead and already produce HTML + // through their module prerender path. + let renderResult: FileRenderResponse | undefined; + if (extractResult.resource && !hasModulePrerender) { + try { + let fileRenderOptions: RenderRouteOptions = { + fileRender: true, + fileDefCodeRef, + }; + renderResult = await this.#prerenderer.prerenderFileRender({ + url: fileURL, + realm: this.#realmURL.href, + auth: this.#auth, + fileData: { + resource: extractResult.resource, + fileDefCodeRef, + }, + types: fileTypes, + renderOptions: fileRenderOptions, + }); + if (renderResult?.error) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)} file render produced error for ${path}, continuing without HTML: ${renderResult.error.error?.message}`, + ); + renderResult = undefined; + } + } catch (err: any) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)} file render failed for ${path}, continuing without HTML: ${err.message}`, + ); + } + } + await this.batch.updateEntry(entryURL, { type: 'file', lastModified, @@ -982,6 +1022,12 @@ export class IndexRunner { }, types: fileTypes, displayNames: [], + isolatedHtml: renderResult?.isolatedHTML ?? undefined, + headHtml: renderResult?.headHTML ?? undefined, + atomHtml: renderResult?.atomHTML ?? undefined, + embeddedHtml: renderResult?.embeddedHTML ?? undefined, + fittedHtml: renderResult?.fittedHTML ?? undefined, + iconHTML: renderResult?.iconHTML ?? undefined, }); this.stats.filesIndexed++; } diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 72f677e9d1..79f6b5a37c 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -111,6 +111,12 @@ export interface FileEntry { resource?: FileMetaResource | null; types?: string[]; displayNames?: string[]; + isolatedHtml?: string; + headHtml?: string; + embeddedHtml?: Record; + fittedHtml?: Record; + atomHtml?: string; + iconHTML?: string; } export class Batch { @@ -341,6 +347,12 @@ export class Batch { search_doc: entry.searchData ?? null, types: entry.types ?? null, display_names: entry.displayNames ?? null, + isolated_html: entry.isolatedHtml ?? null, + head_html: entry.headHtml ?? null, + embedded_html: entry.embeddedHtml ?? null, + fitted_html: entry.fittedHtml ?? null, + atom_html: entry.atomHtml ?? null, + icon_html: entry.iconHTML ?? null, last_modified: entry.lastModified, resource_created_at: entry.resourceCreatedAt, error_doc: null, diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 2be0c95e24..29b6b123ff 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -67,6 +67,24 @@ export interface FileExtractResponse { mismatch?: true; } +export interface FileRenderResponse { + isolatedHTML: string | null; + headHTML: string | null; + atomHTML: string | null; + embeddedHTML: Record | null; + fittedHTML: Record | null; + iconHTML: string | null; + error?: RenderError; +} + +export type FileRenderArgs = ModulePrerenderArgs & { + fileData: { + resource: FileMetaResource; + fileDefCodeRef: ResolvedCodeRef; + }; + types: string[]; +}; + export interface ModuleDefinitionResult { type: 'definition'; moduleURL: string; // node resolution w/o extension @@ -101,6 +119,7 @@ export interface Prerenderer { prerenderCard(args: PrerenderCardArgs): Promise; prerenderModule(args: ModulePrerenderArgs): Promise; prerenderFileExtract(args: ModulePrerenderArgs): Promise; + prerenderFileRender(args: FileRenderArgs): Promise; } export type RealmAction = 'read' | 'write' | 'realm-owner' | 'assume-user'; diff --git a/packages/runtime-common/render-route-options.ts b/packages/runtime-common/render-route-options.ts index 00cf2f9954..2121949001 100644 --- a/packages/runtime-common/render-route-options.ts +++ b/packages/runtime-common/render-route-options.ts @@ -5,6 +5,7 @@ import type { ResolvedCodeRef } from './code-ref'; export interface RenderRouteOptions { clearCache?: true; fileExtract?: true; + fileRender?: true; fileDefCodeRef?: ResolvedCodeRef; fileContentHash?: string; } @@ -30,6 +31,12 @@ export function parseRenderRouteOptions( options.fileContentHash = parsed.fileContentHash; } } + if (parsed.fileRender) { + options.fileRender = true; + if (isResolvedCodeRef(parsed.fileDefCodeRef)) { + options.fileDefCodeRef = parsed.fileDefCodeRef; + } + } return options; } catch { return {}; @@ -52,5 +59,11 @@ export function serializeRenderRouteOptions( serialized.fileContentHash = options.fileContentHash; } } + if (options.fileRender) { + serialized.fileRender = true; + if (options.fileDefCodeRef) { + serialized.fileDefCodeRef = options.fileDefCodeRef; + } + } return stringify(serialized) ?? '{}'; }