Skip to content
Source
Sign inSign up
TanStack/router
Open History to see the last commit

prerender.ts

374 lines10.1 KB

1import { promises as fsp } from 'node:fs'
2import os from 'node:os'
3import path from 'pathe'
4import { joinURL, withBase, withTrailingSlash, withoutBase } from 'ufo'
5import { createLogger } from './utils'
6import { Queue } from './queue'
7import type { Page, TanStackStartOutputConfig } from './schema'
8​
9const DEFAULT_RETRY_DELAY = 500
10​
11export interface PrerenderHandler {
12 getClientOutputDirectory: () => string
13 getOrigin?: () => string
14 request: (path: string, options?: RequestInit) => Promise<Response>
15 close?: () => Promise<void>
17​
18export async function prerender({
19 startConfig,
20 handler,
21}: {
22 startConfig: TanStackStartOutputConfig
23 handler: PrerenderHandler
24}) {
25 const logger = createLogger('prerender')
26 logger.info('Prerendering pages...')
27​
28 if (startConfig.prerender?.enabled) {
29 let pages = startConfig.pages.length ? startConfig.pages : [{ path: '/' }]
30​
31 if (startConfig.prerender.autoStaticPathsDiscovery ?? true) {
32 const pagesMap = new Map(pages.map((item) => [item.path, item]))
33 const discoveredPages = globalThis.TSS_PRERENDABLE_PATHS || []
34​
35 for (const page of discoveredPages) {
36 if (!pagesMap.has(page.path)) {
37 pagesMap.set(page.path, page)
38 }
39 }
40​
41 pages = Array.from(pagesMap.values())
42 }
43​
44 startConfig.pages = pages
45 }
46​
47 const routerBasePath = joinURL('/', startConfig.router.basepath ?? '')
48 const routerBaseUrl = new URL(
49 routerBasePath,
50 handler.getOrigin?.() ?? 'http://localhost',
51 )
52​
53 startConfig.pages = validateAndNormalizePrerenderPages(
54 startConfig.pages,
55 routerBaseUrl,
56 )
57​
58 try {
59 const pages = await prerenderPages({
60 outputDir: handler.getClientOutputDirectory(),
61 })
62​
63 logger.info(`Prerendered ${pages.length} pages:`)
64 pages.forEach((page) => {
65 logger.info(`- ${page}`)
66 })
67 } catch (error) {
68 logger.error(error)
69 throw error
70 } finally {
71 await handler.close?.()
72 }
73​
74 function extractLinks(html: string): Array<string> {
75 const linkRegex = /<a[^>]+href=["']([^"']+)["'][^>]*>/g
76 const links: Array<string> = []
77 let match: RegExpExecArray | null
78​
79 while ((match = linkRegex.exec(html)) !== null) {
80 const href = match[1]
81 if (href && (href.startsWith('/') || href.startsWith('./'))) {
82 links.push(href)
83 }
84 }
85​
86 return links
87 }
88​
89 async function prerenderPages({ outputDir }: { outputDir: string }) {
90 const seen = new Set<string>()
91 const prerendered = new Set<string>()
92 const retriesByPath = new Map<string, number>()
93 const concurrency = startConfig.prerender?.concurrency ?? os.cpus().length
94 logger.info(`Concurrency: ${concurrency}`)
95 const queue = new Queue({ concurrency })
96​
97 startConfig.pages = validateAndNormalizePrerenderPages(
98 startConfig.pages,
99 routerBaseUrl,
100 )
101​
102 startConfig.pages.forEach((page) => addCrawlPageTask(page))
103​
104 if (queue.isSettled()) {
105 logger.info('No pages matched prerender filter; skipping.')
106 return Array.from(prerendered)
107 }
108​
109 await queue.start()
110​
111 return Array.from(prerendered)
112​
113 function addCrawlPageTask(page: Page) {
114 if (seen.has(page.path)) return
115​
116 seen.add(page.path)
117​
118 if (page.fromCrawl) {
119 startConfig.pages.push(page)
120 }
121​
122 if (!(page.prerender?.enabled ?? true)) return
123​
124 if (
125 startConfig.prerender?.filter &&
126 !startConfig.prerender.filter(page)
127 ) {
128 return
129 }
130​
131 const prerenderOptions = {
132 ...startConfig.prerender,
133 ...page.prerender,
134 }
135​
136 queue.add(async () => {
137 logger.info(`Crawling: ${page.path}`)
138 const retries = retriesByPath.get(page.path) || 0
139​
140 try {
141 const res = await requestWithRedirects(
142 withTrailingSlash(withBase(page.path, routerBasePath)),
143 {
144 headers: {
145 ...(prerenderOptions.headers ?? {}),
146 },
147 },
148 prerenderOptions.maxRedirects,
149 )
150​
151 if (!res.ok) {
152 if (isRedirectResponse(res)) {
153 logger.warn(`Max redirects reached for ${page.path}`)
154 }
155​
156 throw new Error(`Failed to fetch ${page.path}: ${res.statusText}`, {
157 cause: res,
158 })
159 }
160​
161 const cleanPagePath = (
162 prerenderOptions.outputPath || page.path
163 ).split(/[?#]/)[0]!
164​
165 const contentType = res.headers.get('content-type') || ''
166 const isImplicitHTML =
167 !cleanPagePath.endsWith('.html') && contentType.includes('html')
168​
169 const routeWithIndex = cleanPagePath.endsWith('/')
170 ? cleanPagePath + 'index'
171 : cleanPagePath
172​
173 const isSpaShell =
174 startConfig.spa?.prerender.outputPath === cleanPagePath
175​
176 let htmlPath: string
177 if (isSpaShell) {
178 htmlPath = cleanPagePath + '.html'
179 } else if (
180 cleanPagePath.endsWith('/') ||
181 (prerenderOptions.autoSubfolderIndex ?? true)
182 ) {
183 htmlPath = joinURL(cleanPagePath, 'index.html')
184 } else {
185 htmlPath = cleanPagePath + '.html'
186 }
187​
188 const filename = withoutBase(
189 isImplicitHTML ? htmlPath : routeWithIndex,
190 routerBasePath,
191 )
192​
193 const html = await res.text()
194 const resolvedOutputDir = path.resolve(outputDir)
195 const filepath = path.resolve(outputDir, filename.replace(/^\/+/, ''))
196 const outputPrefix = resolvedOutputDir.endsWith(path.sep)
197 ? resolvedOutputDir
198 : resolvedOutputDir + path.sep
199 if (
200 filepath !== resolvedOutputDir &&
201 !filepath.startsWith(outputPrefix)
202 ) {
203 throw new Error(
204 `Prerender output path must stay within the client output directory: ${filename}`,
205 )
206 }
207​
208 await fsp.mkdir(path.dirname(filepath), {
209 recursive: true,
210 })
211​
212 await fsp.writeFile(filepath, html)
213​
214 prerendered.add(page.path)
215​
216 const newPage = await prerenderOptions.onSuccess?.({ page, html })
217​
218 if (newPage) {
219 Object.assign(page, newPage)
220 }
221​
222 if (prerenderOptions.crawlLinks ?? true) {
223 const links = extractLinks(html)
224 for (const link of links) {
225 addCrawlPageTask({ path: link, fromCrawl: true })
226 }
227 }
228 } catch (error) {
229 if (retries < (prerenderOptions.retryCount ?? 0)) {
230 const retryDelay = normalizeRetryDelay(prerenderOptions.retryDelay)
231 logger.warn(
232 `Encountered error, retrying: ${page.path} in ${retryDelay}ms`,
233 )
234 await new Promise((resolve) => setTimeout(resolve, retryDelay))
235 retriesByPath.set(page.path, retries + 1)
236 addCrawlPageTask(page)
237 } else if (prerenderOptions.failOnError ?? true) {
238 throw error
239 }
240 }
241 })
242 }
243 }
244​
245 function normalizeRetryDelay(value: number | undefined): number {
246 const retryDelay = Number(value)
247​
248 if (!Number.isFinite(retryDelay) || retryDelay < 0) {
249 return DEFAULT_RETRY_DELAY
250 }
251​
252 return Math.trunc(retryDelay)
253 }
254​
255 async function requestWithRedirects(
256 requestPath: string,
257 options?: RequestInit,
258 maxRedirects: number = 5,
259 currentUrl = resolveInternalUrl(requestPath, routerBaseUrl, routerBaseUrl),
260 ): Promise<Response> {
261 const path = currentUrl && toPreviewPath(currentUrl, routerBaseUrl)
262 if (!path) {
263 throw new Error(`Prerender request path must be relative: ${requestPath}`)
264 }
265​
266 const response = await handler.request(path, {
267 ...options,
268 redirect: 'manual',
269 })
270​
271 if (isRedirectResponse(response) && maxRedirects > 0) {
272 const location = response.headers.get('location')!.trim()
273 const url = resolveInternalUrl(location, currentUrl, routerBaseUrl)
274 const redirectPath = url && toPreviewPath(url, routerBaseUrl)
275​
276 if (url && redirectPath) {
277 return requestWithRedirects(
278 redirectPath,
279 options,
280 maxRedirects - 1,
281 url,
282 )
283 }
284​
285 logger.warn(`Skipping redirect outside the preview basepath: ${location}`)
286 }
287​
288 return response
289 }
291​
292function isRedirectResponse(res: Response) {
293 return (
294 [301, 302, 303, 307, 308].includes(res.status) &&
295 !!res.headers.get('location')?.trim()
296 )
298​
299function resolveInternalUrl(
300 href: string,
301 baseUrl: URL,
302 allowedOrigin: URL,
303): URL | undefined {
304 try {
305 const url = new URL(href, baseUrl)
306 if (
307 url.protocol !== allowedOrigin.protocol ||
308 url.origin !== allowedOrigin.origin ||
309 url.username ||
310 url.password
311 ) {
312 return undefined
313 }
314 return url
315 } catch {
316 return undefined
317 }
319​
320function toPreviewPath(url: URL, routerBaseUrl: URL): string | undefined {
321 const basepath = routerBaseUrl.pathname.replace(/\/$/, '')
322 if (
323 basepath &&
324 url.pathname !== basepath &&
325 !url.pathname.startsWith(basepath + '/')
326 ) {
327 return undefined
328 }
329 if (url.pathname.startsWith('//')) {
330 return url.href
331 }
332 return url.pathname + url.search
334​
335export function validateAndNormalizePrerenderPages(
336 pages: Array<Page>,
337 routerBaseUrl: URL,
338): Array<Page> {
339 return pages.map((page) => {
340 let url: URL
341 try {
342 url = new URL(page.path, routerBaseUrl)
343 } catch (err) {
344 throw new Error(`prerender page path must be relative: ${page.path}`, {
345 cause: err,
346 })
347 }
348​
349 if (
350 url.protocol !== routerBaseUrl.protocol ||
351 url.origin !== routerBaseUrl.origin ||
352 url.username ||
353 url.password
354 ) {
355 throw new Error(`prerender page path must be relative: ${page.path}`)
356 }
357​
358 const decodedPathname = decodeURIComponent(url.pathname)
359 const normalizedPath = decodedPathname + url.search + url.hash
360 const normalizedUrl = resolveInternalUrl(
361 normalizedPath,
362 routerBaseUrl,
363 routerBaseUrl,
364 )
365 if (!normalizedUrl) {
366 throw new Error(`prerender page path must be relative: ${page.path}`)
367 }
368​
369 return {
370 ...page,
371 path: normalizedPath,
372 }
373 })