diff --git a/package.json b/package.json index 729ccb3..0fdf034 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "express-prometheus-middleware": "^0.9.6", "greenlock": "^4.0.4", "greenlock-express": "^4.0.3", + "html-minifier": "^4.0.0", "lodash": "^4.17.20", "mime-types": "^2.1.27", "pretty-bytes": "^5.4.1", @@ -32,6 +33,7 @@ "@types/etag": "^1.8.0", "@types/express": "^4.17.8", "@types/express-http-proxy": "^1.6.1", + "@types/html-minifier": "^4.0.0", "@types/lodash": "^4.14.161", "@types/mime-types": "^2.1.0", "@types/node": "^14.6.4", diff --git a/src/app.ts b/src/app.ts index d36a153..fb6807f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,10 +1,10 @@ -import express from 'express' +import express, { Request, Response } from 'express' import errorHandler from 'errorhandler' import path from 'path' import proxy from 'express-http-proxy' import promMid from 'express-prometheus-middleware' import { yellow, red, cyan } from 'chalk' -import config, { Route } from './config' +import config, { envConfig, Route } from './config' import cache from './cache' import preRender from './pre-render' import { proxyRoute } from './routes/proxy-route' @@ -20,6 +20,10 @@ const app = express() app.enable('etag') app.use(promMid()) app.use(errorHandler()) +app.use((req, res, next) => { + console.log(`[app] request: ${req.method} ${req.url}`) + next() +}) const shortRouteDescription = (route: Route): string => { switch (route.type) { @@ -73,11 +77,6 @@ app.post('/__ssr/admin/clear-cache', async (req, res) => { if (!config.preRender) { return res.status(200).send(`Cache cleared. Pre-rendering has not been run: pre-render is not configured.`) } - if (config.preRenderPaths.length === 0) { - return res - .status(200) - .send(`Cache cleared. Pre-rendering has not been run: pre-render path are not configured.`) - } await preRender() return res.status(200).send(`Cache cleared. Pre-rendering has been run.`) } @@ -133,9 +132,36 @@ config.routes.forEach((route) => { if (req.header('User-Agent') === config.userAgent) { req.url = req.originalUrl if (config.log.selfRequests) { - console.log(`[app] self request: ${req.originalUrl}, proxying -> ${route.target}${req.url}`) + console.log(`[app] self request: ${cyan(req.originalUrl)} proxying -> ${route.target}${req.url}`) } - return proxy(route.target)(req, res, next) + return proxy(route.target, { + userResDecorator: (proxyRes: Response, proxyResData: any, userReq: Request, userRes: Response) => { + if (proxyRes.statusCode >= 301 && proxyRes.statusCode <= 303) { + if (config.log.selfRequests) { + console.log( + `[app] self request: ${cyan(req.originalUrl)} proxy redirect: ${proxyRes.statusCode} ${ + (proxyRes as any).headers.location + }` + ) + } + userRes.status(proxyRes.statusCode) + const locationUrl = new URL((proxyRes as any).headers.location) + locationUrl.hostname = envConfig.hostname + locationUrl.protocol = 'http' + locationUrl.port = String(config.httpPort) + userRes.setHeader('location', locationUrl.toString()) + return '' + } + if (!(proxyRes.statusCode >= 200 && proxyRes.statusCode < 300)) { + if (config.log.selfRequests) { + console.log( + `[app] self request: ${cyan(req.originalUrl)} proxy non-200 response: ${proxyRes.statusCode}` + ) + } + } + return proxyResData + }, + })(req, res, next) } return pageRoute(route, req, res) diff --git a/src/cache.ts b/src/cache.ts index 4d0cf2a..c6b8d12 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,10 +1,12 @@ import etag from 'etag' +import { Request } from 'puppeteer-core' import config, { runtimeConfig } from './config' export interface CacheEntry { content: Buffer etag: string status: number + location?: URL } export class Cache { @@ -33,26 +35,48 @@ export class Cache { return undefined } - set(url: string, content: string, status: number): CacheEntry { + set(url: string, content: string, status: number, location?: URL): CacheEntry { const entry = { content: Buffer.from(content), etag: etag(content), status, + location, } if (this.enabled || runtimeConfig.cacheEverything) { this.cache.set(url, entry) + if (config.log.cache) { + console.log( + `[cache] cached page ${url} status: ${entry.status}${entry.location ? ` location: ${entry.location}` : ''}` + ) + } + } else { + console.log( + `[cache] NOT caching page ${url} status: ${entry.status}${entry.location ? ` location: ${entry.location}` : ''}` + ) } return entry } - setAsset(url: string, buffer: Buffer, status: number): CacheEntry { + setAsset(url: string, buffer: Buffer, status: number, location?: URL): CacheEntry { const entry = { content: buffer, etag: etag(buffer), status, + location, } if (this.enabled || runtimeConfig.cacheEverything) { this.assetsCache.set(url, entry) + if (config.log.cache) { + console.log( + `[cache] cached asset ${url} status: ${entry.status}${entry.location ? ` location: ${entry.location}` : ''}` + ) + } + } else { + console.log( + `[cache] NOT caching asset ${url} status: ${entry.status}${ + entry.location ? ` location: ${entry.location}` : '' + }` + ) } return entry } @@ -66,4 +90,27 @@ export class Cache { const cache: Cache = new Cache(config.enableCache) +export const cachePageRenderResult = ({ + url, + html, + status, + redirects, +}: { + url: string + html: string + status: number + location?: string + redirects?: Request[] +}): CacheEntry => { + let endUrl = url + if (redirects?.length > 0) { + redirects.forEach((r) => { + console.log(`[render-page] ${r.url()} -> ${r.frame().url()}`) + cache.set(r.url(), '', 302, new URL(r.frame().url())) + endUrl = r.frame().url() + }) + } + return cache.set(endUrl, html, status) +} + export default cache diff --git a/src/config.ts b/src/config.ts index bb3a21a..28a65c4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,7 +2,8 @@ import _ from 'lodash' import fs from 'fs' import os from 'os' import yargsRaw from 'yargs' -import { envString, envNumber, envBoolean, envStringList } from './env-config' +import { Options } from 'html-minifier' +import { envString, envNumber, envBoolean, envStringList, envNumberList } from './env-config' export interface Config { browserExecutable?: string @@ -11,48 +12,69 @@ export interface Config { httpPort: number enableCache: boolean adminAccessKey: string - preRender: boolean - preRenderScrape: boolean - preRenderScrapeDepth?: number + origins: string[] + pathifySingleParams: boolean + preRender?: PreRenderConfig log: LogConfig page: PageConfig routes: Route[] - preRenderPaths: [] static?: StaticSiteConfig } +export interface PreRenderConfig { + enabled: boolean + paths?: string[] + scrape?: boolean + scrapeDepth?: number + pause?: number +} + export interface StaticSiteConfig { contentOutput?: string - nginxConfigFile?: string - nginxServerName?: string pageReplace?: [[string, string]] - nginxExtraConfig?: any + pageReplaceAfterMinimize?: [[string, string]] + nginx?: StaticSiteNginxConfig + s3?: StaticSiteS3Config + minify?: Options +} + +export interface StaticSiteNginxConfig { + configFile: string + serverName?: string + extraConfig?: unknown notFoundPage?: string errorCodes: number[] errorPage?: string } +export interface StaticSiteS3Config { + uploadScript: string + bucketName: string + awsProfile: string +} + export interface PageConfig { waitSelector: string - statusCodeSelector?: string - statusCodeFunction?: string + statusCodeSelector: string + statusCodeFunction: string preNavigationScript?: string - navigateFunction: string + navigateFunction?: string abortResourceRequests: boolean requestBlacklist: string[] } export interface LogConfig { - navigation?: boolean - renderTime?: boolean - selfRequests?: boolean - routeMatch?: number // 0 -- disable - pageLocation?: boolean - pageErrors?: boolean - pageConsole?: boolean - pageResponses?: boolean - pageAbortedRequests?: boolean - pageFailedRequests?: boolean + navigation: boolean + renderTime: boolean + selfRequests: boolean + routeMatch: number // 0 -- disable + pageErrors: boolean + pageConsole: boolean + pageRequests: boolean + pageResponses: boolean + pageAbortedRequests: boolean + pageFailedRequests: boolean + cache: boolean } export interface ProxyRoute { @@ -62,7 +84,7 @@ export interface ProxyRoute { regex?: string[] target: string modifyUrl?: string - nginxExtraConfig?: any + nginxExtraConfig?: unknown } export interface AssetRoute { @@ -72,7 +94,7 @@ export interface AssetRoute { regex?: string[] dir: string maxAge?: number // default 31557600000 - nginxExtraConfig?: any + nginxExtraConfig?: unknown nginxExpires?: string nginxCacheControlPublic?: boolean } @@ -83,7 +105,7 @@ export interface AssetProxyRoute { path?: string[] regex?: string[] target: string - nginxExtraConfig?: any + nginxExtraConfig?: unknown nginxExpires?: string nginxCacheControlPublic?: boolean } @@ -94,7 +116,7 @@ export interface PageRoute { path?: string[] regex?: string[] source: string - nginxExtraConfig?: any + nginxExtraConfig?: unknown } export interface PageProxyRoute { @@ -103,7 +125,7 @@ export interface PageProxyRoute { path?: string[] regex?: string[] target: string - nginxExtraConfig?: any + nginxExtraConfig?: unknown } export interface NotFoundRoute { @@ -130,33 +152,32 @@ const defaultConfig: Config = { userAgent: 'ssr/proxy', httpPort: 40080, enableCache: true, - preRender: false, - preRenderScrape: false, - preRenderPaths: [], adminAccessKey: '', + origins: [], + pathifySingleParams: false, page: { waitSelector: 'title[data-status]', preNavigationScript: "document.head.querySelector('title').removeAttribute('data-status')", statusCodeSelector: 'title[data-status]', statusCodeFunction: '(e) => e.dataset.status', // eslint-disable-next-line no-template-curly-in-string - navigateFunction: '(url) => window.location.href = url', abortResourceRequests: true, requestBlacklist: [], }, log: { navigation: false, + renderTime: true, + selfRequests: false, routeMatch: 2, pageErrors: true, pageConsole: false, pageResponses: false, + pageRequests: false, pageAbortedRequests: false, pageFailedRequests: true, + cache: false, }, routes: [], - static: { - errorCodes: [500, 502], - }, } export const { argv } = yargsRaw @@ -193,6 +214,16 @@ export const { argv } = yargsRaw description: 'Enable the rendered pages cache', demandOption: false, }, + origins: { + type: 'array', + description: 'Origins to consider part of the site (when scraping, rewriting links, etc)', + demandOption: false, + }, + 'pathify-single-params': { + boolean: true, + description: 'Replace ..path?param=value with ..path/param/value', + demandOption: false, + }, 'pre-render': { boolean: true, description: 'Pre-render the pages', @@ -208,6 +239,11 @@ export const { argv } = yargsRaw description: 'Scrape depth', demandOption: false, }, + 'pre-render-pause': { + type: 'number', + description: 'Pause between pages (millis)', + demandOption: false, + }, 'pre-render-paths': { type: 'array', description: 'Paths to pre-render (besides /)', @@ -243,6 +279,11 @@ export const { argv } = yargsRaw description: 'Log page console', demandOption: false, }, + 'log-page-requests': { + boolean: true, + description: 'Log page HTTP requests', + demandOption: false, + }, 'log-page-responses': { boolean: true, description: 'Log page HTTP responses', @@ -258,6 +299,11 @@ export const { argv } = yargsRaw description: 'Log page failed HTTP requests', demandOption: false, }, + 'log-cache': { + boolean: true, + description: 'Log caching', + demandOption: false, + }, 'page-wait-selector': { type: 'string', description: 'CSS selector to wait on when rendering pages (default "title[data-status]")', @@ -330,6 +376,21 @@ export const { argv } = yargsRaw description: 'Origin base URL (when using the default config)', demandOption: false, }, + 'static-s3-upload-script': { + type: 'string', + description: 'Path to the target file for s3 upload script (static site)', + demandOption: false, + }, + 'static-s3-bucket-name': { + type: 'string', + description: 'Bucket name to be used in the upload script (static site)', + demandOption: false, + }, + 'static-s3-aws-profile': { + type: 'string', + description: 'AWS profile to be used in the upload script (static site)', + demandOption: false, + }, }) .demandCommand(1, 1) @@ -365,10 +426,23 @@ config.userAgent = argv['user-agent'] || envString('USER_AGENT') || config.userA config.httpPort = argv['http-port'] || envNumber('HTTP_PORT') || config.httpPort config.adminAccessKey = envString('ADMIN_ACCESS_KEY') || config.adminAccessKey config.enableCache = argv['enable-cache'] || envBoolean('ENABLE_CACHE') || config.enableCache -config.preRender = argv['pre-render'] || envBoolean('PRE_RENDER') || config.preRender -config.preRenderScrape = argv['pre-render-scrape'] || envBoolean('PRE_RENDER_SCRAPE') || config.preRenderScrape -config.preRenderScrapeDepth = - argv['pre-render-scrape-depth'] || envNumber('PRE_RENDER_SCRAPE_DEPTH') || config.preRenderScrapeDepth +config.origins = + (argv.origins ? argv.origins.map((s) => String(s)) : undefined) || envStringList('ORIGINS') || config.origins + +config.preRender = { + ...(config.preRender || {}), + ...{ + enabled: argv['pre-render'] || envBoolean('PRE_RENDER') || config.preRender?.enabled, + paths: + (argv['pre-render-paths'] && argv['pre-render-paths'].map((s) => s as string)) || + envStringList('PRE_RENDER_PATHS') || + config.preRender?.paths, + scrape: argv['pre-render-scrape'] || envBoolean('PRE_RENDER_SCRAPE') || config.preRender?.scrape, + scrapeDepth: + argv['pre-render-scrape-depth'] || envNumber('PRE_RENDER_SCRAPE_DEPTH') || config.preRender?.scrapeDepth, + pause: argv['pre-render-pause'] || envNumber('PRE_RENDER_PAUSE') || config.preRender?.pause, + }, +} config.log.navigation = argv['log-navigation'] || envBoolean('LOG_NAVIGATION') || config.log.navigation config.log.selfRequests = argv['log-self-requests'] || envBoolean('LOG_SELF_REQUESTS') || config.log.selfRequests @@ -377,10 +451,12 @@ config.log.routeMatch = argv['log-route-match'] || envNumber('LOG_ROUTE_MATCH') config.log.pageErrors = argv['log-page-errors'] || envBoolean('LOG_PAGE_ERRORS') || config.log.pageErrors config.log.pageConsole = argv['log-page-console'] || envBoolean('LOG_PAGE_CONSOLE') || config.log.pageConsole config.log.pageResponses = argv['log-page-responses'] || envBoolean('LOG_PAGE_RESPONSES') || config.log.pageResponses +config.log.pageRequests = argv['log-page-requests'] || envBoolean('LOG_PAGE_REQUESTS') || config.log.pageRequests config.log.pageAbortedRequests = argv['log-page-aborted-requests'] || envBoolean('LOG_PAGE_ABORTED_REQUESTS') || config.log.pageAbortedRequests config.log.pageFailedRequests = argv['log-page-failed-requests'] || envBoolean('LOG_PAGE_FAILED_REQUESTS') || config.log.pageFailedRequests +config.log.cache = argv['log-cache'] || envBoolean('LOG_CACHE') || config.log.cache config.page.waitSelector = argv['page-wait-selector'] || envString('PAGE_WAIT_SELECTOR') || config.page.waitSelector config.page.statusCodeSelector = @@ -404,16 +480,48 @@ config.page.requestBlacklist = config.static.contentOutput = argv['static-content-output'] || envString('STATIC_CONTENT_OUTPUT') || config.static.contentOutput -config.static.nginxConfigFile = - argv['static-nginx-config-file'] || envString('STATIC_NGINX_CONFIG_FILE') || config.static.nginxConfigFile -config.static.nginxServerName = - argv['static-nginx-server-name'] || envString('STATIC_NGINX_SERVER_NAME') || config.static.nginxServerName -config.static.notFoundPage = - argv['static-not-found-page'] || envString('STATIC_NOT_FOUND_PAGE') || config.static.notFoundPage -config.static.errorPage = argv['static-error-page'] || envString('STATIC_ERROR_PAGE') || config.static.errorPage -config.static.errorCodes = - (argv['static-error-codes'] ? argv['static-error-codes'].map((s) => Number(s)) : undefined) || - config.static.errorCodes + +config.pathifySingleParams = + argv['pathify-single-params'] || envBoolean('PATHIFY_SINGLE_PARAMS') || config.pathifySingleParams + +config.static = { + ...(config.static || {}), + ...{ + nginx: { + ...(config.static?.nginx || {}), + ...{ + configFile: + argv['static-nginx-config-file'] || envString('STATIC_NGINX_CONFIG_FILE') || config.static?.nginx?.configFile, + serverName: + argv['static-nginx-server-name'] || envString('STATIC_NGINX_SERVER_NAME') || config.static?.nginx?.serverName, + notFoundPage: + argv['static-not-found-page'] || envString('STATIC_NOT_FOUND_PAGE') || config.static?.nginx?.notFoundPage, + errorPage: argv['static-error-page'] || envString('STATIC_ERROR_PAGE') || config.static?.nginx?.errorPage, + errorCodes: (argv['static-error-codes'] ? argv['static-error-codes'].map((s) => Number(s)) : undefined) || + envNumberList('STATIC_ERROR_CODES') || + config.static?.nginx?.errorCodes || [500, 502], + }, + }, + s3: { + ...(config.static?.s3 || {}), + ...{ + uploadScript: + argv['static-s3-upload-script'] || envString('STATIC_S3_UPLOAD_SCRIPT') || config.static?.s3?.uploadScript, + bucketName: + argv['static-s3-bucket-name'] || envString('STATIC_S3_BUCKET_NAME') || config.static?.s3?.bucketName, + awsProfile: + argv['static-s3-aws-profile'] || envString('STATIC_S3_AWS_PROFILE') || config.static?.s3?.awsProfile, + }, + }, + }, +} + +if (config.static && Object.entries(config.static.nginx || {}).length === 0) { + config.static.nginx = undefined +} +if (config.static && Object.entries(config.static.s3 || {}).length === 0) { + config.static.s3 = undefined +} export const envConfig: EnvConfig = { greenlock: envBoolean('GREENLOCK'), @@ -421,6 +529,8 @@ export const envConfig: EnvConfig = { hostname: os.hostname(), } +config.origins.push(`http://${envConfig.hostname}:${config.httpPort}`) + console.log(JSON.stringify(config, null, 4)) console.log('env config', JSON.stringify(envConfig, null, 4)) diff --git a/src/create-page.ts b/src/create-page.ts index 0c5f84e..1e388e6 100644 --- a/src/create-page.ts +++ b/src/create-page.ts @@ -20,12 +20,6 @@ const createPage = async (browser: Browser): Promise => { }) } - if (config.log.pageLocation) { - page.on('load', () => { - page.evaluate('console.log("Browser location", document.location)') - }) - } - if (config.log.pageErrors) { page.on('pageerror', ({ message }) => console.log(red('[page] error:'), message)) } @@ -38,11 +32,15 @@ const createPage = async (browser: Browser): Promise => { page.on('requestfailed', (request) => { const resourceRequest = ['image', 'stylesheet', 'font'].indexOf(request.resourceType()) !== -1 if ( - (resourceRequest && !pageConfig.abortResourceRequests) || - (!resourceRequest && !isRequestBlacklisted(request)) + request.failure()?.errorText !== 'net::ERR_ABORTED' && + ((resourceRequest && !pageConfig.abortResourceRequests) || (!resourceRequest && !isRequestBlacklisted(request))) ) { console.error( - red(`[page] request failed: ${request.resourceType()} ${request.failure().errorText} ${request.url()}`), + red( + `[page] request failed: ${request.resourceType()} ${ + request.failure().errorText + } ${request.url()} failure: ${request.failure()?.errorText}` + ), request.failure() ) } @@ -52,33 +50,32 @@ const createPage = async (browser: Browser): Promise => { if (pageConfig.abortResourceRequests) { console.log(magenta(`[create-page] will abort resource requests: ${resourcesToAbort}`)) - page.on('request', (request) => { - if (resourcesToAbort.indexOf(request.resourceType()) !== -1) { - if (config.log.pageAbortedRequests) { - console.log(gray(`[page] request aborted - type==${request.resourceType()}: ${request.url()}`)) - } - request.abort().catch(() => Promise.resolve()) - } else { - request.continue().catch(() => Promise.resolve()) - } - }) } else { console.log(magenta(`[create-page] will NOT abort resource requests`)) } if (pageConfig.requestBlacklist.length > 0) { console.log(magenta(`[create-page] will abort blacklisted requests: ${pageConfig.requestBlacklist}`)) - page.on('request', (request) => { - if (isRequestBlacklisted(request)) { - if (config.log.pageAbortedRequests) { - console.log(gray(`[page] request aborted - blacklist: ${request.url()}`)) - } - request.abort().catch(() => Promise.resolve()) - } else { - request.continue().catch(() => Promise.resolve()) - } - }) } + page.on('request', (request) => { + if (config.log.pageRequests) { + console.log(gray(`[page] request - type==${request.resourceType()}: ${request.url()}`)) + } + if (pageConfig.abortResourceRequests && resourcesToAbort.indexOf(request.resourceType()) !== -1) { + if (config.log.pageAbortedRequests) { + console.log(gray(`[page] request aborted - type==${request.resourceType()}: ${request.url()}`)) + } + request.abort().catch(() => Promise.resolve()) + } else if (isRequestBlacklisted(request)) { + if (config.log.pageAbortedRequests) { + console.log(gray(`[page] request aborted - blacklist: ${request.url()}`)) + } + request.abort().catch(() => Promise.resolve()) + } else { + request.continue().catch(() => Promise.resolve()) + } + }) + return page } diff --git a/src/env-config.ts b/src/env-config.ts index 5cc97ce..8c281c7 100644 --- a/src/env-config.ts +++ b/src/env-config.ts @@ -5,6 +5,11 @@ export const envStringList = (name: string): string[] | undefined => { return str && str.split(' ') } +export const envNumberList = (name: string): number[] | undefined => { + const strList = envStringList(name) + return strList && strList.map((s) => Number(s)) +} + export const envBoolean = (name: string): boolean | undefined => envString(name) && envString(name) === '1' export const envNumber = (name: string): number | undefined => envString(name) && Number(envString(name)) diff --git a/src/nginx-config-builder.ts b/src/nginx-config-builder.ts new file mode 100644 index 0000000..a599892 --- /dev/null +++ b/src/nginx-config-builder.ts @@ -0,0 +1,197 @@ +import { red } from 'chalk' +import config, { envConfig, ProxyRoute, StaticSiteNginxConfig } from './config' +import cache from './cache' +import { buildMatcher, matcherToNginx } from './util' + +export class NginxConfigBuilder { + private _str: string + + private indentation = '' + + constructor() { + this._str = '' + } + + toString(): string { + return this._str + } + + appendConfig(str: string): void { + this._str += `${this.indentation}${str}\n` + } + + appendBlock(prefix: string, suffix: string, fn: () => void): void { + this.appendConfig(prefix) + this.indentation += ' ' + fn() + this.indentation = this.indentation.slice(0, -4) + this.appendConfig(suffix) + this._str += '\n' + } + + writeExtraConfig(extraConfig: unknown): void { + if (typeof extraConfig === 'string') { + this.appendConfig(extraConfig) + } else if (typeof extraConfig === 'object') { + if (Array.isArray(extraConfig)) { + for (const [, value] of Object.entries(extraConfig)) { + this.writeExtraConfig(value) + } + } else { + for (const [key, value] of Object.entries(extraConfig)) { + this.appendBlock(`${key} {`, '}', () => { + this.writeExtraConfig(value) + }) + } + } + } + } +} + +export const buildNginxConfig = ({ + contentRoot, + serverName, + notFoundPage, + errorPage, + errorCodes, + extraConfig, +}: // pathifySingleParams, +{ + pathifySingleParams: boolean + contentRoot: string +} & StaticSiteNginxConfig): string => { + const entries = cache.listEntries() + const nginxConfig = new NginxConfigBuilder() + + const proxyRoutes: (ProxyRoute & { + upstream: string + url: URL + })[] = config.routes + .filter((route) => route.type === 'proxy') + .map((proxyRoute: ProxyRoute, index) => { + return { + upstream: `proxy-${index}`, + url: new URL(proxyRoute.target), + ...proxyRoute, + } + }) + + proxyRoutes.forEach((proxyRoute) => { + nginxConfig.appendBlock(`upstream ${proxyRoute.upstream} {`, '}', () => { + nginxConfig.appendConfig( + `server ${proxyRoute.url.hostname}${proxyRoute.url.port ? `:${proxyRoute.url.port}` : ''};` + ) + }) + }) + + nginxConfig.appendBlock('server {', '}', () => { + if (serverName) { + nginxConfig.appendConfig(`server_name "${serverName}";\n`) + } + + if (extraConfig) { + nginxConfig.writeExtraConfig(extraConfig) + } + + nginxConfig.appendConfig(`root ${contentRoot};\n`) + + proxyRoutes.forEach((proxyRoute) => { + const matchers = buildMatcher(proxyRoute) + matchers.forEach((matcher) => { + nginxConfig.appendBlock(`${matcherToNginx(matcher)} {`, '}', () => { + nginxConfig.appendConfig(`proxy_pass ${proxyRoute.url.protocol}//${proxyRoute.upstream};`) + }) + }) + }) + + // if (pathifySingleParams) { + // const pageEntries = cache.listEntries() + // for (const [urlStr, entry] of entries) { + // const url = new URL(urlStr) + // const searchEntries = Array.from(url.searchParams.entries()) + // if (searchEntries.length === 1 && pathifySingleParams) { + // nginxConfig.appendBlock(`location ${url.pathname}${singleParamPathSuffix(searchEntries[0])} {`, '}', () => { + // nginxConfig.appendConfig('try_files $uri.html =404;') + // }) + // } + // } + // } + config.routes.forEach(async (route) => { + const matchers = buildMatcher(route) + switch (route.type) { + case 'page': + case 'page-proxy': + matchers.forEach((matcher) => { + nginxConfig.appendBlock(`${matcherToNginx(matcher)} {`, '}', () => { + if (route.nginxExtraConfig) { + nginxConfig.writeExtraConfig(route.nginxExtraConfig) + } + nginxConfig.appendConfig('try_files $uri.html =404;') + }) + }) + break + case 'asset': + case 'asset-proxy': + matchers.forEach((matcher) => { + nginxConfig.appendBlock(`${matcherToNginx(matcher)} {`, '}', () => { + if (route.nginxExtraConfig) { + nginxConfig.writeExtraConfig(route.nginxExtraConfig) + } + if (route.nginxExpires) { + nginxConfig.appendConfig(`expires ${route.nginxExpires};`) + } + if (route.nginxCacheControlPublic) { + nginxConfig.appendConfig('add_header Cache-Control "public";') + } + nginxConfig.appendConfig('try_files $uri =404;') + }) + }) + break + default: + break + } + }) + + for (const [urlStr, entry] of entries) { + const url = new URL(urlStr) + const { pathname, searchParams } = url + if (searchParams.toString().length > 0) { + console.warn('[static-site] cannot route URLs with search query in nginx config', urlStr) + } else { + const { status, location } = entry + if (status >= 301 && status <= 303) { + console.log(`[static-site] page redirect: ${pathname} ${status} location:${location}`) + if (location) { + nginxConfig.appendBlock(`location ${pathname} {`, '}', () => { + nginxConfig.appendConfig(`return 301 ${location.pathname};`) + }) + } else { + console.warn(`[static-site] redirect without location: ${pathname}`) + } + } + } + } + + if (notFoundPage) { + if (!cache.get(`http://${envConfig.hostname}:${config.httpPort}${notFoundPage}`)) { + console.warn( + red(`not configuring a Not Found page: ${notFoundPage} was not rendered (is it listed in pre-render?)`) + ) + } else { + nginxConfig.appendConfig(`error_page 404 ${notFoundPage};`) + } + } + + if (errorPage) { + if (!cache.get(`http://${envConfig.hostname}:${config.httpPort}${errorPage}`)) { + console.warn(red(`not configuring an error page: ${errorPage} was not rendered (is it listed in pre-render?)`)) + } else { + errorCodes.forEach((errorCode) => { + nginxConfig.appendConfig(`error_page ${errorCode} ${errorPage};`) + }) + } + } + }) + + return nginxConfig.toString() +} diff --git a/src/page-scraper.ts b/src/page-scraper.ts new file mode 100644 index 0000000..af58eda --- /dev/null +++ b/src/page-scraper.ts @@ -0,0 +1,92 @@ +import { red } from 'chalk' +import { Page } from 'puppeteer-core' + +export interface PathToVisit { + path: string + initial: boolean + depth: number +} + +export class PageScraper { + private _seenPaths: string[] = [] + + private _visitedPaths: string[] = [] + + private _pathsToVisit: PathToVisit[] = [] + + private readonly _scrapeDepth: number + + private readonly _origins: string[] + + constructor(scrapeDepth: number, origins: string[]) { + this._scrapeDepth = scrapeDepth + this._origins = origins + } + + seen(path: string[]): void { + this._seenPaths.push(...path) + } + + pathsToVisit(p: PathToVisit[]): void { + this._pathsToVisit.push(...p) + } + + pathsVisited(): number { + return this._visitedPaths.length + } + + shift(): PathToVisit | undefined { + let next = this._pathsToVisit.shift() + while (next) { + if (this._visitedPaths.indexOf(next.path) === -1) { + this._visitedPaths.push(next.path) + break + } + next = this._pathsToVisit.shift() + } + return next + } + + remaining(): number { + return this._pathsToVisit.length + } + + async scrape(page: Page, depth: number): Promise { + if (depth <= this._scrapeDepth) { + console.log(`[page-scraper] scraping page, depth ${depth}`) + + // eslint-disable-next-line no-undef + const urlStrings = await page.$$eval('a', (links) => links.map((link: HTMLAnchorElement) => link.href)) + const urls = urlStrings + .filter((a) => a !== '') + .filter((a) => a !== '#') + // eslint-disable-next-line no-script-url + .filter((a) => a !== 'javascript:void(0)') + .map((s) => { + try { + return new URL(s) + } catch (e) { + console.error(red(`[page-scraper] invalid URL: '${s}'`)) + return null + } + }) + .filter((u) => u !== null) + .filter((a) => this._origins.some((origin) => origin === a.origin)) + + console.log('[page-scraper] found links: ', urls.length) + urls.forEach((a) => { + if (this._seenPaths.indexOf(a.pathname) === -1 && !this._pathsToVisit.some((p) => p.path === a.pathname)) { + console.log(`[page-scraper] found link: ${a.pathname}`) + this._seenPaths.push(a.pathname) + this._pathsToVisit.push({ + path: a.pathname, + initial: false, + depth: depth + 1, + }) + } + }) + } else { + console.log(`[page-scraper] not scraping: ${depth} > ${this._scrapeDepth}`) + } + } +} diff --git a/src/pre-render.ts b/src/pre-render.ts index 3d38ecd..33479a5 100644 --- a/src/pre-render.ts +++ b/src/pre-render.ts @@ -1,143 +1,130 @@ -import { red, yellow } from 'chalk' +import { blueBright, yellow, magenta } from 'chalk' +import { Request } from 'puppeteer-core' import createPage from './create-page' import renderPage from './render-page' -import cache from './cache' +import { cachePageRenderResult } from './cache' import config, { envConfig } from './config' import createBrowser from './create-browser' import { renderTimeMetric } from './metrics' +import { snooze } from './util' +import { PageScraper } from './page-scraper' -const { preRender: preRenderEnabled, preRenderPaths, page: pageConfig, preRenderScrape, preRenderScrapeDepth } = config - -const preRender = async (): Promise => { - if (!preRenderEnabled) { +const preRender = async (): Promise<{ pagesRendered: number }> => { + const { preRender: preRenderConfig, page: pageConfig } = config + if (!preRenderConfig) { + console.error(`pre-render is not configured`) + return { pagesRendered: 0 } + } + if (!preRenderConfig.enabled) { console.error(`pre-render is not enabled`) + return { pagesRendered: 0 } } const target = `http://${envConfig.hostname}:${config.httpPort}` const browser = await createBrowser() - const page = await createPage(browser) - - const start = Date.now() - const timerHandle = renderTimeMetric.startTimer({ url: `${target}/` }) - if (config.log.navigation) { - console.log(`[pre-render] navigating to: ${yellow(`${target}/`)}`) - } - await page.goto(`${target}/`, { waitUntil: 'networkidle0' }) - const [status, html] = await renderPage(page) - timerHandle() - const ttRenderMs = Date.now() - start - if (config.log.renderTime) { - console.info(`[pre-render] rendered ${target}/: ${yellow(`${ttRenderMs}ms`)}`) - } - cache.set(`${target}/`, html, status) - - const pathsToVisit: { path: string; depth: number }[] = preRenderPaths.map((path) => ({ - path, - depth: 1, - })) - const seenPaths: string[] = ['/', ...preRenderPaths] - const visitedPaths: string[] = ['/'] + let page = await createPage(browser) - const scrapePaths = async (depth: number) => { - if (preRenderScrape && depth <= preRenderScrapeDepth) { - // eslint-disable-next-line no-undef - const urlStrings = await page.$$eval('a', (links) => links.map((link: HTMLAnchorElement) => link.href)) - const urls = urlStrings - .filter((a) => a !== '') - .filter((a) => a !== '#') - // eslint-disable-next-line no-script-url - .filter((a) => a !== 'javascript:void(0)') - .map((s) => { - try { - return new URL(s) - } catch (e) { - console.error(red(`[pre-render] invalid URL: '${s}'`)) - return null - } - }) - .filter((u) => u !== null) + const { paths, scrape, scrapeDepth } = preRenderConfig - console.log('[pre-render] found links: ', urls.length) - urls - .filter((a) => a.hostname === envConfig.hostname && a.protocol === 'http:') - .filter((a) => seenPaths.indexOf(a.pathname) === -1) - .forEach((a) => { - console.log(`[pre-render] found link: ${a.pathname}`) - seenPaths.push(a.pathname) - pathsToVisit.push({ - path: a.pathname, - depth: depth + 1, - }) - }) - } - } + const pageScraper = new PageScraper(scrapeDepth, config.origins) + pageScraper.seen([...(paths || ['/'])]) + pageScraper.pathsToVisit( + (paths?.length > 0 ? paths : ['/']).map((path, index) => ({ + path, + initial: index === 0, + depth: 1, + })) + ) + // const pathsToVisit: { path: string; initial: boolean; depth: number }[] = console.log( + // 'initial paths to visit', + // pathsToVisit + // ) - await scrapePaths(1) + // const seenPaths: string[] = [...(paths || ['/'])] + // const visitedPaths: string[] = [] - const processPath = async (path: string, depth: number) => { + const processPath = async (path: string, initial: boolean, depth: number) => { + console.log(`[pre-render] processing path: ${path} depth: ${depth}${initial ? ' initial' : ''}`) const url = `${target}${path}` const pathStart = Date.now() - const pathTimerHandle = renderTimeMetric.startTimer({ url: `${target}/` }) - if (pageConfig.preNavigationScript) { - if (config.log.navigation) { - console.log(`[pre-render] running pre-navigation script`) + const timerHandle = renderTimeMetric.startTimer({ url }) + let redirects: Request[] | undefined + if (initial) { + console.log(`[pre-render] navigating to: ${yellow(url)} (initial)`) + const response = await page.goto(url, { waitUntil: 'networkidle0' }) // networkidle0 waits for the network to be idle (no requests for 500ms). + redirects = response.request().redirectChain() + } else { + if (pageConfig.preNavigationScript) { + if (config.log.navigation) { + console.log(`[pre-render] running pre-navigation script`) + } + await page.evaluate((preNavigationScript) => { + // eslint-disable-next-line no-eval + eval(preNavigationScript) + }, pageConfig.preNavigationScript) + console.log('[pre-render] wait for navigation...') + } + + if (pageConfig.navigateFunction) { + if (config.log.navigation) { + console.log(`[pre-render] navigating to: ${yellow(url)} (${pageConfig.navigateFunction})`) + } + await page.evaluate( + (navigateFunction, navigateToURL) => { + // eslint-disable-next-line no-eval + eval(`(${navigateFunction})('${navigateToURL}')`) + }, + pageConfig.navigateFunction, + url + ) + await page.waitForNavigation({ waitUntil: 'networkidle0' }) // networkidle0 waits for the network to be idle (no requests for 500ms). + } else { + if (config.log.navigation) { + console.log(`[pre-render] navigating to: ${yellow(url)}`) + } + const response = await page.goto(url, { waitUntil: 'networkidle0' }) // networkidle0 waits for the network to be idle (no requests for 500ms). + redirects = response.request().redirectChain() } - await page.evaluate((preNavigationScript) => { - // eslint-disable-next-line no-eval - eval(preNavigationScript) - }, pageConfig.preNavigationScript) - } - if (config.log.navigation) { - console.log(`[pre-render] navigating to: ${yellow(url)} (${pageConfig.navigateFunction})`) } - await page.evaluate( - (navigateFunction, navigateToURL) => { - // eslint-disable-next-line no-eval - eval(`(${navigateFunction})('${navigateToURL}')`) - }, - pageConfig.navigateFunction, - url - ) - const [pathStatus, pathHtml] = await renderPage(page) - pathTimerHandle() + const [status, html] = await renderPage(page, config.origins) + timerHandle() const pathRenderMs = Date.now() - pathStart if (config.log.renderTime) { console.info(`[pre-render] rendered ${url}: ${yellow(`${pathRenderMs}ms`)}`) } - cache.set(url, pathHtml, pathStatus) - await scrapePaths(depth) + + cachePageRenderResult({ url, html, status, redirects }) + if (scrape) { + await pageScraper.scrape(page, depth) + } + if (!pageConfig.navigateFunction) { + await page.close() + page = await createPage(browser) + } } const processPaths = async () => { - try { - while (pathsToVisit.length > 0) { - console.log( - `[pre-render] path to visit remaining: ${pathsToVisit.length}, considering path: ${JSON.stringify( - pathsToVisit[0] - )}` - ) - const { path, depth } = pathsToVisit[0] - pathsToVisit.shift() - if (visitedPaths.indexOf(path) === -1 && (!preRenderScrapeDepth || depth <= preRenderScrapeDepth)) { - console.log(`[pre-render] pushing path to visited: ${path} ${depth}`) - visitedPaths.push(path) - console.log(`[pre-render] processing path: ${path} ${depth}`) - // eslint-disable-next-line no-await-in-loop - await processPath(path, depth) - } else if (visitedPaths.indexOf(path) !== -1) { - console.log(`[pre-render] path already visited: ${path}`) - } else { - console.log(`[pre-render] depth exceeded: ${depth} > ${preRenderScrapeDepth}`) - } - } - } catch (e) { - console.error(red('[pre-render] process paths failed'), e) + const nextPathToVisit = pageScraper.shift() + if (!nextPathToVisit) { + return + } + console.log(magenta(`[pre-render] considering path: ${JSON.stringify(nextPathToVisit)}`)) + const { path, initial, depth } = nextPathToVisit + + // eslint-disable-next-line no-await-in-loop + await processPath(path, initial, depth) + console.log(magenta(`[pre-render] path to visit remaining: ${pageScraper.remaining()}`)) + if (preRenderConfig.pause) { + console.log(blueBright(`[pre-render] pause between pages: ${preRenderConfig.pause}ms`)) + await snooze(preRenderConfig.pause) } + await processPaths() } await processPaths() - console.log(yellow(`[pre-render] pre-render finished: ${visitedPaths.length} pages rendered`)) + console.log(yellow(`[pre-render] pre-render finished: ${pageScraper.pathsVisited()} pages rendered`)) + return { pagesRendered: pageScraper.pathsVisited() } } export default preRender diff --git a/src/render-page.ts b/src/render-page.ts index 95af859..aabc039 100644 --- a/src/render-page.ts +++ b/src/render-page.ts @@ -4,9 +4,8 @@ import config from './config' const { page: pageConfig } = config -async function renderPage(page: Page): Promise<[number, string]> { +async function renderPage(page: Page, origins: string[], pathifySingleParams?: boolean): Promise<[number, string]> { try { - // networkidle0 waits for the network to be idle (no requests for 500ms). console.log(`[render-page] wait for selector: ${pageConfig.waitSelector}`) await page.waitForSelector(pageConfig.waitSelector) } catch (err) { @@ -16,18 +15,36 @@ async function renderPage(page: Page): Promise<[number, string]> { throw new Error(`Wait for selector (${pageConfig.waitSelector}) timed out`) } - if (pageConfig.statusCodeSelector) { + let status = 200 + if (pageConfig.statusCodeSelector !== '') { // eslint-disable-next-line no-eval - const status = Number(await page.$eval(pageConfig.statusCodeSelector, eval(pageConfig.statusCodeFunction))) + status = Number(await page.$eval(pageConfig.statusCodeSelector, eval(pageConfig.statusCodeFunction))) if (status >= 200 && status < 300) { console.log(`[render-page] status: ${cyan(status)}`) } else { console.log(`[render-page] status: ${red(status)}`) } - return [status, await page.content()] } - return [200, await page.content()] + if (pathifySingleParams) { + await page.$$eval('a', (elements) => { + elements.forEach((element: any) => { + const { href } = element + const url = new URL(href) + if (origins.some((o) => o === url.origin)) { + const searchEntries = Array.from(url.searchParams.entries()) + if (searchEntries.length === 1) { + // eslint-disable-next-line no-param-reassign + element.href = `${href}/${encodeURIComponent(searchEntries[0][0])}/${encodeURIComponent( + searchEntries[0][1] + )}` + } + } + }) + }) + } + + return [status, await page.content()] } export default renderPage diff --git a/src/render-url.ts b/src/render-url.ts index cfc7647..0b96174 100644 --- a/src/render-url.ts +++ b/src/render-url.ts @@ -2,7 +2,7 @@ import { yellow } from 'chalk' import config from './config' import createPage from './create-page' import renderPage from './render-page' -import cache, { CacheEntry } from './cache' +import cache, { CacheEntry, cachePageRenderResult } from './cache' import createBrowser from './create-browser' import { renderTimeMetric } from './metrics' @@ -18,7 +18,8 @@ const renderUrl = async (url: string): Promise { + const uploadScript = new S3UploadScriptBuilder(awsProfile, bucketName) + + config.routes + .filter((route) => route.type === 'proxy') + .forEach((proxyRoute: ProxyRoute) => { + console.log(red(`[s3] proxy routes are not supported in S3 static sites: ${JSON.stringify(proxyRoute)}`)) + }) + + uploadScript.append('#!/usr/bin/env bash\n') + + const entries = cache.listEntries() + for (const [urlStr, entry] of entries) { + console.log(`[s3] entry: ${urlStr} ${entry.status} ${entry.location || ''}`) + + const { status } = entry + if (!(status >= 301 && status <= 303)) { + const fileName = getFileName(contentRoot, urlStr, pathifySingleParams) + uploadScript.cp({ + source: `${contentRoot}${fileName}.html`, + target: fileName, + contentType: 'text/html', + acl: 'public-read', + }) + uploadScript.cp({ + source: `${contentRoot}${fileName}.html`, + target: `${fileName}.html`, + contentType: 'text/html', + acl: 'public-read', + }) + } + } + + const assetEntries = cache.listAssetEntries() + for (const [urlStr, entry] of assetEntries) { + if (!(entry.status >= 301 && entry.status <= 303)) { + const fileName = getFileName(contentRoot, urlStr, pathifySingleParams) + uploadScript.cp({ + source: `${contentRoot}${fileName}`, + target: fileName, + contentType: mime.lookup(urlStr) || 'application/octet-stream', + acl: 'public-read', + }) + } + } + + return uploadScript.toString() +} + +export const uploadFileScript = ( + uploadScript: S3UploadScriptBuilder, + contentRoot: string, + source: string, + target: string +): void => { + let targetFile = target + + // if target is a directory a new file with the same name will be created + if (fs.existsSync(target)) { + if (fs.lstatSync(target).isDirectory()) { + targetFile = path.join(target, path.basename(source)) + } + } + + uploadScript.cp({ + source: source.replace(contentRoot, ''), + target: targetFile.replace(contentRoot, ''), + contentType: mime.lookup(source) || 'application/octet-stream', + acl: 'public-read', + }) +} + +export const uploadDirRecursiveScript = ( + uploadScript: S3UploadScriptBuilder, + contentRoot: string, + source: string, + target: string +): void => { + let files = [] + + // check if folder needs to be created or integrated + const targetFolder = target + if (!fs.existsSync(targetFolder)) { + fs.mkdirSync(targetFolder) + } + + // copy + if (fs.lstatSync(source).isDirectory()) { + files = fs.readdirSync(source) + files.forEach((file) => { + const curSource = path.join(source, file) + if (fs.lstatSync(curSource).isDirectory()) { + uploadDirRecursiveScript(uploadScript, contentRoot, curSource, targetFolder) + } else { + uploadFileScript(uploadScript, contentRoot, curSource, targetFolder) + } + }) + } +} diff --git a/src/server.ts b/src/server.ts index 27c7cc5..a27618f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,13 +1,19 @@ import greenlock from 'greenlock-express' import { Server } from 'http' +import { yellow } from 'chalk' import app from './app' import config, { argv, envConfig } from './config' import preRender from './pre-render' -import staticGen from './static-gen' +import staticSite from './static-site' -const startHttpServer = (callback: (server: Server) => void) => { - console.log(`starting http server...`) - const server = app.listen(config.httpPort, '0.0.0.0', () => callback(server)) +const startHttpServer = (callback?: (server: Server) => void) => { + const server = app.listen(config.httpPort, '0.0.0.0', () => { + console.log(yellow(`server started at http://${envConfig.hostname}:${config.httpPort}`)) + if (callback) { + callback(server) + } + }) + return server } const command = argv._[0] @@ -29,10 +35,7 @@ switch (command) { }) .serve(app) } else { - console.log(`starting express http server...`) - startHttpServer(() => { - console.log(`server started at http://${envConfig.hostname}:${config.httpPort}`) - }) + startHttpServer() } if (config.preRender) { console.log(`pre-rendering the pages...`) @@ -45,12 +48,18 @@ switch (command) { break case 'static-site': // eslint-disable-next-line no-case-declarations - startHttpServer(async (server) => { - console.log(`server started at http://${envConfig.hostname}:${config.httpPort}`) - await staticGen() - server.close() - process.exit(0) - }) + const server = startHttpServer() + staticSite() + .then(() => { + console.info('static site finished') + server.close() + process.exit(0) + }) + .catch((e) => { + console.error('static site failed', e) + server.close() + process.exit(1) + }) break default: console.log(`unrecognized command: ${command}`) diff --git a/src/static-gen.ts b/src/static-gen.ts deleted file mode 100644 index f348f2c..0000000 --- a/src/static-gen.ts +++ /dev/null @@ -1,258 +0,0 @@ -import fs from 'fs' -import path from 'path' -import prettyBytes from 'pretty-bytes' -import { cyan, green, blue, yellow, red } from 'chalk' -import preRender from './pre-render' -import config, { envConfig, ProxyRoute, Route, runtimeConfig } from './config' -import cache from './cache' -import { buildMatcher } from './util' -import { copyDirRecursiveSync } from './copy-dir' - -const { static: staticGenConfig, preRender: preRenderEnabled } = config - -const getFileName = (outputDir: string, urlString: string, fileNameSuffix?: string): string => { - const url = new URL(urlString) - const { pathname, searchParams } = url - const rawDirName = path.dirname(pathname) - const dirName = rawDirName.endsWith('/') ? rawDirName : `${rawDirName}/` - const baseName = path.basename(pathname) - const fileNameWithoutSearchParams = baseName === '' ? 'index' : baseName - - const searchParamsEncoded = - searchParams.toString().length > 0 && Buffer.from(searchParams.toString()).toString('base64') - - const fileName = searchParamsEncoded - ? `${fileNameWithoutSearchParams}-${searchParamsEncoded}` - : fileNameWithoutSearchParams - - const fullDirName = `${outputDir}${dirName}` - if (!fs.existsSync(fullDirName)) { - fs.mkdirSync(fullDirName, { recursive: true }) - } - return `${dirName}${fileName}${fileNameSuffix || ''}` -} - -const outputFile = (outputDir: string, fileName: string, content: Buffer): void => { - console.log(`[static-site] writing file -> ${outputDir}${yellow(fileName)} (${cyan(prettyBytes(content.length))})`) - fs.writeFileSync(`${outputDir}${fileName}`, content) -} - -const matcherToNginx = (matcher: string | RegExp): string => { - if (typeof matcher === 'string') { - if (matcher === '/') { - return `location ~ /.+` - } - return `location ~ ${matcher}/.+` - } - const regex = `${matcher}`.slice(1, -1) - return `location ~ ${regex}` -} - -const staticGen = async (): Promise => { - runtimeConfig.cacheEverything = true - if (config.routes.some((route) => route.type === 'asset-proxy')) { - config.page.abortResourceRequests = false - } - if (!preRenderEnabled) { - console.error(red('[static-site] pre-render is not enabled')) - process.exit(1) - } - if (!staticGenConfig.contentOutput) { - console.error(red('content output dir is not configured')) - process.exit(1) - } - console.log(green(`[static-site] generating static site into ${staticGenConfig.contentOutput}...`)) - console.log(`[static-site] pre-rendering...`) - await preRender() - - const entries = cache.listEntries() - - const processRoute = async (route: Route) => { - switch (route.type) { - case 'asset': - console.log(`[static-site] copying assets: ${yellow(route.dir)} -> ${yellow(staticGenConfig.contentOutput)}`) - copyDirRecursiveSync(route.dir, staticGenConfig.contentOutput) - break - - default: - break - } - } - - const processRoutes = async () => { - for (const route of config.routes) { - // eslint-disable-next-line no-await-in-loop - await processRoute(route) - } - } - - await processRoutes() - - let nginxConfig = '' - let indentation = '' - - const appendConfig = (str: string) => { - nginxConfig += `${indentation}${str}\n` - } - - const appendBlock = (prefix: string, suffix: string, fn: () => void) => { - appendConfig(prefix) - indentation += ' ' - fn() - indentation = indentation.slice(0, -4) - appendConfig(suffix) - nginxConfig += '\n' - } - - const proxyRoutes: (ProxyRoute & { - upstream: string - url: URL - })[] = config.routes - .filter((route) => route.type === 'proxy') - .map((proxyRoute: ProxyRoute, index) => { - return { - upstream: `proxy-${index}`, - url: new URL(proxyRoute.target), - ...proxyRoute, - } - }) - - const writeExtraConfig = (extraConfig: any): void => { - if (typeof extraConfig === 'string') { - appendConfig(extraConfig) - } else if (typeof extraConfig === 'object') { - if (Array.isArray(extraConfig)) { - for (const [, value] of Object.entries(extraConfig)) { - writeExtraConfig(value) - } - } else { - for (const [key, value] of Object.entries(extraConfig)) { - appendBlock(`${key} {`, '}', () => { - writeExtraConfig(value) - }) - } - } - } - } - - proxyRoutes.forEach((proxyRoute) => { - appendBlock(`upstream ${proxyRoute.upstream} {`, '}', () => { - appendConfig(`server ${proxyRoute.url.hostname}${proxyRoute.url.port ? `:${proxyRoute.url.port}` : ''};`) - }) - }) - - appendBlock('server {', '}', () => { - if (staticGenConfig.nginxServerName) { - appendConfig(`server_name "${staticGenConfig.nginxServerName}";\n`) - } - - if (staticGenConfig.nginxExtraConfig) { - writeExtraConfig(staticGenConfig.nginxExtraConfig) - } - - appendConfig(`root ${staticGenConfig.contentOutput};\n`) - - proxyRoutes.forEach((proxyRoute) => { - const matchers = buildMatcher(proxyRoute) - matchers.forEach((matcher) => { - appendBlock(`${matcherToNginx(matcher)} {`, '}', () => { - appendConfig(`proxy_pass ${proxyRoute.url.protocol}//${proxyRoute.upstream};`) - }) - }) - }) - - config.routes.forEach(async (route) => { - const matchers = buildMatcher(route) - switch (route.type) { - case 'asset': - case 'asset-proxy': - matchers.forEach((matcher) => { - appendBlock(`${matcherToNginx(matcher)} {`, '}', () => { - if (route.nginxExtraConfig) { - writeExtraConfig(route.nginxExtraConfig) - } - if (route.nginxExpires) { - appendConfig(`expires ${route.nginxExpires};`) - } - if (route.nginxCacheControlPublic) { - appendConfig('add_header Cache-Control "public";') - } - appendConfig('try_files $uri =404;') - }) - }) - break - default: - break - } - }) - - for (const [urlStr, entry] of entries) { - const fileName = getFileName(staticGenConfig.contentOutput, urlStr, '.html') - let { content } = entry - if (staticGenConfig.pageReplace) { - console.log(`[static-site] applying ${staticGenConfig.pageReplace.length} replacements`) - staticGenConfig.pageReplace.forEach(([regex, replacement]) => { - if (content.toString().match(new RegExp(regex))) { - content = Buffer.from(content.toString().replace(new RegExp(regex), replacement)) - } else { - console.log(red(`[static-site] no matches: ${regex}`)) - } - }) - } - outputFile(staticGenConfig.contentOutput, fileName, content) - const url = new URL(urlStr) - const { pathname, searchParams } = url - if (searchParams.toString().length > 0) { - console.warn('[static-site] cannot route URLs with search query in nginx config', urlStr) - } else { - appendBlock(`location ${pathname} {`, '}', () => { - appendConfig('try_files $uri.html =404;') - }) - } - } - - if (staticGenConfig.notFoundPage) { - const notFoundPage = cache.get(`http://${envConfig.hostname}:${config.httpPort}${staticGenConfig.notFoundPage}`) - if (!notFoundPage) { - console.warn( - red( - `not configuring a Not Found page: ${staticGenConfig.notFoundPage} was not rendered (is it listed in pre-render?)` - ) - ) - } else { - appendConfig(`error_page 404 ${staticGenConfig.notFoundPage};`) - } - } - - if (staticGenConfig.errorPage) { - const errorPage = cache.get(`http://${envConfig.hostname}:${config.httpPort}${staticGenConfig.errorPage}`) - if (!errorPage) { - console.warn( - red( - `not configuring an error page: ${staticGenConfig.errorPage} was not rendered (is it listed in pre-render?)` - ) - ) - } else { - staticGenConfig.errorCodes.forEach((errorCode) => { - appendConfig(`error_page ${errorCode} ${staticGenConfig.errorPage};`) - }) - } - } - - const assetEntries = cache.listAssetEntries() - for (const [urlStr, entry] of assetEntries) { - const fileName = getFileName(staticGenConfig.contentOutput, urlStr) - outputFile(staticGenConfig.contentOutput, fileName, entry.content) - } - }) - - if (staticGenConfig.nginxConfigFile) { - console.log(`[static-site] writing nginx config into ${yellow(staticGenConfig.nginxConfigFile)}`) - fs.writeFileSync(staticGenConfig.nginxConfigFile, nginxConfig) - } else { - console.log('[static-site] nginx config:\n') - console.log(blue(nginxConfig)) - } -} - -export default staticGen diff --git a/src/static-site.ts b/src/static-site.ts new file mode 100644 index 0000000..97dc139 --- /dev/null +++ b/src/static-site.ts @@ -0,0 +1,129 @@ +import fs from 'fs' +import prettyBytes from 'pretty-bytes' +import { cyan, green, red, yellow } from 'chalk' +import { minify } from 'html-minifier' +import preRender from './pre-render' +import config, { Route, runtimeConfig } from './config' +import cache from './cache' +import { getFileName } from './util' +import { copyDirRecursiveSync } from './copy-dir' +import { buildNginxConfig } from './nginx-config-builder' +import { buildS3UploadScript } from './s3-upload-script-builder' + +const outputFile = (outputDir: string, fileName: string, content: Buffer): void => { + console.log(`[static-site] writing file -> ${outputDir}${yellow(fileName)} (${cyan(prettyBytes(content.length))})`) + fs.writeFileSync(`${outputDir}${fileName}`, content) +} + +const staticSite = async (): Promise => { + const { static: staticSiteConfig } = config + if (!staticSiteConfig.contentOutput) { + console.error(red('content output dir is not configured')) + process.exit(1) + } + console.log(green(`[static-site] generating static site into ${staticSiteConfig.contentOutput}...`)) + + config.preRender = { + ...(config.preRender || {}), + ...{ enabled: true }, + } + + runtimeConfig.cacheEverything = true + if (config.routes.some((route) => route.type === 'asset-proxy')) { + config.page.abortResourceRequests = false + } + console.log(`[static-site] pre-rendering...`) + await preRender() + + const processRoute = async (route: Route) => { + switch (route.type) { + case 'asset': + console.log(`[static-site] copying assets: ${yellow(route.dir)} -> ${yellow(staticSiteConfig.contentOutput)}`) + copyDirRecursiveSync(route.dir, staticSiteConfig.contentOutput) + break + + default: + break + } + } + + for (const route of config.routes) { + // eslint-disable-next-line no-await-in-loop + await processRoute(route) + } + + console.log(yellow('[static-site] writing page files...')) + const pageEntries = cache.listEntries() + for (const [urlStr, entry] of pageEntries) { + console.log(`[static-site] entry: ${urlStr} ${entry.status} ${entry.location || ''}`) + + const { status } = entry + if (!(status >= 301 && status <= 303)) { + const fileName = getFileName( + staticSiteConfig.contentOutput, + urlStr, + staticSiteConfig.pathifySingleParams, + '.html' + ) + let { content } = entry + if (staticSiteConfig.pageReplace) { + console.log(`[static-site] applying ${staticSiteConfig.pageReplace.length} replacements`) + staticSiteConfig.pageReplace.forEach(([regex, replacement]) => { + if (content.toString().match(new RegExp(regex))) { + content = Buffer.from(content.toString().replace(new RegExp(regex), replacement)) + } else { + console.log(`[static-site] ${red('no matches')}: ${regex}`) + } + }) + } + content = Buffer.from(minify(content.toString(), staticSiteConfig.minify)) + if (staticSiteConfig.pageReplaceAfterMinimize) { + console.log( + `[static-site] applying ${staticSiteConfig.pageReplaceAfterMinimize.length} replacements (after html-minimize)` + ) + staticSiteConfig.pageReplaceAfterMinimize.forEach(([regex, replacement]) => { + if (content.toString().match(new RegExp(regex))) { + content = Buffer.from(content.toString().replace(new RegExp(regex), replacement)) + } else { + console.log(`[static-site] ${red('no matches')}: ${regex}`) + } + }) + } + outputFile(staticSiteConfig.contentOutput, fileName, content) + } + } + + console.log(yellow('[static-site] writing asset files...')) + const assetEntries = cache.listAssetEntries() + for (const [urlStr, entry] of assetEntries) { + if (!(entry.status >= 301 && entry.status <= 303)) { + const fileName = getFileName(staticSiteConfig.contentOutput, urlStr, false) + try { + outputFile(staticSiteConfig.contentOutput, fileName, entry.content) + } catch (e) { + console.error(red(`[static-site] failed to write asset file for ${urlStr} ${entry.status}: ${e.message}`)) + } + } + } + + if (staticSiteConfig.nginx) { + const nginxConfig = buildNginxConfig({ + contentRoot: staticSiteConfig.contentOutput, + pathifySingleParams: staticSiteConfig.pathifySingleParams, + ...staticSiteConfig.nginx, + }) + console.log(`[static-site] writing nginx config into ${yellow(staticSiteConfig.nginx.configFile)}`) + fs.writeFileSync(staticSiteConfig.nginx.configFile, nginxConfig) + } + if (staticSiteConfig.s3) { + const s3UploadScript = buildS3UploadScript({ + contentRoot: staticSiteConfig.contentOutput, + pathifySingleParams: staticSiteConfig.pathifySingleParams, + ...staticSiteConfig.s3, + }) + console.log(`[static-site] writing s3 upload script into ${yellow(staticSiteConfig.s3.uploadScript)}`) + fs.writeFileSync(staticSiteConfig.s3.uploadScript, s3UploadScript) + } +} + +export default staticSite diff --git a/src/util.ts b/src/util.ts index 3c29ece..306a4c4 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,3 +1,5 @@ +import path from 'path' +import fs from 'fs' import { Route } from './config' export const modifyUrl = (modifyScript: string, url: string): string => { @@ -13,3 +15,60 @@ export const buildMatcher = (route: Route): (string | RegExp)[] => { return [...paths, ...regexes, ...extensions] } + +export const matcherToNginx = (matcher: string | RegExp): string => { + if (typeof matcher === 'string') { + if (matcher === '/') { + return `location ~ /.+` + } + return `location ~ ${matcher}/.+` + } + const regex = `${matcher}`.slice(1, -1) + return `location ~ ${regex}` +} + +export const singleParamPathSuffix = (url: URL): string | undefined => { + const { searchParams } = url + const searchEntries = Array.from(searchParams.entries()) + if (searchEntries.length === 1) { + return `/${encodeURIComponent(searchEntries[0][0])}/${encodeURIComponent(searchEntries[0][1])}` + } + return undefined +} + +export const getFileName = ( + outputDir: string, + urlString: string, + pathifySingleParams: boolean, + fileNameSuffix?: string +): string => { + const url = new URL(urlString) + const { pathname, searchParams } = url + const rawDirName = path.dirname(pathname) + const dirName = rawDirName.endsWith('/') ? rawDirName : `${rawDirName}/` + const baseName = path.basename(pathname) + const fileNameWithoutSearchParams = baseName === '' ? 'index' : baseName + + const searchEntries = Array.from(searchParams.entries()) + const singleParamPathified = singleParamPathSuffix(url) + let searchParamsEncoded: string + if (singleParamPathified && pathifySingleParams) { + searchParamsEncoded = singleParamPathified + } else if (searchEntries.length > 0) { + searchParamsEncoded = `-${Buffer.from(searchParams.toString()).toString('base64')}` + } + + const fileName = searchParamsEncoded + ? `${fileNameWithoutSearchParams}${searchParamsEncoded}` + : fileNameWithoutSearchParams + + const fullDirName = searchParamsEncoded + ? `${outputDir}${dirName}/${path.dirname(`${fileNameWithoutSearchParams}${searchParamsEncoded}`)}` + : `${outputDir}${dirName}` + if (!fs.existsSync(fullDirName)) { + fs.mkdirSync(fullDirName, { recursive: true }) + } + return `${dirName}${fileName}${fileNameSuffix || ''}` +} + +export const snooze = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/yarn.lock b/yarn.lock index 917a975..322ff2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -197,6 +197,13 @@ dependencies: chalk "*" +"@types/clean-css@*": + version "4.2.2" + resolved "https://registry.yarnpkg.com/@types/clean-css/-/clean-css-4.2.2.tgz#99fd79f6939c2b325938a1c569712e07dd97d709" + integrity sha512-xiTJn3bmDh1lA8c6iVJs4ZhHw+pcmxXlJQXOB6G1oULaak8rmarIeFKI4aTJ7849dEhaO612wgIualZfbxTJwA== + dependencies: + "@types/node" "*" + "@types/color-name@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" @@ -261,6 +268,15 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/html-minifier@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/html-minifier/-/html-minifier-4.0.0.tgz#2065cb9944f2d1b241146707c6935aa7b947d279" + integrity sha512-eFnGhrKmjWBlnSGNtunetE3UU2Tc/LUl92htFslSSTmpp9EKHQVcYQadCyYfnzUEFB5G/3wLWo/USQS/mEPKrA== + dependencies: + "@types/clean-css" "*" + "@types/relateurl" "*" + "@types/uglify-js" "*" + "@types/json-schema@^7.0.3": version "7.0.6" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" @@ -322,6 +338,11 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c" integrity sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA== +"@types/relateurl@*": + version "0.2.28" + resolved "https://registry.yarnpkg.com/@types/relateurl/-/relateurl-0.2.28.tgz#6bda7db8653fa62643f5ee69e9f69c11a392e3a6" + integrity sha1-a9p9uGU/piZD9e5p6facEaOS46Y= + "@types/serve-static@*": version "1.13.5" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.5.tgz#3d25d941a18415d3ab092def846e135a08bbcf53" @@ -330,6 +351,13 @@ "@types/express-serve-static-core" "*" "@types/mime" "*" +"@types/uglify-js@*": + version "3.9.3" + resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.9.3.tgz#d94ed608e295bc5424c9600e6b8565407b6b4b6b" + integrity sha512-KswB5C7Kwduwjj04Ykz+AjvPcfgv/37Za24O2EDzYNbwyzOo8+ydtvzUfZ5UMguiVu29Gx44l1A6VsPPcmYu9w== + dependencies: + source-map "^0.6.1" + "@types/yargs-parser@*": version "15.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" @@ -689,6 +717,14 @@ callsites@^3.0.0: resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== +camel-case@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73" + integrity sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M= + dependencies: + no-case "^2.2.0" + upper-case "^1.1.1" + camelcase@^5.0.0, camelcase@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" @@ -749,6 +785,13 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== +clean-css@^4.2.1: + version "4.2.3" + resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.3.tgz#507b5de7d97b48ee53d84adb0160ff6216380f78" + integrity sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA== + dependencies: + source-map "~0.6.0" + cli-boxes@^2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" @@ -808,6 +851,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +commander@^2.19.0: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -1703,11 +1751,29 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +he@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" + integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== + hosted-git-info@^2.1.4: version "2.8.8" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== +html-minifier@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/html-minifier/-/html-minifier-4.0.0.tgz#cca9aad8bce1175e02e17a8c33e46d8988889f56" + integrity sha512-aoGxanpFPLg7MkIl/DDFYtb0iWz7jMFGqFhvEDZga6/4QTjneiD8I/NXL1x5aaoCp7FSIT6h/OhykDdPsbtMig== + dependencies: + camel-case "^3.0.0" + clean-css "^4.2.1" + commander "^2.19.0" + he "^1.2.0" + param-case "^2.1.1" + relateurl "^0.2.7" + uglify-js "^3.5.1" + http-cache-semantics@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" @@ -2048,6 +2114,11 @@ lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== +lower-case@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac" + integrity sha1-miyr0bno4K6ZOkv31YdcOcQujqw= + lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" @@ -2203,6 +2274,13 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== +no-case@^2.2.0: + version "2.3.2" + resolved "https://registry.yarnpkg.com/no-case/-/no-case-2.3.2.tgz#60b813396be39b3f1288a4c1ed5d1e7d28b464ac" + integrity sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ== + dependencies: + lower-case "^1.1.1" + node-pre-gyp@^0.13.0: version "0.13.0" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz#df9ab7b68dd6498137717838e4f92a33fc9daa42" @@ -2464,6 +2542,13 @@ package-json@^6.3.0: registry-url "^5.0.0" semver "^6.2.0" +param-case@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/param-case/-/param-case-2.1.1.tgz#df94fd8cf6531ecf75e6bef9a0858fbc72be2247" + integrity sha1-35T9jPZTHs915r75oIWPvHK+Ikc= + dependencies: + no-case "^2.2.0" + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -2815,6 +2900,11 @@ registry-url@^5.0.0: dependencies: rc "^1.2.8" +relateurl@^0.2.7: + version "0.2.7" + resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" + integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -3015,7 +3105,7 @@ source-map-support@^0.5.17: buffer-from "^1.0.0" source-map "^0.6.0" -source-map@^0.6.0: +source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== @@ -3350,6 +3440,11 @@ typescript@^4.0.2: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2" integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ== +uglify-js@^3.5.1: + version "3.10.4" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.10.4.tgz#dd680f5687bc0d7a93b14a3482d16db6eba2bfbb" + integrity sha512-kBFT3U4Dcj4/pJ52vfjCSfyLyvG9VYYuGYPmrPvAxRw/i7xHiT4VvCev+uiEMcEEiu6UNB6KgWmGtSUYIWScbw== + unbzip2-stream@^1.3.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" @@ -3396,6 +3491,11 @@ update-notifier@^4.0.0: semver-diff "^3.1.1" xdg-basedir "^4.0.0" +upper-case@^1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/upper-case/-/upper-case-1.1.3.tgz#f6b4501c2ec4cdd26ba78be7222961de77621598" + integrity sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg= + uri-js@^4.2.2: version "4.4.0" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.0.tgz#aa714261de793e8a82347a7bcc9ce74e86f28602"