diff --git a/CHANGELOG.md b/CHANGELOG.md index 0228ed8..326db28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] +> [!NOTE] +> Breaking release. Every project will need a re-scaffold. + +### Added + +- Named Collections + - Support for named collections of assets that are managed together and can be individually previewed. + - Expiration trio: --expires-in, --expires-at, --expires-never. + - Collection selector utilities that can be run at the edge code to activate a collection. + - Default collection never expires. + +- git-style subcommands + - Actions separated into subcommands, such as `clean`, `publish-content`, and `collections`. + - Dry-run mode (--dry-run) for commands that mutates KV or disk. + +- KV Store + - Bytes of assets are stored in the Fastly KV Store. + - Items in the KV Store are keyed by the hash of the file, keeping storage efficient and deduplicated, even across collections. + - Metadata (file sizes, encodings, compression) for static assets is stored in KV Store Item metadata. + - Large-object chunking: files > 20 MB are split into segments behind the scenes and reassembled at read time. + - Upload process has been optimized - files are uploaded in parallel, and are only compressed and uploaded when necessary. + - Automatic retry with exponential back-off when the Fastly KV API rate-limits a burst of uploads. + - Fully supported in the local development environment during development. + - `--local` flag for all management commands. Passing this flag makes the command operate on the local KV Store instead of the Fastly KV Store. + +### Changed + +- Separate config files: + - `static-publisher.rc.js` now owns behavior that is common to the scaffolded Compute app + - publish-time settings are in `publish-content.config.js`. + +- Script names in the scaffold are now grouped by environment: `dev:publish`, `dev:start`, `fastly:publish`, `fastly:deploy`. + +- Asset inclusion test renamed to `kvStoreAssetInclusionTest` and now expects a boolean return value. + +### Removed / Deprecated + +- This tool drops `wasm-inline`, and no longer inlines bytes of assets into the Wasm binary. + +- Static files metadata is no longer stored in the Wasm binary, + +- This tool drops `moduleAssets`. + ## [6.3.0] - 2025-03-19 ### Added diff --git a/LICENSE b/LICENSE index cb8d11d..56ccc75 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Fastly, Inc. +Copyright (c) 2025 Fastly, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MIGRATING.md b/MIGRATING.md index 795c4d1..3b919b3 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -5,256 +5,6 @@ are generated during scaffolding. For this reason, it is recommended that you re This is straightforward if you're using `compute-js-static-publisher` out-of-the-box. Otherwise, read on. -## KV Store - -Starting with `v5.0.0`, this tool refers to the KV Store using its finalized product name, "KV Store". References in -code and in configuration that used the previous "Object Store" name have been changed to the new name. If you have -been using the feature, you can take the following steps: - -* In your `static-publish.rc.js` file: - * Rename the `objectStore` key to `kvStoreName`. For example, if your KV Store is named `'my-store'`, then change: - ``` - objectStore: 'my-store' - ``` - to - ``` - kvStoreName: 'my-store' - ``` - -* In your `fastly.toml` file, find all lines that pertain to object store entries. - * Such lines may look like this and there may be many: - ```toml - [[local_server.object_store.my-store]] - key = "4QEM5nIpyLiNF3wwsbnda5:/404.html_aeed29478691e636b4ded20c17e9eae437614617067a8751882368b965a21bcb" - path = "output/404.html" - ``` - * Change the `object_store` of these lines to `kv_stores`. For example: - ```toml - [[local_server.kv_stores.my-store]] - key = "4QEM5nIpyLiNF3wwsbnda5:/404.html_aeed29478691e636b4ded20c17e9eae437614617067a8751882368b965a21bcb" - path = "output/404.html" - ``` - -## Webpack - -Starting with `v4.0.0` of this tool, webpack is no longer required and is disabled by default for new applications. This can simplify development and result in shorter build times. - -You may still wish to use webpack if you need some of the features it provides, e.g., the ability to use loaders, asset modules, module replacement, dynamic imports, etc. - -To migrate away from using webpack, make the following changes in your `./compute-js` directory: - -* First, check your `webpack.config.js` file to make sure you aren't actually depending on any custom webpack features. When you're ready, continue to the next step. -* Delete `webpack.config.js`. -* Modify `static-publish.rc.js`: - * Change the line `module.exports = {` to `const config = {` - * At the end of the file, add `export default config;` -* In your `package.json` file: - * At the top level, add a `"type"` key if one doesn't already exist, with the value `"module"`. - * Under `devDependencies`, remove the `webpack` and `webpack-cli` entries. - * Under `scripts`, modify the `prebuild` script by removing the `&& webpack` at the end - of it. - * Under `scripts`, modify the `build` script by replacing the parameter `./bin/index.js` - with `./src/index.js`. - * In the end, the two scripts should look like this (along with any other scripts you may have): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static", - "build": "js-compute-runtime ./src/index.js ./bin/main.wasm" - } - ``` - -If you aren't moving away from webpack just yet, check that your `webpack.config.js` is up-to-date. Refer -to the [default `webpack.config.js` in this package](./resources/webpack.config.js) and add your changes, -or modify your configuration file using the following steps: - -* To make the resulting bundle easier to debug, it is recommended to set the `devtool` value to `false`. - -* The JavaScript SDK automatically adds the named condition `fastly` when resolving dependency packages. - To match the behavior when bundling with webpack, set `resolve.conditionsNames` to the following: - ``` - resolve: { - conditionNames: [ - 'fastly', - '...', - ], - ], - ``` - -* Starting `v3.0.0`, we depend on `v1.0.0` of the `js-compute` library, which provides namespaced exports for Fastly - features. To use them, you'll need to make the following changes to `webpack.config.js`: - - * Set the `target` value to `false`. - - * The `output` section should look like this: - ``` - output: { - filename: "index.js", - path: path.resolve(__dirname, "bin"), - chunkFormat: 'commonjs', - library: { - type: 'commonjs', - }, - }, - ``` - - * Add a new `externals` array to the bottom if it doesn't exist already. Add the following entry: - - ```javascript - module.exports = { - /* ... other config ... */ - externals: [ - /^fastly:.*$/, - ], - } - ``` - -* Starting `v3.0.0`, we no longer use webpack static assets to include the contents of static files, and instead [use the - `includeBytes` function](https://js-compute-reference-docs.edgecompute.app/docs/fastly:experimental/includeBytes) - to enable more performant loading, as well as a more size-efficient Wasm binary. As a result, the following code can - safely be removed from the `module.rules` array. - - ```javascript - { - // asset/source exports the source code of the asset. - resourceQuery: /staticText/, - type: "asset/source", - }, - { - // asset/inline exports the raw bytes of the asset. - // We base64 encode them here - resourceQuery: /staticBinary/, - type: "asset/inline", - generator: { - /** - * @param {Buffer} content - * @returns {string} - */ - dataUrl: content => { - return content.toString('base64'); - }, - } - }, - ``` - -If you need webpack for a new project you are scaffolding with this site, specify the `--webpack` command-line option -when you scaffold your application, e.g.: - -``` -npx @fastly/compute-js-static-publish@latest --webpack --root-dir=./public -``` - -## Removal of Expressly - -Previous versions of `@fastly/compute-js-static-publish` used [Expressly](https://expressly.edgecompute.app) to serve -assets. `v4` does away with this dependency and implements its own server in the `PublisherServer` -class. - -When using `v4`, you can remove the dependency on Expressly by deleting the `@fastly/expressly` entry from -`dependencies` or `devDependencies`, in your `package.json` file. - -If your application depended on Expressly for things like middleware, you will need to make further -changes. - -### The entry point `src/index.js` - -As of `v4`, the `src/index.js` entry point no longer uses Expressly, and looks like this: - -```js -/// -import { getServer } from './statics.js'; -const staticContentServer = getServer(); - -// eslint-disable-next-line no-restricted-globals -addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); -async function handleRequest(event) { - - const response = await staticContentServer.serveRequest(event.request); - if (response != null) { - return response; - } - - // Do custom things here! - // Handle API requests, serve non-static responses, etc. - - return new Response('Not found', { status: 404 }); -} -``` - -If you've previously made changes to `src/index.js`, you will need to make the equivalent changes in this new format. - -## `static-publish.rc.js` - -This configuration file has changed in v4, and you may find that some features have stopped working after -upgrading from v3. - -* In v3, the configuration object was typed `Config`. In v4, it is now typed with a more descriptive name, `StaticPublisherConfig`. - -```js -/** @type {import('@fastly/compute-js-static-publish').StaticPublisherConfig} */ -const config = { - rootDir: './public', - // ... and so on -}; -``` - -* A new key, `server`, was added to group configurations that pertain to Publisher Server. - -To migrate this file, you'll need to make the following changes: - -* `publicDir` - rename this to `rootDir`. All files under this root directory will be included by default in the publishing, - except for those that are excluded using some of the following features. -* `excludeDirs`, `includeDirs`, `excludeTest`, `moduleTest` - In v3, these were used in combination to determine whether - each file would be included in the publishing, and whether files would be included as modules. The interaction between - these four tests was not clearly defined, often having one option exclude files, only to have other options add them - back. In addition, in v3 it was not possible to have a module asset that was not also already a content asset. - In v4, these are more clearly defined. These four options should be rewritten in terms of - `excludeDirs`, `excludeDotFiles`, `includeWellKnown`, `contentAssetInclusionTest`, and `moduleAssetInclusionTest`. -* `staticDirs` - in v4, this was renamed to `staticItems` and moved under the new `server` key. -* `spa` - in v4, this was renamed to `spaFile` and moved under the new `server` key. -* `notFoundPage` - in v4, this was renamed to `notFoundPageFile` and moved under the new `server` key. -* `autoExt` - in v4, this was moved under the new `server` key. -* `autoIndex` - in v4, this was moved under the new `server` key. -* `contentTypes` - This is unchanged. - -See [static-publish.rc.js config file](./README.md#static-publish-rc) for a detailed explanation of each of these new values. - -* `.gitignore` - - Depending on the version of `compute-js-static-publisher` used to scaffold your application, your `.gitignore` file - may have been generated with different entries. Add any of the following entries that may be missing from your - `.gitignore` file: - - ```gitignore - /src/statics.js - /src/statics.d.ts - /src/statics-metadata.js - /src/statics-metadata.d.ts - /src/static-content - ``` - -* Build scripts - * Various versions of `@fastly/compute-js-static-publish` have specified different build scripts. We recommend the following setup, regardless of the version of `@fastly/compute-js-static-publish` or Fastly CLI. - - * The build script listed in `fastly.toml` of your `compute-js` directory should look like this: - ```toml - [scripts] - build = "npm run build" - ``` - - * If you're using webpack, then the `scripts` section of `package.json` of your `compute-js` directory should contain - the following items (along with any other scripts): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static && webpack", - "build": "js-compute-runtime ./bin/index.js ./bin/main.wasm" - } - ``` - - * If you're not using webpack, then the `scripts` section of `package.json` of your `compute-js` directory should - contain the following items (along with any other scripts): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static", - "build": "js-compute-runtime ./src/index.js ./bin/main.wasm" - } - ``` +> [!NOTE] +> This document is under construction. +> diff --git a/README.md b/README.md index 08bf715..f0eaaaf 100644 --- a/README.md +++ b/README.md @@ -1,631 +1,843 @@ # Static Publisher for JavaScript on Fastly Compute -Using a static site generator to build your website? Do you simply need to serve some static files? With `compute-js-static-publish`, now you can deploy and serve everything from Fastly's blazing-fast [Compute](https://developer.fastly.com/learning/compute/). +> [!NOTE] +> These docs are for v7, a major rewrite that adds powerful new features such as named collections. +> If you're looking for v6, please check out the [v6 branch](https://github.com/fastly/compute-js-static-publish/tree/v6). -## Prerequisites +> [!WARNING] +> Version 7 no longer supports module assets. If you require this feature, consider using [v6](https://github.com/fastly/compute-js-static-publish/tree/v6). -Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20 or newer. +## πŸ“– Table of Contents -## How it works +- [✨ Key Features](#-key-features) +- [🏁 Quick Start](#-quick-start) +- [βš™οΈ Configuring `static-publish.rc.js`](#️-configuring-static-publishrcjs) +- [🧾 Config for Publishing and Server: `publish-content.config.js`](#-config-for-publishing-and-server-publish-contentconfigjs) +- [πŸ“¦ Collections (Publish, Preview, Promote)](#-collections-publish-preview-promote) +- [🧹 Cleaning Up](#-cleaning-up) +- [πŸ”„ Content Compression](#-content-compression) +- [🧩 Using PublisherServer in Custom Apps](#-using-publisherserver-in-custom-apps) +- [πŸ“₯ Using Published Assets in Your Code](#-using-published-assets-in-your-code) +- [πŸ“š CLI Reference](#-cli-reference) +- [πŸ“• Appendix](#-appendix) +- [πŸ“š Next Steps](#-next-steps) -You have some HTML files, along with some accompanying CSS, JavaScript, image, and font files in a directory. Perhaps you've used a framework or static site generator to build these files. +Serve static websites and web apps at the edge — no backends and no CDN invalidation delays. -Assuming the root directory that contains your static files is `./public`, +`@fastly/compute-js-static-publish` helps you deploy static files to [Fastly Compute](https://developer.fastly.com/learning/compute/) using Fastly's KV Store for fast, cacheable, and content-addressed delivery. -### 1. Run `compute-js-static-publish` +## ✨ Key Features -```shell -npx @fastly/compute-js-static-publish@latest --root-dir=./public -``` +- πŸ“¦ Easy to scaffold and deploy +- πŸš€ Content stored in Fastly KV Store using hashed keys +- πŸ” Publish new content without deploying new Wasm binaries +- πŸ—‚ Organize releases into named collections which can be previewed (e.g. `live`, `staging`, `preview-123`) +- 🧼 Cleanup tools to remove expired or orphaned files +- βš™οΈ Configurable per-collection server configurations (e.g. fallback files) +- πŸ”’ Supports Brotli/gzip compression, conditional GET, and long cache TTLs -This will generate a Compute application at `./compute-js`. It will add a default `./compute-js/src/index.js` file that instantiates the [`PublisherServer`](#publisherserver) class and runs it to serve the static files from your project. +--- -> [!TIP] -> This process creates a `./compute-js/static-publish.rc.js` to hold your configuration. This, as well as the other files created in your new Compute program at `./compute-js`, can be committed to source control (except for the ones we specify in `.gitignore`!) +## 🏁 Quick Start + +### 1. Scaffold a Compute App + +Create a directory for your project, place your static files in `./public`, then type: + +```sh +npx @fastly/compute-js-static-publish \ + --root-dir=./public \ + --kv-store-name=site-content +``` + +You get a Compute app in `./compute-js` with: -> [!IMPORTANT] -> This step generates an application that includes your files and a program that serves them, and needs to be run only once. To make modifications to your application, simply make changes to your static files and rebuild it. Read the rest of this section for more details. +- `fastly.toml` (service config) +- `src/index.js` (entry point) +- `static-publish.rc.js` (app config) +- `publish-content.config.js` (publish-time / runtime behavior) -### 2. Test your application locally +### 2. Preview Locally -The `start` script builds and runs your application using [Fastly's local development server](https://developer.fastly.com/learning/compute/testing/#running-a-local-testing-server). +Type the following — no Fastly account or service required yet! -```shell -cd ./compute-js +```sh +cd compute-js npm install -npm run start +npm run dev:publish +npm run dev:start ``` -The build process scans your `./public` directory to generate files in the `./compute-js/static-publisher` directory. These files are packaged into your application's Wasm binary. +Fastly's [local development environment](https://www.fastly.com/documentation/guides/compute/testing/#running-a-local-testing-server) serves your static website at `http://127.0.0.1:7676`, powered by a simulated KV Store. -Once your application is running, your files will be served under `http://127.0.0.1:7676/` at the corresponding paths relative to the `./public` directory. For example, making a request to `http://127.0.0.1:7676/foo/bar.html` will attempt to serve the file at `./public/foo/bar.html`. +### 3. Deploy Your App -### 3. Make changes to your application +Ready to go live? All you need is a [free Fastly account](https://www.fastly.com/signup/?tier=free)! -Now, you're free to make changes to your static files. Add, modify, or remove files in the `./public` directory, and then re-build and re-run your application by typing `npm run start` again. +```sh +npm run fastly:deploy +``` -Each time you re-build the project, `compute-js-static-publish` will re-scan your `./public` directory and regenerate the files in the `./compute-js/static-publisher` directory. +The command publishes your Compute app and creates the KV Store. (No content uploaded yet!) -> [!TIP] -> You can make further customizations to the behavior of your application, such as specifying directories of your static files, specifying whether to use GZIP compression on your files, specifying custom MIME types of your files, and more. You can also run custom code alongside the default server behavior, or even access the contents of the files directly from custom code. See [Advanced Usages](#advanced-usages) below for details. Rebuilding will not modify the files in your `./compute-js/src` directory, so feel safe making customizations to your code. +### 4. Publish Content + +```sh +npm run fastly:publish +``` -### 4. When you're ready to go live, deploy your Compute service +Upload static files to the KV Store and applies the server config. Your website is now up and live! -The `deploy` script builds and [publishes your application to a Compute service in your Fastly account](https://developer.fastly.com/reference/cli/compute/publish/). +--- + +## πŸ—‚ Project Layout + +Here's what your project might look like after scaffolding: -```shell -npm run deploy +``` +my-project/ +β”œβ”€β”€ public/ # Your static site files (HTML, CSS, JS, etc.) +β”‚ β”œβ”€β”€ index.html +β”‚ β”œβ”€β”€ scripts.js +β”‚ β”œβ”€β”€ styles.css +β”‚ └── (... other static files ...) +└── compute-js/ # The scaffolded Compute application + β”œβ”€β”€ fastly.toml # Fastly service configuration + β”œβ”€β”€ package.json # Scaffolded package metadata + β”œβ”€β”€ package-lock.json # Dependency lockfile + β”œβ”€β”€ .gitignore # Ignores build artifacts by default + β”œβ”€β”€ static-publish.rc.js # App config + β”œβ”€β”€ publish-content.config.js # Publishing / runtime config + β”œβ”€β”€ static-publisher/ # ⚠️ Do not commit - generated content for local dev and publishing + β”‚ β”œβ”€β”€ kvstore.json # Simulates KV Store content for local preview + β”‚ └── kv-store-content/ # Preprocessed and compressed files for KV upload + └── src/ + └── index.js # Your Compute app entry point ``` -## Features +## βš™οΈ Configuring `static-publish.rc.js` -- Simple to set up, with a built-in server module. -- Or, make file contents directly available to your application, so you can write your own server. -- Content and metadata are available to your application, accessible by files' pre-package file paths. -- Brotli and Gzip compression. -- Support for `If-None-Match` and `If-Modified-Since` request headers. -- Optionally use webpack as a module bundler. -- Selectively serve files from Fastly's [KV Store](#kv-store), or embedded into your Wasm module. -- Supports loading JavaScript files as code into your Compute application. -- Presets for several static site generators. +This file defines your compute-js-static-publish application's settings. A copy of this is also baked into the Wasm binary and loaded when running your Compute app locally or on the edge. -Some of these features are new! If you wish to update to this version, you may need to re-scaffold your application, or follow the steps outlined in [MIGRATING.md](./MIGRATING.md). +### Example: `static-publish.rc.js` -## How does it work? Where are the files? +```js +const rc = { + kvStoreName: 'site-content', + defaultCollectionName: 'live', + publishId: 'default', + staticPublisherWorkingDir: './static-publisher', +}; + +export default rc; +``` -Once your application is scaffolded, `@fastly/compute-js-static-publish` integrates into your development process by -running as part of your build process. +### Fields: -The files you have configured to be included (`--root-dir`) are enumerated and prepared. Their contents are included into -your Wasm binary (or made available via [KV Store](#kv-store), if so configured). This process is called "publishing". +All fields are required. -Once the files are published, they are available to the other source files in the Compute application. For example, -the stock application runs the [PublisherServer](#publisherserver) class to serve these files. +- `kvStoreName` - The name of the KV Store used for publishing. +- `defaultCollectionName` - Collection to serve when none is specified. +- `publishId` - Unique prefix for all keys in the KV Store. Override only for advanced setups (e.g., multiple apps sharing the same KV Store). +- `staticPublisherWorkingDir` - Directory to hold working files during publish. -For more advanced uses, such as accessing the contents of these file in your application, see the -[Using the packaged objects in your own application](#using-published-assets-in-your-own-application) section below. +> [!NOTE] +> Changes to this file require rebuilding the Compute app, since a copy of it is baked into the Wasm binary. -Publishing is meant to run each time before building your Compute application into a Wasm file. -If the files in `--root-dir` have changed, then a new set of files will be published. +## 🧾 Config for Publishing and Server: `publish-content.config.js` -### Content Compression +This file is included as part of the scaffolding. Every time you publish content, the publish settings in this file are used for publishing the content, and the server settings are taken from this file and saved as the settings used by the server for that collection. -During publishing, this tool supports pre-compression of content. By default, your assets for select content types are -compressed using the Brotli and gzip algorithms, and then stored alongside the original files in your Wasm binary (or -[KV Store](#kv-store)). +```js +const config = { + // these paths are relative to compute-js dir + rootDir: '../public', + + // Include/exclude filters (optional): + excludeDirs: ['node_modules'], + excludeDotfiles: true, + includeWellKnown: true, + + // Advanced filtering (optional): + kvStoreAssetInclusionTest: (key, contentType) => { + return true; // include everything by default + }, + + // Override which compressed variants to create for each asset during publish (optional): + contentCompression: ['br', 'gzip'], + + // Content type definitions/overrides (optional): + contentTypes: [ + { test: /\.custom$/, contentType: 'application/x-custom', text: false }, + ], + + // Server settings + server: { + publicDir: './public', + spaFile: '/spa.html', + notFoundPageFile: '/404.html', + autoIndex: ['index.html'], + autoExt: ['.html'], + staticItems: ['/static/', '/assets/'], + allowedEncodings: ['br', 'gzip'], + } +}; -The content types that are compressed include any that are considered `text` types, as well as certain binary types that -expect to see a benefit of being compressed. +export default config; +``` -* Compressed text types: `.txt` `.html` `.xml` `.json` `.map` `.js` `.css` `.svg` -* Compressed binary types: `.bmp` `.tar` +You can override this file for a single `publish-content` command by specifying an alternative using `--config` on the command line. + +### Fields: + +- `rootDir` - Directory to scan for content, relative to this file (required). +- `excludeDirs` - Array of directory names or regex patterns to exclude (default: `['./node_modules']`). +- `excludeDotFiles` - Exclude dotfiles and dot-named directories (default: true). +- `includeWellKnown` - Always include `.well-known` even if dotfiles are excluded (default: true). +- `kvStoreAssetInclusionTest` - Function to determine inclusion and variant behavior per file. +- `contentCompression` - Array of compression formats to pre-generate (`['br', 'gzip']` by default). +- `contentTypes` - Additional or override content type definitions. + +- `server` - Server runtime config that contains the following fields: + - `publicDir` - The 'public' directory. The Publisher Server will + resolve requests relative to this directory (default: same value as 'root-dir'). + - `spaFile` - Path to a file to be used to serve in a SPA application. + - `notFoundPageFile` - Path to a file to be used to serve as a 404 not found page. + - `autoIndex` - List of files to automatically use as index. + - `autoExt` - List of extensions to automatically apply to a path and retry when + the requested path is not found. + - `staticItems` - Directories to specify as containing 'static' files. The + Publisher Server will serve files from these directories with a long TTL. + - `allowedEncodings` - Specifies which compression formats the server is allowed + to serve based on the client's `Accept-Encoding` header. + +## πŸ“¦ Collections (Publish, Preview, Promote) + +Collections are a powerful feature that allow you to publish and manage multiple versions of your site simultaneously. Each collection is a named set of: + +- Static files +- Server configuration (e.g., fallback file, static directories, etc.) +- An index file that maps paths to those hashes + +Collections are published using a `--collection-name`, and can be reused, updated, or deleted independently. For example, you can create a staging version of your site using: + +```sh +npx @fastly/compute-js-static-publish publish-content \ + --collection-name=staging \ + --expires-in=7d \ + --config=./publish-content.config.js +``` -To configure these content types, use the `contentTypes` field of the [`static-publish.rc.js` config file](#static-publish-rc). +You can overwrite or republish any collection at any time. Old file hashes will be reused automatically where contents match. -> [!IMPORTANT] -> By default, pre-compressed content assets are not generated when the KV Store is not used. -This is done to prevent the inclusion multiple of copies of each asset from making the Wasm binary too large. -If you want to pre-compress assets when not using KV Store, add a value for 'contentCompression' to your -`static-publish.rc.js` file. +### Expiration (Auto-cleanup) -## CLI options +Collections can expire automatically: -Except for `--root-dir`, most arguments are optional. +- Expired collections are ignored by the server and return 404s +- The default collection never expires +- Expiration limits can be modified (shortened, extended, reenstated) using `collections update-expiration` +- They are cleaned up by `clean --delete-expired-collections` -```shell -npx @fastly/compute-js-static-publish \ - --root-dir=./build \ - --public-dir=./build/public \ - --static-dir=./build/public/static \ - --output=./compute-js \ - --spa=./build/spa.html +```sh +--expires-in=3d # relative (e.g. 1h, 2d, 1w) +--expires-at=2025-05-01T12:00Z # absolute (ISO 8601) +--expires-never # the collection never expires ``` -If you provide options, they override the defaults described below. +*(Only one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified)* -Any configuration options will be written to a `static-publish.rc.js` file, and used each time you build your Compute -application. +### Switching the active collection -On subsequent builds of your Compute application, `compute-js-static-publish` will run with a special flag, `build-static`, -reading from stored configuration, then scanning the `--public-dir` directory to recreate `./compute-js/static-publisher/statics.js`. +By default, the server app serves assets from the "default collection", named in `static-publish.rc.js` under `defaultCollectionName`. To switch the active collection, you add custom code to your Compute app that calls `publisherServer.setActiveCollectionName(name)`: -Any relative file and directory paths passed at the command line are handled as relative to the current directory. +```js +publisherServer.setActiveCollectionName("preview-42"); +``` -### Publishing options: +This only affects the current request (in Compute, requests do not share state). -| Option | Default | Description | -|-----------------------------|------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--preset` | (None) | Apply default options from a specified preset. See ["Frameworks and Static Site Generators"](#usage-with-frameworks-and-static-site-generators). | -| `--output` | `./compute-js` | The directory in which to create the Compute application. | -| `--static-content-root-dir` | (output directory) + `/static-publisher` | The directory under the Compute application where static asset and metadata are written. | -| `--root-dir` | (None) | **Required**. The root of the directory that contains the files to include in the publishing. All files you wish to include must reside under this root. | +#### Example: Subdomain-based Routing -### Server options: +In the following example, assume that the Compute application is hosted using a wildcard domain `*.example.com`. A request for `preview-pr-123.example.com` would activate the collection `'pr-123'`. -Used to populate the `server` key under `static-publish.rc.js`. +```js +import { PublisherServer, collectionSelector } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; -| Option | Default | Description | -|--------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--public-dir` | | The directory that contains your website's public files. | -| `--static-dir` | (None) | Any directories under `--public-dir` that contain the website's static assets that will be served with a very long TTL. You can specify as many such directories as you wish, by listing multiple items. | -| `--auto-ext` | `.html,.htm` | Specify automatic file extensions. | -| `--auto-index` | `index.html,index.htm` | Specify filenames for automatically serving an index file. | -| `--spa` | (None) | Path to a fallback file for SPA applications. | -| `--not-found-page` | `/404.html` | Path to a fallback file for 404 Not Found. | +const publisherServer = PublisherServer.fromStaticPublishRc(rc); -See [PublisherServer](#publisherserver) for more information about these features. +addEventListener("fetch", event => { + const request = event.request; + const collectionName = collectionSelector.fromHostDomain(request, /^preview-([^\.]*)\./); + if (collectionName != null) { + publisherServer.setActiveCollectionName(collectionName); + } -For backwards compatibility, if you do not specify a `--root-dir` but you have provided a `--public-dir`, then that value is used for `--root-dir`. + event.respondWith(publisherServer.serveRequest(request)); +}); +``` -Note that the files referenced by `--spa` and `--not-found-page` do not necessarily have to reside inside `--public-dir`. +### πŸ”€ Selecting a Collection at Runtime -### Fastly service options +The `collectionSelector` module provides helpers to extract a collection name from different parts of a request: -These arguments are used to populate the `fastly.toml` and `package.json` files of your Compute application. +```js +collectionSelector.fromHostDomain(request, /^preview-([^\.]*)\./); +``` + +#### From the Request URL -| Option | Default | Description | -|-------------------|--------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--name` | `name` from `package.json`, or `compute-js-static-site` | The name of your Compute application. | -| `--description` | `description` from `package.json`, or `Compute static site` | The description of your Compute application. | -| `--author` | `author` from `package.json`, or `you@example.com` | The author of your Compute application. | -| `--service-id` | (None) | The ID of an existing Fastly WASM service for your Compute application. | -| `--kv-store-name` | (None) | The name of an existing [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) to hold the content assets. In addition, a Resource Link to this KV Store must already exist on the service specified by `--service-id` and have the same name as the KV Store. | +```js +collectionSelector.fromRequestUrl(request, url => url.pathname.split('/')[2]); +``` -## Usage with frameworks and static site generators +#### With a Custom Request Matcher -`compute-js-static-publish` supports preset defaults for a number of frameworks and static site generators: +```js +collectionSelector.fromRequest(request, req => req.headers.get('x-collection') ?? 'live'); +``` -| `--preset` | `--root-dir` | `--static-dir` | Notes | -|-------------------------------|--------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------| -| `cra` (or `create-react-app`) | `./build` | `./build/static` | For apps written using [Create React App](https://create-react-app.dev). Checks for a dependency on `react-scripts`. | -| `cra-eject` | `./build` | `./build/static` | For apps written using Create React App, but which have since been ejected via `npm run eject`. Does not check for `react-scripts`. | -| `vite` | `./dist` | (None) | For apps written using [Vite](https://vitejs.dev). | -| `sveltekit` | `./dist` | (None) | For apps written using [SvelteKit](https://kit.svelte.dev). | -| `vue` | `./dist` | (None) | For apps written using [Vue](https://vuejs.org), and that were created using [create-vue](https://github.com/vuejs/create-vue). | -| `next` | `./out` | (None) | For apps written using [Next.js](https://nextjs.org), using `npm run export`. *1 | -| `astro` | `./dist` | (None) | For apps written using [Astro](https://astro.build) (static apps only). *2 | -| `gatsby` | `./public` | (None) | For apps written using [Gatsby](https://www.gatsbyjs.com). | -| `docusaurus` | `./build` | (None) | For apps written using [Docusaurus](https://docusaurus.io) | +#### From a Cookie -You may still override any of these options individually. +See [fromCookie](#fromcookie) for details on this feature. -*1 - For Next.js, consider using `@fastly/next-compute-js`, a Next.js server implementation that allows you to run - your Next.js application on Compute. +```js +const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); +``` -*2 - Astro support does not support SSR. +#### From a Fastly Config Store -## PublisherServer +```js +collectionSelector.fromConfigStore('my-config-store', 'collection-key'); +``` -`PublisherServer` is a simple yet powerful server that can be used out of the box to serve the files prepared by this tool. +### πŸš€ Promoting a Collection -This server handles the following automatically: +If you're happy with a preview or staging collection and want to make it live, use the `collections promote` command: -* Maps the path of your request to a path under `--public-dir` and serves the content of the asset -* Sources the content from the content packaged in the Wasm binary, or from the [KV Store](#kv-store), if configured. -* Applies long-lived cache headers to files served from `--static-dir` directories. Files under these directories will be cached by the browser for 1 year. (Use versioned or hashed filenames to avoid serving stale assets.) -* Performs Brotli and gzip compression as requested by the `Accept-Encoding` headers. -* Provides `Last-Modified` and `ETag` response headers, and uses them with `If-Modified-Since` and `If-None-Match` request headers to produce `304 Not Modified` responses. -* If an exact match is not found for the request path, applies automatic extensions (e.g., `.html`) and automatic index files (e.g., `index.html`). -* Can be configured to serve a fallback file for SPA apps. Useful for apps that use [client-side routing](https://create-react-app.dev/docs/deployment#serving-apps-with-client-side-routing). -* Can be configured to serve a 404 not found file. -* Returns `null` if nothing matches, so that you can add your own handling if necessary. +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=staging + --to=live +``` -During initial scaffolding, the configuration based on the command-line parameters and preset are written to your `./static-publisher.rc.js` file under the `server` key. +This copies all content and server settings from the `staging` collection to `live`. -### Configuring PublisherServer +You can also specify a new expiration: -You can further configure the server by making modifications to the `server` key under `./static-publisher.rc.js`. +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=preview-42 \ + --to=staging \ + --expires-in=7d +``` -| Key | Default | Description | -|--------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `publicDirPrefix` | `''` | Prefix to apply to web requests. Effectively, a directory within `rootDir` that is used by the web server to determine the asset to respond with. | -| `staticItems` | `[]` | A test to apply to item names to decide whether to serve them as "static" files, in other words, with a long TTL. These are used for files that are not expected to change. They can be provided as a string or array of strings. | -| `compression` | `[ 'br', 'gzip' ]` | If the request contains an `Accept-Encoding` header, they are checked for the values listed here. The compression algorithm that produces the smallest transfer size is applied. | -| `autoExt` | `[]` | When a file is not found, and it doesn't end in a slash, then try auto-ext: we try to serve a file with the same name post-fixed with the specified strings, tested in the order listed. These are tested before auto-index, if any. | -| `autoIndex` | `[]` | When a file is not found, then try auto-index: we treat it as a directory, then try to serve a file that has the specified strings, tested in the order listed. | -| `spaFile` | `null` | Asset key of a content item to serve with a status code of `200` when a GET request comes arrives for an unknown asset, and the Accept header includes text/html. | -| `notFoundPageFile` | `null` | Asset key of a content item to serve with a status code of `404` when a GET request comes arrives for an unknown asset, and the Accept header includes text/html. | +> [!NOTE] +> The collection denoted by `defaultCollectionName` is exempt from expiration. -For `staticItems`: -* Items that contain asterisks are interpreted as glob patterns (for example, `/public/static/**/*.js`) -* Items that end with a trailing slash are interpreted as a directory name. -* Items that don't contain asterisks and that do not end in slash are checked for exact match. +## πŸ›  Development β†’ Production Workflow -For `compression`, the following values are allowed: -* `'br'` - Brotli -* `'gzip'` - Gzip +### Local development -## Associating your project with a Fastly Service +During development, the local preview server (`npm run dev:start`) will run against assets loaded into the simulated KV Store provided by the local development environment. -The project created by this tool is a Fastly Compute JavaScript application, complete with a `fastly.toml` file that -describes your project to the Fastly CLI. +Prior to starting the server, publish the content to the simulated KV Store: -To deploy your project to production, you deploy it to a [Fastly service](https://developer.fastly.com/reference/glossary#term-service) -in your account. Usually, you create your service automatically as part of your first deployment of the project. +```sh +npm run dev:publish # 'publish' your files to the simulated local KV Store +npm run dev:start # preview locally +``` -In this case, `fastly.toml` has no value for `service_id` at the time you deploy, so the Fastly CLI will prompt -you to create a Fastly service in your account, after which it will save the new service's ID to your `fastly.toml` file. +This simulates publishing by writing to `kvstore.json` instead of uploading to the actual KV Store. You can preview your site at `http://127.0.0.1:7676` - no Fastly account or service required. -Alternatively, you may deploy to a service that already exists. You can create this service using the -[Fastly CLI](https://developer.fastly.com/reference/cli/service/create/) or the [Fastly web app](https://manage.fastly.com/). -Note that since this is a Compute application, the service must be created as a Wasm service. +Note that for local development, you will have to stop and restart the local development server each time you publish updates to your content. -Before deploying your application, specify the service by setting the `service_id` value in the `fastly.toml` file to the -ID of the service. The Fastly CLI will deploy to the service identified by this value. +To publish to an alternative collection name, use: -To specify the service at the time you are scaffolding the project (for example, if you are running this tool and deploying -as part of a CI process), specify the `--service-id` command line argument to populate `fastly.toml` with this value. +```sh +npm run dev:publish -- --collection-name=preview-123 +``` -## Using the KV Store (BETA) +### Production -
+When you're ready for production: -Starting with v4, it's now possible to upload assets to and serve them from a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores). +1. [Create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already. +2. Run `npm run fastly:deploy` + - This builds your Compute app into a Wasm binary + - Deploys it to a new or existing Fastly Compute service + - If creating a new service: + - you'll be prompted for backend info - **you can skip this**, as no backend is needed (all content is served from KV) + - KV Store will be created if necessary and automatically linked to your new service. -When this mode is enabled, you build your application as normal, and as a step during the build, your files -are uploaded to the Fastly KV Store, and metadata in the application is marked to source them from there instead -of from bytes in the Wasm binary. +Once deployed, publish content like so: -You can enable the use of KV Store with `@fastly/compute-js-static-publish` as you scaffold your application, or -at any later time. +```sh +npm run fastly:publish +``` -At the time you enable the use of KV Store: +This: -* Your Fastly service must already exist. See [Associating your project with a Fastly Service](#associating-your-project-with-a-fastly-service) above. +- Uses the default collection name +- Uploads static files to the KV Store +- Stores server configuration for the collection -* Your [KV Store](https://www.fastly.com/documentation/guides/concepts/edge-state/data-stores/#kv-stores) must already - exist under the same Fastly account, and be linked to the service. The - [Resource Link](https://www.fastly.com/documentation/guides/concepts/resources/) must have the same name as the - KV Store. +> [!TIP] +> Upload to a specific collection by specifying the collection name when publishing content: +> ```sh +> npm run fastly:publish -- --collection-name=preview-42 +> ``` -To create your KV Store and link it to the service, follow these steps: +**No Wasm redeploy needed** unless you: -```shell -# Create a KV Store -$ npx fastly kv-store create --name=example-store -SUCCESS: Created KV Store 'example-store' (onvbnv9ntdvlnldtn1qwnb) -``` +- Modify `src/index.js` - such as when you update your custom routing logic (e.g. collection selection) or +- Change `static-publish.rc.js` -Make a note of the KV Store ID in this response (`onvbnv9ntdvlnldtn1qwnb` in the above example). Next, use this ID -value to create a Resource Link between your service and the KV Store. You **MUST** use the same name for your Resource -Link as you use for your KV Store. After linking the KV Store, activate the new version of the service. +If you do need to rebuild and redeploy the Compute app, simply run: -```shell -# Link the KV Store to a service provide the KV Store ID as resource-id -$ npx fastly resource-link create --version=latest --autoclone --resource-id=onvbnv9ntdvlnldtn1qwnb --name=example-store -$ npx fastly service-version activate --version=latest +```sh +npm run dev:deploy ``` -Once the KV Store is created and linked to your service, add the name of the KV Store (`example-store` in -γ€€the above example) name to your `static-publish.rc.js` file under the `kvStoreName` key. +## 🧹 Cleaning Up -To specify the KV Store at the time you are scaffolding the project (for example, if you are running this tool and -deploying as part of a CI process), specify the `--service-id` and `--kv-store-name` command line arguments to populate -the respective files with these values. +Every time you publish, old files are left behind for safety. **However, files with the same content will be re-used across collections and publishing events.** They are only stored once in the KV Store using their content hash as a key. This ensures that unchanged files aren't duplicated, keeping storage efficient and deduplicated. -After you have performed the above steps, go ahead and build your application as normal. -As a new step during the build process, the tool will send these files to the KV Store. +Over time, however, collections may expire, old versions of files will be left behind, and some assets in the KV Store will no longer be referenced by any live collection. To avoid bloat, use: -> [!IMPORTANT] -> This step writes to your KV Store. When building your application, you must set the environment variable `FASTLY_API_TOKEN` to a Fastly API token that has access to write to this KV Store. -> -> Alternatively, if this environment variable is not found, the tool will attempt to detect an API token by calling `fastly profile token`. +```sh +npm run dev:clean +``` +and +```sh +npm run fastly:clean +``` -> [!TIP] -> By running `npx @fastly/cli compute build --verbose` (or `npm run build` directly), you should see output in your logs saying that files are being sent to the KV Store. +These scripts run against the local and Fastly KV Stores respectively, and run the following command: +``` +npx @fastly/compute-js-static-publish clean --delete-expired-collections +``` -The `statics-metadata.js` file should now show `"type": "kv-store"` for content assets. -Your Wasm binary should also be smaller, as the content of the files are no longer inlined in the build artifact. -You can deploy this and run it from Fastly, and the referenced files will be served from KV Store. +This removes: -You will also see entries in `fastly.toml` that represent the local KV Store. -These enable the site to also run correctly when served using the local development environment. +- Expired collection index files (only if `--delete-expired-collections` is passed) +- Unused content blobs (no longer referenced) +- Orphaned server config files -### Cleaning unused items from KV Store +### πŸ” Dry Run Mode -The files that are uploaded to the KV Store are submitted using keys of the following format: +Preview what will be deleted without making changes: -`:__` +```sh +npx @fastly/compute-js-static-publish clean --dry-run +``` -For example: -`12345abcde67890ABCDE00:/public/index.html_br_aeed29478691e67f6d5s36b4ded20c17e9eae437614617067a8751882368b965` +> ⚠️ Cleanup never deletes the default collection and never deletes content that’s still in use. -Using such a key ensures that whenever the file contents are identical, the same key will be generated. -This enables to detect whether an unchanged file already exists in the KV Store, avoiding having to re-submit -files that have not changed. If the file contents have changed, then a new hash is generated. This ensures that -even during the brief amount of time between deploys, any request served by a prior version will still serve the same -corresponding previous version of the content. - -However, this system never deletes files automatically. After many deployments, extraneous files may be left over. - -`@fastly/compute-js-static-publish` includes a feature to delete these old versions of the files that are no longer being -used. To run it, type the following command: - -`npx @fastly/compute-js-static-publish --clean-kv-store` +## πŸ”„ Content Compression -It works by scanning `statics-metadata.js` for all currently-used keys. Then it enumerates all the existing -keys in the configured KV Store and that belong to this application (can do so by narrowing down all keys to the ones -that begin with the "publish id"). If any of the keys is not in the list of currently-used keys, then a request is made -to delete that KV Store value. +This project supports pre-compressing and serving assets in Brotli and Gzip formats. Compression is controlled at two different stages: -And that's it! It should be possible to run this task to clean up once in a while. +- **During publishing**, the `contentCompression` field in the `publish` section of `publish-content.config.js` defines which compressed variants (e.g., `br`, `gzip`) should be generated and uploaded to the KV Store. -## Advanced Usages +Assets are stored in multiple formats (uncompressed + compressed) if configured. The following file types are compressed by default: -### The `static-publish.rc.js` config file +- Text-based: `.html`, `.js`, `.css`, `.svg`, `.json`, `.txt`, `.xml`, `.map` +- Certain binary formats: `.bmp`, `.tar` -* `rootDir` - All files under this root directory will be included by default in the publishing, - except for those that are excluded using some of the following features. Files outside this root cannot be - included in the publishing. +- **At runtime**, the `allowedEncodings` field in the `server` section of `publish-content.config.js` specifies which compression formats the server is allowed to serve based on the client's `Accept-Encoding` header. -* `staticContentRootDir` - Static asset loader and metadata files are created under this directory. - For legacy compatibility, if not provided, defaults to `'./src'`. +`PublisherServer` will serve the smallest appropriate version based on the `Accept-Encoding` header. -* `kvStoreName` - Set this value to the _name_ of an existing KV Store to enable uploading of content assets - to Fastly KV Store. See [Using the KV Store](#kv-store) for more information. +## 🧩 Using PublisherServer in Custom Apps -* `excludeDirs` - Specifies names of files and directories within `rootDir` to exclude from the publishing. Each entry can - be a string or a JavaScript `RegExp` object. Every file and directory under `rootDir` is checked against each entry of - the array by testing its path relative to `rootDir`. The file or directory (included all children) and excluded if the - condition matches: - * If a string is specified, then an exact match is checked. - * If a `RegExp` is specified, then it is tested with the regular expression. - * If this setting is not set, then the default value is `['./node_modules']`. - * If you specifically set this to the empty array, then no files are excluded by this mechanism. +You can combine PublisherServer with custom logic to support APIs, authentication, redirects, or A/B testing. `PublisherServer` returns `null` when it cannot handle a request, allowing you to chain in your own logic. -* `excludeDotfiles` - Unless disabled, will exclude all files and directories (and their children) - whose names begin with a `'.''`. This is `true` by default. +```js +import { PublisherServer } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; -* `includeWellKnown` - Unless disabled, will include a file or directory called `.well-known` - even if `excludeDotfiles` would normally exclude it. This is `true` by default. +const publisherServer = PublisherServer.fromStaticPublishRc(rc); -* `contentAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during - the publishing, to determine whether to include the asset as a content asset. For every file, this function is passed - the [asset key](#asset-keys), as well as its content type (MIME type string). You may return one of three values from - this function: - * Boolean `true` - Include the file as a content asset in this publishing. Upload the file to and serve it from the - KV Store if KV Store mode is enabled, or include the contents of the file in the Wasm binary if KV Store - mode is not enabled. - * String `"inline"` - Include the file as a content asset in this publishing. Include the contents of the file in the - Wasm binary, regardless of whether KV Store mode is enabled. - * Boolean `false` - Do not include this file as a content asset in this publishing. +addEventListener("fetch", event => { + event.respondWith(handleRequest(event.request)); +}); - If you do not provide a function, then every file will be included in this publishing as a content asset, and their - contents will be uploaded to and served from the KV Store if KV Store mode is enabled, or included in the Wasm - binary if KV Store mode is not enabled. +async function handleRequest(request) { + const response = await publisherServer.serveRequest(request); + if (response) { + return response; + } -* `contentCompression` - During the publishing, the tool will pre-generate compressed versions of content assets in these - formats and make them available to the Publisher Server or your application. Default value is [ 'br' | 'gzip' ] if - KV Store is enabled, or [] if KV Store is not enabled. + // Add your custom logic here + if (request.url.endsWith('/api/hello')) { + return new Response(JSON.stringify({ greeting: "hi" }), { + headers: { 'content-type': 'application/json' } + }); + } -* `moduleAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during - the publishing, to determine whether to include the asset as a module asset. For every file, this function is passed - the [asset key](#asset-keys), as well as its content type (MIME type string). You may return one of three values from this function: - * `true` (boolean) - Include the file as a module asset in this publishing. - * `"static-import"` (string) - Include the file as a module asset in this publishing, and statically import it. This causes - any top-level code in these modules to run at application initialization time. - * `false` (boolean) - Do not include this file as a module asset in this publishing. + return new Response("Not Found", { status: 404 }); +} +``` - If you do not provide a function, then no module assets will be included in this publishing. +## πŸ“₯ Using Published Assets in Your Code -* `contentTypes` - Provide custom content types and/or override them. +To access files you've published, use the `getMatchingAsset()` and `loadKvAssetVariant()` methods on `publisherServer`. - This tool comes with a [default set of content types](./src/util/content-types.ts) defined for many common - file extensions. This list can be used to add to and/or override items in the default list. - Content type definitions are checked in the provided order, and if none of them match, the default content types are - tested afterward. +### Access Metadata for a File: - Provide these as an array of content type definition objects, each with the following keys and values: - * `test` - a RegExp or function to perform on the asset key. If the test succeeds, then the content asset is considered - to be of this content type definition. - * `contentType` - The content type header to apply when serving an asset of this content type definition. - * `text` - If `true`, this content type definition is considered to contain textual data. This makes `.text()` and `.json()` - available for calling on store entries. If not specified, this is treated as `false`. - * `precompressAsset` - When `true`, this tool generates pre-compressed versions of content assets and serves them to user - agents that assert an appropriate `Accept` header. See [Content Compression*](#content-compression) for details. If - not specified, this is `true` if `text` is `true`, and `false` if `text` is not `true`. - - For example, to add a custom content type `application/x-custom` for files that have a `.custom` extension, not treat - it as a text file, but precompress it during the generation of the application, add the following to your - `static-publish.rc.js` file: - - ```javascript - const config = { - /* ... other config ... */ - contentTypes: [ - { test: /\.custom$/, contentType: 'application/x-custom', text: false, precompressAsset: true }, - ], - }; - ``` +```js +const asset = await publisherServer.getMatchingAsset('/index.html'); +if (asset != null) { + // Asset exists with that path + asset.contentType; // e.g., 'text/html' + asset.lastModifiedTime; // Unix timestamp + asset.size; // Size in bytes of the base version + asset.hash; // SHA256 hash of base version + asset.variants; // Available variants (e.g., ['gzip']) +} +``` - > Note that content types are tested at publishing time, not at runtime. - -* `server` - [Configuration of `PublisherServer()`](#configuring-publisherserver). - above. - -### Running custom code alongside Publisher Server - -The generated `./src/index.js` program instantiates the server and simply asks it to respond to a request. - -You are free to add code to this file. - -For example, if the `PublisherServer` is unable to formulate a response to the request, then it returns `null`. You may -add your own code to handle these cases, such as to provide custom responses. +### Load the File from KV Store: ```js -import { getServer } from './statics.js'; -const staticContentServer = getServer(); +const kvAssetVariant = await publisherServer.loadKvAssetVariant(asset, null); // pass 'gzip' or 'br' for compressed -addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); -async function handleRequest(event) { +kvAssetVariant.kvStoreEntry; // KV Store entry (type KVStoreEntry defined in 'fastly:kv-store') +kvAssetVariant.size; // Size of the variant +kvAssetVariant.hash; // SHA256 of the variant +kvAssetVariant.contentEncoding; // 'gzip', 'br', or null +kvAssetVariant.numChunks; // Number of chunks (for large files) +``` - const response = await staticContentServer.serveRequest(event.request); - if (response != null) { - return response; - } - - // Do custom things here! - // Handle API requests, serve non-static responses, etc. +You can stream `kvAssetVariant.kvStoreEntry.body` directly to a `Response`, or read it using `.text()`, `.json()`, or `.arrayBuffer()` depending on its content type. - return new Response('Not found', { status: 404 }); -} -``` +--- -### Using published assets in your own application +## πŸ“š CLI Reference -Publishing, as described earlier, is the process of preparing files for inclusion into your application. -This process also makes metadata available about each of the files that are included, such as its content type, the last -modified date, the file hash, and so on. +### Available Commands -The [`PublisherServer` class](#publisherserver) used by the default scaffolded application is a simple application of this content -and metadata. By importing `./statics.js` into your Compute application, you can just as easily access this -information about the assets that were included during publishing. +#### Outside a Compute App Directory +- `npx @fastly/compute-js-static-publish [options]` - Scaffold a new Compute app -> IMPORTANT: Use a static `import` statement, rather than using `await import()` to load `./statics.js`, in order to -ensure that its top-level code runs during the initialization phase of your Compute application. +#### Inside a Compute App Directory +- `publish-content` - Publish static files to the KV Store under a named collection +- `clean` - Delete expired and unreferenced KV entries +- `collections list` - List all published collections +- `collections delete` - Delete a specific collection index +- `collections promote` - Copy a collection to another name +- `collections update-expiration` - Modify expiration time for an existing collection -#### Assets +### πŸ›  App Scaffolding -There are two categories of assets: Content Assets and Module Assets. +Run outside an existing Compute app directory: + +```sh +npx @fastly/compute-js-static-publish \ + --root-dir=./public \ + --kv-store-name=site-content \ + [--output=./compute-js] \ + [--static-publisher-working-dir=/static-publisher] \ + [--publish-id=] \ + [--public-dir=./public] \ + [--static-dir=./public/static] \ + [--spa=./public/spa.html] \ + [--not-found-page=./public/404.html] \ + [--auto-index=index.html,index.htm] \ + [--auto-ext=.html,.htm] \ + [--name=my-site] \ + [--description='My Compute static site'] \ + [--author='you@example.com'] \ + [--service-id=your-fastly-service-id] +``` -* A Content Asset is a type of asset where your application or a user of your application is interested in the text or -binary contents of an asset. +#### Options: + +**Used to generate the Compute app:** +- `--kv-store-name`: Required. Name of KV Store to use. +- `--output`: Compute app destination (default: `./compute-js`) +- `--static-publisher-working-dir`: Directory to hold working files (default: `/static-publisher`). +- `--name`: Application name to insert into `fastly.toml` +- `--description`: Application description to insert into `fastly.toml` +- `--author`: Author to insert into `fastly.toml` +- `--service-id`: Optional existing Fastly Compute service ID + +**Used in building config files:** +- `--root-dir`: Required. Directory of static site content. +- `--publish-id`: Optional key prefix for KV entries (default: `'default'`). +- `--public-dir`: Public files base directory (default: same as `--root-dir`). +- `--static-dir`: One or more directories to serve with long cache TTLs. +- `--spa`: SPA fallback file path (e.g., `./public/spa.html`). +- `--not-found-page`: 404 fallback file path (e.g., `./public/404.html`). +- `--auto-index`: List of filenames to use as index files. +- `--auto-ext`: Extensions to try when resolving URLs. + +### πŸš€ Inside a Compute App Directory + +Once you're in the scaffolded Compute app directory (with `static-publish.rc.js` present), you can run these subcommands: + +#### `publish-content` + +```sh +npx @fastly/compute-js-static-publish publish-content \ + [--root-dir=./public] \ + [--collection-name=preview-42] \ + [--config=./publish-content.config.js] \ + [--expires-in=7d | --expires-at=2025-05-01T12:00Z | --expires-never] \ + [--local] \ + [--fastly-api-token=...] +``` - The data of each content asset can exist in one of two stores: - * Inline Store - this is a data store that exists within the Wasm binary. - * Fastly KV Store - Fastly's distributed edge data store. Data can be placed here without impacting the size of - your Wasm binary. +Publishes static files from your local root directory into a named collection, either in the Fastly KV Store (default) or to a local dev directory (`--local`). Files that already exist with the same hash are skipped automatically. - Your application can stream the contents of these assets to a visitor, or read from the stream itself and access its - contents. +After this process is complete, the PublisherServer object in the Compute application will see the updated index of files and updated server settings from the `publish-content.config.js` file. -* A Module Asset is a type of asset where your application wants to load the asset as a module, and use it as part of its -running code. Their contents are actually built at publishing time and their built representation is included in the Wasm -binary. They can be imported statically at will, and your application is able to execute the code exported by these modules. +##### Options: -##### Asset Keys +- `--collection-name`: Name of the collection to create/update (default: value in `static-publish.rc.js`) +- `--config`: Path to a config file to configure server behavior for this collection (default: `./publish-content.config.js`) +- `--root-dir`: Source directory to read files from (overrides value in `publish-content.config.js`) +- `--kv-overwrite`: Cannot be used with `--local`. When using Fastly KV Store, always overwrites existing entries, even if unchanged. -When working with content assets or module assets from your application, they are referenced by their asset key, which -is the relative path of the file from `rootDir`, including the leading slash. +**Expiration:** -#### Content Assets +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format: `2025-05-01T12:00Z`) +- `--expires-never`: Collection never expires -You can obtain the content assets included in publishing by importing the `contentAssets` object exported from -`./statics.js`. +*At most one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified* -```js -import { contentAssets } from './statics'; +**Global Options:** -// Obtain a content asset named '/public/index.html' -const asset = contentAssets.getAsset('/public/index.html'); +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. -// 'wasm-inline' if object's data is 'inlined' into Wasm binary -// 'kv-store' if object's data exists in Fastly KV Store -asset.type; +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile -// Get the "store entry" -const storeEntry = await asset.getStoreEntry(); +#### `clean` -storeEntry.contentEncoding; // null, 'br', 'gzip' -storeEntry.hash; // SHA256 of the contents of the file -storeEntry.size; // Size of file in bytes +```sh +npx @fastly/compute-js-static-publish clean \ + [--delete-expired-collections] \ + [--dry-run] ``` -Regardless of which store these objects come from, they implement the `Body` interface as defined by `@fastly/js-compute`. -As such, you are able to work with them in the same way to obtain their contents: +Cleans up expired or unreferenced items in the Fastly KV Store. +This can include expired collection indexes and orphaned content assets. -```js -storeEntry.body; // ReadableStream -storeEntry.bodyUsed; // true if consumed or distrubed +##### Options: + +- `--delete-expired-collections`: If set, expired collection index files will be deleted. +- `--dry-run`: Show what would be deleted without performing any deletions. + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile -// Get the data as ArrayBuffer, parsed JSON, or string -// The latter two are only available if the data is a text type -const arrayBuffer = await storeEntry.arrayBuffer(); -const json = await storeEntry.json(); -const text = await storeEntry.text(); +#### `collections list` + +```sh +npx @fastly/compute-js-static-publish collections list ``` +Lists all collections currently published in the KV Store. -Or, if you don't care about the contents but just want to stream it to the visitor, you can pass the `.body` field directly -to the Response constructor: +##### Options: -```js -const response = new Response(storeEntry.body, { status: 200 }); +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile + +#### `collections promote` + +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=preview-42 \ + --to=live \ + [--expires-in=7d | --expires-at=2025-06-01T00:00Z | --expires-never] ``` +Copies an existing collection (content + config) to a new collection name. -> IMPORTANT: Once a store entry is consumed, its body cannot be read from again. If you need to access the contents of the -same asset more than once, you may obtain another store entry, as in the following example: -```js -import { contentAssets } from './statics'; -const asset = contentAssets.getAsset('/public/index.html'); -const entry1 = await asset.getStoreEntry(); // Get a new store entry -const json1a = await entry1.json(); -const json1b = await entry1.json(); // Can't do this, the body has already been consumed! +##### Options: + +- `--collection-name`: The name of the source collection to promote (required) +- `--to`: The name of the new (target) collection to create or overwrite (required) + +**Expiration:** + +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format) +- `--expires-never`: Collection never expires + +*At most one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified*. If not provided, then the existing expiration rule of the collection being promoted is used. + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile + +#### `collections update-expiration` -const entry2 = await asset.getStoreEntry(); // Get a new store entry for same asset -const json2a = await entry2.json(); // This will work. +```sh +npx @fastly/compute-js-static-publish collections update-expiration \ + --collection-name=preview-42 \ + --expires-in=3d | --expires-at=2025-06-01T00:00Z | --expires-never ``` +Sets or updates the expiration time of an existing collection. -#### Module Assets +##### Options: +- `--collection-name`: The name of the collection to update (required) -Module assets are useful when an asset includes executable JavaScript code that you may want to execute at runtime. +**Expiration:** -You can obtain the module assets included in publishing by importing the `moduleAssets` object exported from -`./statics.js`. Keep in mind that by default, no modules are included in `moduleAssets`. If you wish to include module -assets, you must configure your publishing to include them. See [`moduleAssetInclusionTest` in the `static-publish.rc.js` -config file](#static-publish-rc) for more details. +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format) +- `--expires-never`: Collection never expires -`/module/hello.js` -```js -export function hello() { - console.log('Hello, World!'); -} +*Exactly one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** must be specified* + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile + +#### `collections delete` + +```sh +npx @fastly/compute-js-static-publish collections delete \ + --collection-name=preview-42 ``` -```js -import { moduleAssets } from './statics'; +Deletes a collection index from the KV Store. The content files will remain as they may still be referenced by other collection indexes. + +Use the `npx @fastly/compute-js-static-publish clean` command afterward to remove content files that are no longer referenced by any collection. + +##### Options: + +- `--collection-name`: The name of the collection to delete (required) + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile + +--- -// Obtain a module asset named '/module/hello.js' -const asset = contentAssets.getAsset('/module/hello.js'); +## πŸ“• Appendix -// Load the module -const helloModule = await asset.getModule(); +### fromCookie -helloModule.hello(); // Will print "Hello, World!" +`collectionSelector.fromCookie` is a utility function that enables the use of a browser +cookie to select the active collection. + +```js +import { collectionSelector } from '@fastly/compute-js-static-publish'; + +const { collectionName, redirectResponse } = + collectionSelector.fromCookie(request, { + // Everything here is optional – these are just examples + cookieName: 'publisher-collection', // default + activatePath: '/activate', // default + resetPath: '/reset', // default + cookieHttpOnly: true, // default + cookieMaxAge: 60 * 60 * 24 * 7, // 7-days. default is `null`, never expires. + cookiePath: '/', // default + }); + +if (redirectResponse) { + // honor the redirect + return redirectResponse; +} + +// `collectionName` now holds the active collection (or `null` if none) ``` -#### Metadata +#### What it does, in plain English -In some use cases, you may have a use case where you need to know about the files that were included during publishing, -but not in the context of Compute. (e.g., a tool that runs in Node.js that performs some maintenance task on assets). +1. Reads a cookie -You cannot import `./statics.js` from a Node.js application, as it holds dependencies on Compute. + It looks for a cookie named cookieName (default `publisher-collection`) and hands you the value as `collectionName`. -Instead, you can import `./statics-metadata.js`, a companion file that is generated in the same directory. This file -exposes plain JavaScript objects that contain the metadata about your content assets that were included in the final -publishing event. +2. Handles two helper endpoints for you -See the definition of `ContentAssetMetadataMapEntry` in the [`types/content-assets` file](./src/types/content-assets.ts) for more details. + | Endpoint | Purpose | Query params | Result | + |------------------------------------|------------------|------------------------------------------------------------------------------------------------|-------------------------------------------| + | activatePath (default `/activate`) | Set the cookie | `collection` (required) - name of the collection
`redirectTo` (optional, `/` by default) | `302` redirect with a `Set-Cookie` header | + | resetPath (default `/reset`) | Clear the cookie | `redirectTo` (optional, `/` by default) | `302` redirect that expires the cookie | + + If a visitor accesses `/activate?collection=blue&redirectTo=/preview`, the helper will issue a redirect and drop `publisher-collection=blue` into their cookie jar. + - If someone forgets `?collection=?`? Then `/activate` replies with HTTP `400`. + + When the visitor hits `/reset`, the cookie is deleted. -### Using webpack +3. Safety flags are baked in -As of v4, webpack is no longer required, and is no longer part of the default scaffolded application. -If you wish to use some features of webpack, you may include webpack in your generated application by specifying -`--webpack` at the command line. + - `HttpOnly` is on by default (configurable), to help avoid XSS issues. + - `Secure` is automatically added on HTTPS requests. + - `SameSite=Lax` is always set – reasonable default for previews. -## Migrating +#### Option reference -See [MIGRATING.md](./MIGRATING.md). +| Option | Type | Default | Notes | +|------------------|-------------------|--------------------------|------------------------------------------------| +| `cookieName` | string | `'publisher-collection'` | Name of the cookie to read/write | +| `activatePath` | string | `'/activate'` | Path that sets the cookie | +| `resetPath` | string | `'/reset'` | Path that clears the cookie | +| `cookieHttpOnly` | boolean | `true` | Turn off if the client-side needs to read it | +| `cookieMaxAge` | number\|undefined | `undefined` | Seconds until expiry; omit for session cookies | +| `cookiePath` | string | `'/'` | Path attribute in the cookie | -## Issues +#### Example -If you encounter any non-security-related bug or unexpected behavior, please [file an issue][bug]. +`fromCookie` can be dropped into a Fastly Compute app: -[bug]: https://github.com/fastly/compute-js-static-publish/issues/new?labels=bug +```js +async function handleRequest(event) { + const request = event.request; -### Security issues + // --- Cookie handling starts here + const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); + if (redirectResponse) { + // obey redirect first + return redirectResponse; + } + if (collectionName != null) { + publisherServer.setActiveCollectionName(collectionName); + } + // --- Cookie handling ends here + + // Regular routing follows... + const response = await publisherServer.serveRequest(request); + if (response != null) { + return response; + } + return new Response('Not found', { status: 404 }); +} +``` -Please see our [SECURITY.md](./SECURITY.md) for guidance on reporting security-related issues. +--- -## License +## πŸ“š Next Steps -[MIT](./LICENSE). +- View CLI command help: `npx @fastly/compute-js-static-publish --help` +- Use in CI to automate branch previews +- Visit [https://developer.fastly.com](https://developer.fastly.com) for Compute platform docs diff --git a/README.short.md b/README.short.md new file mode 100644 index 0000000..50f0c1a --- /dev/null +++ b/README.short.md @@ -0,0 +1,61 @@ +# @fastly/compute-js-static-publish + +Fastly Compute + KV Store for static websites and web apps. + +This CLI tool helps you: + +- βœ… Deploy static sites to Fastly Compute with zero backend +- πŸ“¦ Store files in Fastly KV Store efficiently +- πŸ—‚ Publish to named collections (`live`, `preview-42`, etc.) +- πŸ”„ Switch between collections at runtime +- 🧹 Clean up old or expired assets + +--- + +## Quick Start + +Create a directory for your project, place your static files in `./public`, then type: + +```sh +npx @fastly/compute-js-static-publish --root-dir=./public --kv-store-name=site-content +``` + +### πŸ”§ Local Preview + +```sh +cd compute-js +npm install +npm run dev:publish # 'publish' your files to the simulated local KV Store +npm run dev:start # preview locally +``` + +Serves your app at `http://127.0.0.1:7676`, powered by a simulated KV Store. + +### πŸš€ Deploy to Production + +When you're ready to go live, [create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already, and then: + +```sh +cd compute-js +npm run fastly:deploy # deploy the app +npm run fastly:publish # upload your static files +``` + +In the future, unless you have further changes to make to your app itself, you can +upload further updates to your static files: +```sh +cd compute-js +npm run fastly:publish # upload your static files +``` + +## Features + +- Named collections for previews, staging, production +- SPA + fallback handling +- Precompressed Brotli/gzip support +- CLI tools for publish, promote, and cleanup + +## Documentation + +πŸ“˜ Full documentation available on GitHub: +[https://github.com/fastly/compute-js-static-publish](https://github.com/fastly/compute-js-static-publish) diff --git a/package-lock.json b/package-lock.json index 0d34eb7..e354e78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,33 +1,35 @@ { "name": "@fastly/compute-js-static-publish", - "version": "6.3.0", + "version": "7.0.0-beta.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "6.3.0", + "version": "7.0.0-beta.5", "license": "MIT", "dependencies": { + "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", - "glob-to-regexp": "^0.4.1" + "glob-to-regexp": "^0.4.1", + "toml": "^3.0.0" }, "bin": { "compute-js-static-publish": "build/cli/index.js" }, "devDependencies": { - "@fastly/js-compute": "^3.0.0", + "@fastly/js-compute": "^3.33.2", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^20.0.0", + "@types/node": "^20.11.0", "rimraf": "^4.3.0", - "typescript": "^5.0.2" + "typescript": "^5.8.0" }, "engines": { - "node": ">=20" + "node": ">=20.11.0" }, "peerDependencies": { - "@fastly/js-compute": "^2.0.0 || ^3.0.0" + "@fastly/js-compute": "^3.33.2" } }, "node_modules/@bytecodealliance/componentize-js": { @@ -673,10 +675,160 @@ "node": ">=18" } }, + "node_modules/@fastly/cli": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli/-/cli-11.2.0.tgz", + "integrity": "sha512-1J2ksB5FtEqA9JMgJlD4ooMOUTS1zpsONJItSnu0jkA1v8g3n3yngpLMtbjFJ9ZWcuYlrHbjrlZnbQ44FNd6qA==", + "license": "Apache-2.0", + "bin": { + "fastly": "fastly.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@fastly/cli-darwin-arm64": "=11.2.0", + "@fastly/cli-darwin-x64": "=11.2.0", + "@fastly/cli-linux-arm64": "=11.2.0", + "@fastly/cli-linux-x32": "=11.2.0", + "@fastly/cli-linux-x64": "=11.2.0", + "@fastly/cli-win32-arm64": "=11.2.0", + "@fastly/cli-win32-x32": "=11.2.0", + "@fastly/cli-win32-x64": "=11.2.0" + } + }, + "node_modules/@fastly/cli-darwin-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-arm64/-/cli-darwin-arm64-11.2.0.tgz", + "integrity": "sha512-U/AFa0+B5aG1B0ye4F4aEi1xwBwV9iqC/hpXIKNzv9MjXwxhFpAxWEym58RDyQdivr8ID5bOA+v4T4KZscNGsA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "cli-darwin-arm64": "fastly" + } + }, + "node_modules/@fastly/cli-darwin-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-x64/-/cli-darwin-x64-11.2.0.tgz", + "integrity": "sha512-O0nnkO6jZjsfV72Gyr49jKTJ2jKL3gNJCzkq5ImYGX2fLIc+0Fd2BGFi2MUVSZbfktAnogHXPZcy8iCshE2Zgg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "cli-darwin-x64": "fastly" + } + }, + "node_modules/@fastly/cli-linux-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-arm64/-/cli-linux-arm64-11.2.0.tgz", + "integrity": "sha512-ofUn4uqdgbM0gMk6K3eV2PqplNA6jZ13MtZSFh0Bp4DYVunIPqhBbDxu5ndmxRan14KGTt+EH4vNa6trh3Za7w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-arm64": "fastly" + } + }, + "node_modules/@fastly/cli-linux-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x32/-/cli-linux-x32-11.2.0.tgz", + "integrity": "sha512-owq6DVR7snlM0fAOLWrp/FbS7f2/xbwjS3n8Vxy9Kjjjfe1DzEUGBKBzpIBZqqqdZ55CKz6Xf9gC/pjRR6f3DQ==", + "cpu": [ + "x32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-x32": "fastly" + } + }, + "node_modules/@fastly/cli-linux-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x64/-/cli-linux-x64-11.2.0.tgz", + "integrity": "sha512-cdQYWBDuk3fH+7J9LscB3bxWnBn+Iy82DYPdA1NFfDqNTLEjQ+xiePa3EoTv3x6v8sw8b3IjTDiuPmUMBS0w7A==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-x64": "fastly" + } + }, + "node_modules/@fastly/cli-win32-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-arm64/-/cli-win32-arm64-11.2.0.tgz", + "integrity": "sha512-8qOqKpGU6lymV7IXnL3rdItle5MeDq7ntO90JKIx66PxaZIpAznKvqxGQlFBYaj58xjnGBMcRe+Zcxm1j5tItw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-arm64": "fastly.exe" + } + }, + "node_modules/@fastly/cli-win32-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x32/-/cli-win32-x32-11.2.0.tgz", + "integrity": "sha512-OjJaoAkg3rDqTrQKabZ0e1a+l97ELAT4kVCMxDLdd7Y13qWGrKzSx6G3tpYo03Ud9DHV8rnPvfaZtVlTvyHEEw==", + "cpu": [ + "x32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-x32": "fastly.exe" + } + }, + "node_modules/@fastly/cli-win32-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x64/-/cli-win32-x64-11.2.0.tgz", + "integrity": "sha512-3Kh4D4Ii1avHulaJO7qi4Hpapt1GS5JJC+0hX+NsrypqVwuTsevChZlwKLcBYEFKfM5gFvnDsjOE1WMm9APnBA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-x64": "fastly.exe" + } + }, "node_modules/@fastly/js-compute": { - "version": "3.33.1", - "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.1.tgz", - "integrity": "sha512-G93hZpJO5USH3K+URGK/tF2C4OPQq19txPKtemy4OHODBpEQqoqgBspUgO8QI9SXvjrtR/WIJYfTrMaW8HrO+g==", + "version": "3.33.2", + "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.2.tgz", + "integrity": "sha512-CfU/uqyc365+m7O8gAn2p0hxQHTz5uJ6/ifEunrFrVmjgiRv4PssO1X+06JFye1ts8X3OJo8Nm26XhqL3p5XIA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2332,6 +2484,12 @@ "dev": true, "license": "MIT" }, + "node_modules/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -2341,16 +2499,17 @@ "optional": true }, "node_modules/typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/typical": { @@ -2768,10 +2927,73 @@ "dev": true, "optional": true }, + "@fastly/cli": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli/-/cli-11.2.0.tgz", + "integrity": "sha512-1J2ksB5FtEqA9JMgJlD4ooMOUTS1zpsONJItSnu0jkA1v8g3n3yngpLMtbjFJ9ZWcuYlrHbjrlZnbQ44FNd6qA==", + "requires": { + "@fastly/cli-darwin-arm64": "=11.2.0", + "@fastly/cli-darwin-x64": "=11.2.0", + "@fastly/cli-linux-arm64": "=11.2.0", + "@fastly/cli-linux-x32": "=11.2.0", + "@fastly/cli-linux-x64": "=11.2.0", + "@fastly/cli-win32-arm64": "=11.2.0", + "@fastly/cli-win32-x32": "=11.2.0", + "@fastly/cli-win32-x64": "=11.2.0" + } + }, + "@fastly/cli-darwin-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-arm64/-/cli-darwin-arm64-11.2.0.tgz", + "integrity": "sha512-U/AFa0+B5aG1B0ye4F4aEi1xwBwV9iqC/hpXIKNzv9MjXwxhFpAxWEym58RDyQdivr8ID5bOA+v4T4KZscNGsA==", + "optional": true + }, + "@fastly/cli-darwin-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-x64/-/cli-darwin-x64-11.2.0.tgz", + "integrity": "sha512-O0nnkO6jZjsfV72Gyr49jKTJ2jKL3gNJCzkq5ImYGX2fLIc+0Fd2BGFi2MUVSZbfktAnogHXPZcy8iCshE2Zgg==", + "optional": true + }, + "@fastly/cli-linux-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-arm64/-/cli-linux-arm64-11.2.0.tgz", + "integrity": "sha512-ofUn4uqdgbM0gMk6K3eV2PqplNA6jZ13MtZSFh0Bp4DYVunIPqhBbDxu5ndmxRan14KGTt+EH4vNa6trh3Za7w==", + "optional": true + }, + "@fastly/cli-linux-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x32/-/cli-linux-x32-11.2.0.tgz", + "integrity": "sha512-owq6DVR7snlM0fAOLWrp/FbS7f2/xbwjS3n8Vxy9Kjjjfe1DzEUGBKBzpIBZqqqdZ55CKz6Xf9gC/pjRR6f3DQ==", + "optional": true + }, + "@fastly/cli-linux-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x64/-/cli-linux-x64-11.2.0.tgz", + "integrity": "sha512-cdQYWBDuk3fH+7J9LscB3bxWnBn+Iy82DYPdA1NFfDqNTLEjQ+xiePa3EoTv3x6v8sw8b3IjTDiuPmUMBS0w7A==", + "optional": true + }, + "@fastly/cli-win32-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-arm64/-/cli-win32-arm64-11.2.0.tgz", + "integrity": "sha512-8qOqKpGU6lymV7IXnL3rdItle5MeDq7ntO90JKIx66PxaZIpAznKvqxGQlFBYaj58xjnGBMcRe+Zcxm1j5tItw==", + "optional": true + }, + "@fastly/cli-win32-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x32/-/cli-win32-x32-11.2.0.tgz", + "integrity": "sha512-OjJaoAkg3rDqTrQKabZ0e1a+l97ELAT4kVCMxDLdd7Y13qWGrKzSx6G3tpYo03Ud9DHV8rnPvfaZtVlTvyHEEw==", + "optional": true + }, + "@fastly/cli-win32-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x64/-/cli-win32-x64-11.2.0.tgz", + "integrity": "sha512-3Kh4D4Ii1avHulaJO7qi4Hpapt1GS5JJC+0hX+NsrypqVwuTsevChZlwKLcBYEFKfM5gFvnDsjOE1WMm9APnBA==", + "optional": true + }, "@fastly/js-compute": { - "version": "3.33.1", - "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.1.tgz", - "integrity": "sha512-G93hZpJO5USH3K+URGK/tF2C4OPQq19txPKtemy4OHODBpEQqoqgBspUgO8QI9SXvjrtR/WIJYfTrMaW8HrO+g==", + "version": "3.33.2", + "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.2.tgz", + "integrity": "sha512-CfU/uqyc365+m7O8gAn2p0hxQHTz5uJ6/ifEunrFrVmjgiRv4PssO1X+06JFye1ts8X3OJo8Nm26XhqL3p5XIA==", "dev": true, "requires": { "@bytecodealliance/jco": "^1.7.0", @@ -3834,6 +4056,11 @@ "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", "dev": true }, + "toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, "tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -3842,9 +4069,9 @@ "optional": true }, "typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true }, "typical": { diff --git a/package.json b/package.json index 7be042c..d71e839 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,22 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "6.3.0", + "version": "7.0.0-beta.5", "description": "Static Publisher for Fastly Compute JavaScript", - "main": "./build/index.js", - "types": "./build/index.d.ts", + "main": "build/index.js", + "exports": { + ".": { + "types": "./build/index.d.ts", + "default": "./build/index.js" + } + }, "scripts": { "clean": "rimraf build", "prepack": "npm run build", - "build": "tsc", - "test": "echo \"Error: no test specified\" && exit 1" + "build": "npm run clean && npm run compile", + "compile": "tsc", + "test": "echo \"Error: no test specified\" && exit 1", + "prepublishOnly": "cp README.short.md README.md" }, "bin": { "compute-js-static-publish": "./build/cli/index.js" @@ -31,22 +38,24 @@ }, "homepage": "https://github.com/fastly/compute-js-static-publish#readme", "dependencies": { + "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", - "glob-to-regexp": "^0.4.1" + "glob-to-regexp": "^0.4.1", + "toml": "^3.0.0" }, "peerDependencies": { - "@fastly/js-compute": "^2.0.0 || ^3.0.0" + "@fastly/js-compute": "^3.33.2" }, "devDependencies": { - "@fastly/js-compute": "^3.0.0", + "@fastly/js-compute": "^3.33.2", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^20.0.0", + "@types/node": "^20.11.0", "rimraf": "^4.3.0", - "typescript": "^5.0.2" + "typescript": "^5.8.0" }, "engines": { - "node": ">=20" + "node": ">=20.11.0" }, "files": [ "build", diff --git a/resources/README.md b/resources/README.md deleted file mode 100644 index 4a81915..0000000 --- a/resources/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## Resources - -The files in this directory are copied into the output folder. diff --git a/resources/statics-metadata.d.ts b/resources/statics-metadata.d.ts deleted file mode 100644 index f95dd63..0000000 --- a/resources/statics-metadata.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Generated by @fastly/compute-js-static-publish. - */ - -import type { - ContentAssetMetadataMap, -} from "@fastly/compute-js-static-publish"; - -export declare const kvStoreName: string | null; -export declare const contentAssetMetadataMap: ContentAssetMetadataMap; diff --git a/resources/statics.d.ts b/resources/statics.d.ts deleted file mode 100644 index 91658ce..0000000 --- a/resources/statics.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Generated by @fastly/compute-js-static-publish. - */ - -import type { - ModuleAssetMap, - PublisherServerConfigNormalized, - ContentAssets, - ModuleAssets, - PublisherServer, -} from '@fastly/compute-js-static-publish'; - -export declare const moduleAssetMap: ModuleAssetMap; - -export declare const serverConfig: PublisherServerConfigNormalized; -export declare const contentAssets: ContentAssets; -export declare const moduleAssets: ModuleAssets; - -export declare function getServer(): PublisherServer; diff --git a/resources/webpack.config.js b/resources/webpack.config.js deleted file mode 100644 index a7f0737..0000000 --- a/resources/webpack.config.js +++ /dev/null @@ -1,41 +0,0 @@ -const path = require("path"); - -module.exports = { - entry: "./src/index.js", - target: false, - devtool: false, - optimization: { - minimize: true - }, - output: { - filename: "index.js", - path: path.resolve(__dirname, "bin"), - chunkFormat: 'commonjs', - library: { - type: 'commonjs', - }, - }, - module: { - // Loaders go here. - // e.g., ts-loader for TypeScript - // rules: [ - // ], - }, - resolve: { - conditionNames: [ - 'fastly', - '...', - ], - }, - plugins: [ - // Webpack Plugins and Polyfills go here - // e.g., cross-platform WHATWG or core Node.js modules needed for your application. - // new webpack.ProvidePlugin({ - // }), - ], - externals: [ - // Allow webpack to handle 'fastly:*' namespaced module imports by treating - // them as modules rather than trying to process them as URLs - /^fastly:.*$/, - ], -}; diff --git a/src/cli/commands/build-static.ts b/src/cli/commands/build-static.ts deleted file mode 100644 index 4edae95..0000000 --- a/src/cli/commands/build-static.ts +++ /dev/null @@ -1,741 +0,0 @@ -// This program builds static resources out of the files in the -// public folder to be served. -// e.g., with create-react-app, this would be the ./build directory. - -// Outputs: -// -// statics-metadata.js -// -// kvStoreName: string - Name of KV Store, or null if not used. -// contentAssetMetadataMap: Map - mapping of asset keys to their metadata -// - key: string - assetKey, the filename relative to root dir (e.g., /public/foo/index.html) -// - value: ContentAssetMetadataMapEntry - metadata for asset -// -// ContentAssetMetadataMapEntry structure -// - assetKey: string - filename relative to root dir -// - contentType: string - MIME type -// - text: boolean - whether this is a text file -// - lastModifiedTime: number - last modified time as unix time (seconds) -// - fileInfo: FileInfo - information about the file -// - compressedFileInfos: Map - information about any compressed versions -// - key: string - compression algorithm name (e.g., "gzip", "br", etc.) -// - value: FileInfo - information about the compressed version of the file -// - type: string - where the data is available. Usually 'wasm-inline' or 'kv-store'. -// FileInfo structure -// - hash: string - SHA-256 hash of file -// - size: number - file size in bytes -// - staticFilePath: string - path to file in local filesystem, relative to root dir -// - kvStoreKey: string - KV Store key, present only for items in KV Store -// -// statics.js -// imports statics-metadata.js and adds utilities for working with the content assets, as well as -// for loading module assets. Also gives access to StaticPublisher and its config. -// See README.md for details -// -// moduleAssetMap: Map -// - key: string - assetKey, the filename relative to root dir (e.g., /module/hello.js) -// - value: ModuleAssetMapEntry - information about the module -// -// ModuleAssetMapEntry structure -// - isStaticImport: boolean - if true, then uses a static import statement to load the module -// - if false, then module is loaded when getModule is called -// - module: any - the statically loaded module if isStaticImport is true, or null -// - loadModule: function - a function that dynamically loads the module and returns it, if isStaticImport is false, or null -// -// contentAssets: ContentAssets instance -// moduleAssets: ModuleAssets instance -// -// getServer(): function - instantiates PublisherServer singleton and returns it -// serverConfig: PublisherServerConfigNormalized - publisher server config settings -// -// PublisherServerConfigNormalized structure -// - publicDirPrefix: string -// - staticItems: string[] -// - compression: string[] -// - spaFile: string or null -// - notFoundPageFile: string or null -// - autoExt: string[] -// - autoIndex: string[] -// - -import * as fs from "fs"; -import * as path from "path"; -import * as url from "url"; -import commandLineArgs from "command-line-args"; - -import { loadConfigFile } from "../load-config.js"; -import { applyDefaults } from "../util/data.js"; -import { calculateFileSizeAndHash } from "../util/hash.js"; -import { getFiles } from "../util/files.js"; -import { generateOrLoadPublishId } from "../util/publish-id.js"; -import { FastlyApiContext, FetchError, loadApiKey } from "../util/fastly-api.js"; -import { kvStoreEntryExists, kvStoreSubmitFile } from "../util/kv-store.js"; -import { mergeContentTypes, testFileContentType } from "../../util/content-types.js"; -import { algs } from "../compression/index.js"; - -import type { - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, -} from "../../types/content-assets.js"; -import type { - ContentTypeDef, - ContentTypeTestResult, -} from "../../types/content-types.js"; -import type { - ContentAssetInclusionResultNormalized, - ModuleAssetInclusionResultNormalized, - PublisherServerConfigNormalized, -} from "../../types/config-normalized.js"; -import type { - ContentCompressionTypes, -} from "../../constants/compression.js"; -import type { - CompressedFileInfos, - ContentFileInfoForWasmInline, - ContentFileInfoForKVStore, -} from "../../types/content-assets.js"; -import { - attemptWithRetries, -} from "../util/retryable.js"; - -type AssetInfo = - ContentTypeTestResult & - ContentAssetInclusionResultNormalized & - ModuleAssetInclusionResultNormalized & - { - // Full path to file on local filesystem - file: string, - - // Asset key (relative to public dir) - assetKey: string, - - // Last modified time - lastModifiedTime: number, - - // Hash (to be used as etag and as part of file id) - hash: string, - - // Size of file - size: number, - }; - -type KVStoreItemDesc = { - kvStoreKey: string, - staticFilePath: string, - text: boolean, -}; - -async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { - - const maxConcurrent = 12; - let index = 0; // Shared among workers - - async function worker() { - while (index < kvStoreItems.length) { - const currentIndex = index; - index = index + 1; - const { kvStoreKey, staticFilePath, text } = kvStoreItems[currentIndex]; - - try { - await attemptWithRetries( - async() => { - if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) { - console.log(`βœ”οΈ Asset already exists in KV Store with key "${kvStoreKey}".`); - return; - } - const fileData = fs.readFileSync(staticFilePath); - await kvStoreSubmitFile(fastlyApiContext, kvStoreName, kvStoreKey, fileData); - console.log(`βœ”οΈ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`) - }, - { - onAttempt(attempt) { - if (attempt > 0) { - console.log(`Attempt ${attempt + 1} for: ${kvStoreKey}`); - } - }, - onRetry(attempt, err, delay) { - let statusMessage = 'unknown'; - if (err instanceof FetchError) { - statusMessage = `HTTP ${err.status}`; - } else if (err instanceof TypeError) { - statusMessage = 'transport'; - } - console.log(`Attempt ${attempt + 1} for ${kvStoreKey} gave retryable error (${statusMessage}), delaying ${delay} ms`); - }, - } - ); - } catch (err) { - const e = err instanceof Error ? err : new Error(String(err)); - console.error(`❌ Failed: ${kvStoreKey} β†’ ${e.message}`); - console.error(e.stack); - } - } - } - - const workers = Array.from({ length: maxConcurrent }, () => worker()); - await Promise.all(workers); -} - -function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { - - let fastlyToml = fs.readFileSync('./fastly.toml', 'utf-8'); - - let before: string = ''; - let after: string = ''; - - kvStoreName = kvStoreName.includes(".") ? `"${kvStoreName}"` : kvStoreName; - - const tableMarker = `[[local_server.kv_stores.${kvStoreName}]]`; - - const startPos = fastlyToml.indexOf(tableMarker); - if (startPos === -1) { - - // KV Store decl not in fastly.toml yet - - if (fastlyToml.indexOf(kvStoreName) !== -1) { - // don't do this! - console.error(`improperly configured entry for '${kvStoreName}' in fastly.toml`); - // TODO: handle thrown exception from callers - throw "No"! - } - - let newLines; - if (fastlyToml.endsWith('\n\n')) { - newLines = ''; - } else if (fastlyToml.endsWith('\n')) { - newLines = '\n' - } else { - newLines = '\n\n'; - } - - before = fastlyToml + newLines; - after = ''; - - } else { - - const lastObjStoreTablePos = fastlyToml.lastIndexOf(tableMarker); - const nextTablePos = fastlyToml.indexOf('[', lastObjStoreTablePos + tableMarker.length); - - before = fastlyToml.slice(0, startPos); - - if (nextTablePos === -1) { - - after = ''; - - } else { - - after = fastlyToml.slice(nextTablePos); - - } - - } - - let tablesToml = ''; - - for (const {kvStoreKey, staticFilePath} of kvStoreItems) { - // Probably, JSON.stringify is wrong, but it should do its job - tablesToml += tableMarker + '\n'; - tablesToml += 'key = ' + JSON.stringify(kvStoreKey) + '\n'; - tablesToml += 'path = ' + JSON.stringify(path.relative('./', staticFilePath)) + '\n'; - tablesToml += '\n'; - } - - fastlyToml = before + tablesToml + after; - - fs.writeFileSync('./fastly.toml', fastlyToml, 'utf-8'); -} - -export async function buildStaticLoader(commandLineValues: commandLineArgs.CommandLineOptions) { - - const { 'suppress-framework-warnings': suppressFrameworkWarnings } = commandLineValues; - const displayFrameworkWarnings = !suppressFrameworkWarnings; - - const { publishId, created } = generateOrLoadPublishId(); - - if (created) { - console.log("βœ… Created publish ID"); - } - - console.log(`Publish ID: ${publishId}`); - - console.log("πŸš€ Building loader..."); - - const errors: string[] = []; - const { normalized: config, raw: configRaw } = await loadConfigFile(errors); - - if (config == null) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - for (const error of errors) { - console.error(error); - } - process.exitCode = 1; - return; - } - - const outputDir = path.resolve(); - const publicDirRoot = path.resolve(config.rootDir); - - console.log(`βœ”οΈ Public directory '${publicDirRoot}'.`); - - const kvStoreName = config.kvStoreName; - let fastlyApiContext: FastlyApiContext | null = null; - if (kvStoreName != null) { - // TODO: load api key from command line - const apiKeyResult = loadApiKey(); - if (apiKeyResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); - process.exitCode = 1; - return; - } - fastlyApiContext = { apiToken: apiKeyResult.apiToken }; - console.log(`βœ”οΈ Using KV Store mode, with KV Store: ${kvStoreName}`); - console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiKeyResult.source}'`); - } else { - if (displayFrameworkWarnings) { - console.log(`βœ”οΈ Not using KV Store mode.`); - } - } - - const excludeDirs = config.excludeDirs; - if (excludeDirs.length > 0) { - console.log(`βœ”οΈ Using exclude directories: ${excludeDirs.join(', ')}`); - } else { - if (displayFrameworkWarnings) { - console.log(`βœ”οΈ No exclude directories defined.`); - } - } - - const excludeDotFiles = config.excludeDotFiles; - if (excludeDotFiles) { - console.log(`βœ”οΈ Files/Directories starting with . are excluded.`); - } - - const includeWellKnown = config.includeWellKnown; - if (includeWellKnown) { - console.log(`βœ”οΈ (.well-known is exempt from exclusion.)`); - } - - const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); - - // Load content types - const finalContentTypes: ContentTypeDef[] = mergeContentTypes(config.contentTypes); - - // getFiles() applies excludeDirs, excludeDotFiles, and includeWellKnown. - let files = getFiles(publicDirRoot, { ...config, publicDirRoot }); - - // If public dir is not inside the c@e dir, then - // exclude files that come from C@E app dir - if (!publicDirRoot.startsWith(outputDir)) { - files = files.filter(file => !file.startsWith(outputDir)); - } - - // And then we apply assetInclusionTest. - const assetInfos: AssetInfo[] = files.map(file => { - const assetKey = file.slice(publicDirRoot.length) - // in Windows, assetKey will otherwise end up as \path\file.html - .replace(/\\/g, '/'); - - let contentTypeTestResult = testFileContentType(finalContentTypes, assetKey); - - if (contentTypeTestResult == null) { - contentTypeTestResult = { - text: false, - contentType: 'application/octet-stream', - precompressAsset: false, - }; - if (displayFrameworkWarnings) { - console.log('⚠️ Notice: Unknown file type ' + assetKey + '. Treating as binary file.'); - } - } - - let contentAssetInclusionResultValue = config.contentAssetInclusionTest?.(assetKey, contentTypeTestResult.contentType); - if (typeof contentAssetInclusionResultValue === 'boolean') { - contentAssetInclusionResultValue = { - includeContent: contentAssetInclusionResultValue, - }; - } else if (contentAssetInclusionResultValue === 'inline') { - contentAssetInclusionResultValue = { - includeContent: true, - inline: true, - }; - } - const contentAssetInclusionResult = applyDefaults(contentAssetInclusionResultValue ?? null, { - includeContent: true, - inline: false, - }); - - let moduleAssetInclusionResultValue = config.moduleAssetInclusionTest?.(assetKey, contentTypeTestResult.contentType); - if (typeof moduleAssetInclusionResultValue === 'boolean') { - moduleAssetInclusionResultValue = { - includeModule: moduleAssetInclusionResultValue, - }; - } else if (moduleAssetInclusionResultValue === 'static-import') { - moduleAssetInclusionResultValue = { - includeModule: true, - useStaticImport: true, - }; - } - - const moduleAssetInclusionResult = applyDefaults(moduleAssetInclusionResultValue ?? null, { - includeModule: false, - useStaticImport: false, - }); - - const stats = fs.statSync(file); - const lastModifiedTime = Math.floor((stats.mtime).getTime() / 1000); - - const { size, hash } = calculateFileSizeAndHash(file); - - return { - file, - assetKey, - lastModifiedTime, - hash, - size, - ...contentAssetInclusionResult, - ...moduleAssetInclusionResult, - ...contentTypeTestResult, - }; - - }); - - console.log("πŸš€ Preparing content assets ..."); - - if (kvStoreName == null && configRaw != null && !("contentCompression" in configRaw)) { - console.log(`⚠️ Notice: By default, pre-compressed content assets are not generated when KV Store is not used.`); - console.log(" If you want to pre-compress assets, add a value for 'contentCompression' to your static-publish.rc.js."); - } - - // The Static Content Root Dir, which will hold the loaders and a copy of static files. - const staticContentRootDir = config.staticContentRootDir; - - // Create "static content" dir that will be used to hold a copy of static files. - // NOTE: This copy is created so that they can be loaded with includeBytes(), which requires - // all files to be under the Compute project dir. - const staticContentDir = `${staticContentRootDir}/static-content`; - fs.rmSync(staticContentDir, { recursive: true, force: true }); - fs.mkdirSync(staticContentDir, { recursive: true }); - - // Build content assets metadata - const contentAssetMetadataMap: ContentAssetMetadataMap = {}; - - // KV Store items to upload - const kvStoreItems: KVStoreItemDesc[] = []; - - let contentItems = 0; - const counts = { - inline: 0, - kvStore: 0, - excluded: 0, - } - for (const assetInfo of assetInfos) { - if (!assetInfo.includeContent) { - // Non-content asset - counts.excluded++; - continue; - } - contentItems++; - console.log(`βœ”οΈ [${contentItems}] ${assetInfo.assetKey}: ${JSON.stringify(assetInfo.contentType)}`); - - const { - file, - assetKey, - contentType, - text, - precompressAsset, - hash, - size, - lastModifiedTime, - } = assetInfo; - - const entryBase = { - assetKey, - contentType, - lastModifiedTime, - text, - fileInfo: { - hash, - size, - }, - }; - - let metadata: ContentAssetMetadataMapEntry; - - const isInline = kvStoreName == null || assetInfo.inline; - - const staticContentFilePath = `${staticContentDir}/file${contentItems}.${assetInfo.text ? 'txt' : 'bin'}`; - - type PrepareCompressionVersionFunc = (alg: ContentCompressionTypes, staticFilePath: string, hash: string, size: number) => void; - async function prepareCompressedVersions(contentCompressions: ContentCompressionTypes[], func: PrepareCompressionVersionFunc) { - for (const alg of contentCompressions) { - const compressTo = algs[alg]; - if (precompressAsset && compressTo != null) { - - const staticFilePath = `${staticContentFilePath}_${alg}`; - if (await compressTo(file, staticFilePath, text)) { - console.log(`βœ”οΈ Compressed ${text ? 'text' : 'binary'} asset "${file}" to "${staticFilePath}" [${alg}].`) - const { size, hash } = calculateFileSizeAndHash(staticFilePath); - func(alg, staticFilePath, hash, size); - } - - } - } - } - - const contentCompression = config.contentCompression; - - if (isInline) { - // We will inline this file using includeBytes() - // so we copy it to the static-content directory. - const staticFilePath = staticContentFilePath; - fs.cpSync(file, staticFilePath); - console.log(`βœ”οΈ Copied ${text ? 'text' : 'binary'} asset "${file}" to "${staticFilePath}".`); - - // 'prepareCompressedVersions' uses the static content directory for compressed copies, - // even for 'inline' files. - const compressedFileInfos: CompressedFileInfos = {}; - await prepareCompressedVersions(contentCompression, (alg, staticFilePath, hash, size) => { - compressedFileInfos[alg] = { staticFilePath, hash, size }; - }); - - metadata = { - ...entryBase, - type: 'wasm-inline', - fileInfo: { - ...entryBase.fileInfo, - staticFilePath, - }, - compressedFileInfos, - }; - - counts.inline++; - } else { - // For KV Store mode, we don't need to make a copy of the original file - const staticFilePath = file; - - // Use the hash as part of the key name. This avoids having to - // re-upload a file if it already exists. - const kvStoreKey = `${publishId}:${assetKey}_${hash}`; - - kvStoreItems.push({ - kvStoreKey, - staticFilePath, - text, - }); - - const compressedFileInfos: CompressedFileInfos = {}; - await prepareCompressedVersions(contentCompression, (alg, staticFilePath, hash, size) => { - const kvStoreKey = `${publishId}:${assetKey}_${alg}_${hash}`; - compressedFileInfos[alg] = { staticFilePath, kvStoreKey, hash, size }; - kvStoreItems.push({ - kvStoreKey, - staticFilePath, - text, - }); - }); - - metadata = { - ...entryBase, - type: 'kv-store', - fileInfo: { - ...entryBase.fileInfo, - staticFilePath, - kvStoreKey, - }, - compressedFileInfos, - }; - - counts.kvStore++; - } - - contentAssetMetadataMap[assetKey] = metadata; - } - - if (kvStoreName != null) { - await uploadFilesToKVStore(fastlyApiContext!, kvStoreName, kvStoreItems); - writeKVStoreEntriesToFastlyToml(kvStoreName, kvStoreItems); - } - - console.log("βœ… Prepared " + (counts.inline + counts.kvStore) + " content asset(s):"); - if (counts.inline > 0) { - console.log(" " + counts.inline + " inline"); - } - if (counts.kvStore > 0) { - console.log(" " + counts.kvStore + " KV Store"); - } - - // Build statics-metadata.js - let metadataFileContents = `/* - * Generated by @fastly/compute-js-static-publish. - */ - -`; - metadataFileContents += `\nexport const kvStoreName = ${JSON.stringify(kvStoreName)};\n`; - metadataFileContents += `\nexport const contentAssetMetadataMap = {\n`; - for (const [key, value] of Object.entries(contentAssetMetadataMap)) { - metadataFileContents += ` ${JSON.stringify(key)}: ${JSON.stringify(value)},\n`; - } - metadataFileContents += '};\n'; - fs.writeFileSync(`${staticContentRootDir}/statics-metadata.js`, metadataFileContents); - - console.log(`βœ… Wrote static file metadata for ${contentItems} file(s).`); - - // Copy Types file for static file loader - try { - const typesFile = path.resolve(__dirname, '../../../resources/statics-metadata.d.ts'); - fs.copyFileSync(typesFile, `${staticContentRootDir}/statics-metadata.d.ts`); - - console.log("βœ… Wrote content assets metadata types file statics-metadata.d.ts."); - } catch { - console.log("⚠️ Notice: could not write content assets metadata types file statics-metadata.d.ts."); - } - - // Build statics.js - let fileContents = `/* - * Generated by @fastly/compute-js-static-publish. - */ - -`; - - fileContents += 'import { ContentAssets, ModuleAssets, PublisherServer } from "@fastly/compute-js-static-publish";\n\n'; - fileContents += 'import "@fastly/compute-js-static-publish/build/compute-js.js";\n\n'; - fileContents += 'import { kvStoreName, contentAssetMetadataMap } from "./statics-metadata.js";\n'; - - // Add import statements for assets that are modules and that need to be statically imported. - const staticImportModuleNumbers: Record = {}; - let staticImportModuleNumber = 0; - for (const assetInfo of assetInfos) { - if (assetInfo.includeModule && assetInfo.useStaticImport) { - staticImportModuleNumber++; - staticImportModuleNumbers[assetInfo.assetKey] = staticImportModuleNumber; - - // static-content lives in the staticContentRootDir dir, so the import must be declared as - // relative to that file. - const relativeFilePath = path.relative(staticContentRootDir, assetInfo.file); - fileContents += `import * as fileModule${staticImportModuleNumber} from "${relativeFilePath}";\n`; - } - } - - fileContents += `\nexport const moduleAssetsMap = {\n`; - - for (const assetInfo of assetInfos) { - if (!assetInfo.includeModule) { - continue; - } - - let module; - let loadModuleFunction; - - if (assetInfo.useStaticImport) { - const moduleNumber = staticImportModuleNumbers[assetInfo.assetKey]; - if (moduleNumber == null) { - throw new Error(`Unexpected! module asset number for "${assetInfo.assetKey}" was not found!`); - } - loadModuleFunction = `() => Promise.resolve(fileModule${moduleNumber})`; - module = `fileModule${moduleNumber}`; - } else { - // static-content lives in the staticContentRootDir dir, so the import must be declared as - // relative to that file. - const relativeFilePath = path.relative(staticContentRootDir, assetInfo.file); - loadModuleFunction = `() => import("${relativeFilePath}")`; - module = 'null'; - } - - fileContents += ` ${JSON.stringify(assetInfo.assetKey)}: { isStaticImport: ${JSON.stringify(assetInfo.useStaticImport)}, module: ${module}, loadModule: ${loadModuleFunction} },\n`; - } - - fileContents += '};\n'; - - const server = applyDefaults(config.server, { - publicDirPrefix: '', - staticItems: [], - compression: [ 'br', 'gzip' ], - spaFile: null, - notFoundPageFile: null, - autoExt: [], - autoIndex: [], - }); - - let publicDirPrefix = server.publicDirPrefix; - console.log(`βœ”οΈ Server public dir prefix '${publicDirPrefix}'.`); - - let staticItems = server.staticItems; - - let compression = server.compression; - - let spaFile = server.spaFile; - if(spaFile != null) { - console.log(`βœ”οΈ Application SPA file '${spaFile}'.`); - const spaAsset = assetInfos.find(assetInfo => assetInfo.assetKey === spaFile); - if(spaAsset == null || spaAsset.contentType !== 'text/html') { - if (displayFrameworkWarnings) { - console.log(`⚠️ Notice: '${spaFile}' does not exist or is not of type 'text/html'. Ignoring.`); - } - spaFile = null; - } - } else { - if (displayFrameworkWarnings) { - console.log(`βœ”οΈ Application is not a SPA.`); - } - } - - let notFoundPageFile = server.notFoundPageFile; - if(notFoundPageFile != null) { - console.log(`βœ”οΈ Application 'not found (404)' file '${notFoundPageFile}'.`); - const notFoundPageAsset = assetInfos.find(assetInfo => assetInfo.assetKey === notFoundPageFile); - - if(notFoundPageAsset == null || notFoundPageAsset.contentType !== 'text/html') { - if (displayFrameworkWarnings) { - console.log(`⚠️ Notice: '${notFoundPageFile}' does not exist or is not of type 'text/html'. Ignoring.`); - } - notFoundPageFile = null; - } - } else { - if (displayFrameworkWarnings) { - console.log(`βœ”οΈ Application specifies no 'not found (404)' page.`); - } - } - - let autoIndex: string[] = server.autoIndex; - let autoExt: string[] = server.autoExt; - - const serverConfig: PublisherServerConfigNormalized = { - publicDirPrefix, - staticItems, - compression, - spaFile, - notFoundPageFile, - autoExt, - autoIndex, - }; - - fileContents += `\nexport const serverConfig = ${JSON.stringify(serverConfig, null, 2)};\n`; - - fileContents += '\nexport const contentAssets = new ContentAssets(contentAssetMetadataMap, {kvStoreName});'; - fileContents += '\nexport const moduleAssets = new ModuleAssets(moduleAssetsMap);\n'; - - fileContents += '\nlet server = null;'; - fileContents += '\nexport function getServer() {' + - '\n if (server == null) {' + - '\n server = new PublisherServer(serverConfig, contentAssets);' + - '\n }' + - '\n return server;' + - '\n}\n'; - - fs.writeFileSync(`${staticContentRootDir}/statics.js`, fileContents); - - console.log("βœ… Wrote static file loader for " + files.length + " file(s)."); - - // Copy Types file for static file loader - try { - const staticsTypeFile = path.resolve(__dirname, '../../../resources/statics.d.ts'); - fs.copyFileSync(staticsTypeFile, `${staticContentRootDir}/statics.d.ts`); - - console.log(`βœ… Wrote static file loader types file ${staticContentRootDir}/statics.d.ts.`); - } catch { - console.log(`⚠️ Notice: could not write static file loader types file ${staticContentRootDir}/statics.d.ts.`); - } - -} diff --git a/src/cli/commands/clean-kv-store.ts b/src/cli/commands/clean-kv-store.ts deleted file mode 100644 index 79c56b3..0000000 --- a/src/cli/commands/clean-kv-store.ts +++ /dev/null @@ -1,82 +0,0 @@ -import path from 'path'; -import type { CommandLineOptions } from "command-line-args"; - -import { generateOrLoadPublishId } from "../util/publish-id.js"; -import { loadConfigFile } from "../load-config.js"; -import { getKVStoreKeys, kvStoreDeleteFile } from "../util/kv-store.js"; -import { FastlyApiContext, loadApiKey } from "../util/fastly-api.js"; -import { getKVStoreKeysFromMetadata } from "../../util/metadata.js"; -import { ContentAssetMetadataMap } from "../../types/index.js"; - -type StaticsMetadataModule = { - kvStoreName: string | null, - contentAssetMetadataMap: ContentAssetMetadataMap, -}; - -export async function cleanKVStore(commandLineValues: CommandLineOptions) { - - const { publishId } = generateOrLoadPublishId(); - - const errors: string[] = []; - const { normalized: config } = await loadConfigFile(errors); - - if (config == null) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - for (const error of errors) { - console.error(error); - } - process.exitCode = 1; - return; - } - - let fastlyApiContext: FastlyApiContext | null = null; - - const apiKeyResult = loadApiKey(); - if (apiKeyResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Specify one on the command line, or use the FASTLY_API_TOKEN environment variable."); - process.exitCode = 1; - return; - } - - fastlyApiContext = { apiToken: apiKeyResult.apiToken }; - - const staticContentRootDir = config.staticContentRootDir; - - const staticsMetadata: StaticsMetadataModule = await import(path.resolve(`${staticContentRootDir}/statics-metadata.js`)); - - const { kvStoreName, contentAssetMetadataMap } = staticsMetadata; - - if (kvStoreName == null) { - console.error("❌ KV Store not specified."); - console.error("This only has meaning in KV Store mode."); - process.exitCode = 1; - return; - } - - // TODO: Enable getting kvStoreName and publishId from command line - - // These are the items that are currently in the KV Store and that belong to this publish ID. - const items = ((await getKVStoreKeys(fastlyApiContext, kvStoreName)) ?? []) - .filter(x => x.startsWith(`${publishId}:`)); - - // These are the items that are currently are being used. - const keys = getKVStoreKeysFromMetadata(contentAssetMetadataMap); - - // So these are the items that we should be deleting. - const itemsToDelete = items.filter(x => !keys.has(x)); - - console.log("Publish ID: " + publishId); - console.log("KV Store contains " + items.length + " item(s) for this publish ID."); - console.log("Current site metadata contains " + keys.size + " item(s) (including compressed alternates)."); - - console.log("Number of items to delete: " + itemsToDelete.length); - - for (const [index, item] of itemsToDelete.entries()) { - console.log("Deleting item [" + (index+1) + "]: " + item); - await kvStoreDeleteFile(fastlyApiContext, kvStoreName, item); - } - - console.log("βœ… Completed.") -} diff --git a/src/cli/commands/init-app.ts b/src/cli/commands/init-app.ts deleted file mode 100644 index 35a0025..0000000 --- a/src/cli/commands/init-app.ts +++ /dev/null @@ -1,735 +0,0 @@ -import { CommandLineOptions } from "command-line-args"; - -// This program creates a Fastly Compute JavaScript application -// in a subfolder named compute-js. -// This project can be served using fastly compute serve -// or deployed to a Compute service using fastly compute publish. - -import * as child_process from "child_process"; -import * as path from "path"; -import * as fs from "fs"; -import * as url from 'url'; - -import { AppOptions, IPresetBase } from '../presets/preset-base.js'; -import { presets } from '../presets/index.js'; - -const defaultOptions: AppOptions = { - rootDir: undefined, - publicDir: undefined, - staticDirs: [], - staticContentRootDir: undefined, - spa: undefined, - notFoundPage: '[public-dir]/404.html', - autoIndex: [ 'index.html', 'index.htm' ], - autoExt: [ '.html', '.htm' ], - author: 'you@example.com', - name: 'compute-js-static-site', - description: 'Fastly Compute static site', - serviceId: undefined, - kvStoreName: undefined, -}; - -// Current directory of this program that's running. -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); - -function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial { - - // All paths are relative to CWD. - - let preset: string | undefined; - { - const presetValue = commandLineValues['preset']; - if (presetValue == null || typeof presetValue === 'string') { - preset = presetValue; - } - } - - let rootDir: string | undefined; - let publicDir: string | undefined; - { - const rootDirValue = commandLineValues['root-dir']; - if (rootDirValue == null || typeof rootDirValue === 'string') { - rootDir = rootDirValue; - } - if (rootDir != null) { - rootDir = path.resolve(rootDir); - } - - const publicDirValue = commandLineValues['public-dir']; - if (publicDirValue == null || typeof publicDirValue === 'string') { - publicDir = publicDirValue; - } - if (publicDir != null) { - publicDir = path.resolve(publicDir); - } - - // If we don't have a preset, then for backwards compatibility - // we check if we have public-dir but no root-dir. If that is - // the case then we use the value public-dir as root-dir. - if (preset == null && rootDir == null && publicDir != null) { - rootDir = publicDir; - publicDir = undefined; - } - - } - - // Filepaths provided on the command line are always given relative to CWD, - // so we need to resolve them. - - let staticDirs: string[] | undefined; - { - const staticDirsValue = commandLineValues['static-dir']; - - const asArray = Array.isArray(staticDirsValue) ? staticDirsValue : [ staticDirsValue ]; - if (asArray.every((x: any) => typeof x === 'string')) { - staticDirs = (asArray as string[]).map(x => path.resolve(x)); - } - } - - let staticContentRootDir: string | undefined; - { - const staticContentRootDirValue = commandLineValues['static-content-root-dir']; - if (staticContentRootDirValue == null || typeof staticContentRootDirValue === 'string') { - staticContentRootDir = staticContentRootDirValue; - } - if (staticContentRootDir != null) { - staticContentRootDir = path.resolve(staticContentRootDir); - } - } - - let spa: string | undefined; - { - const spaValue = commandLineValues['spa']; - if (spaValue === null) { - // If 'spa' is provided with a null value, then the flag was provided - // with no value. Assumed to be './index.html' relative to the public directory. - spa = '[public-dir]/index.html'; - console.log('--spa provided, but no value specified. Assuming ' + spa); - } else if (spaValue == null || typeof spaValue === 'string') { - spa = spaValue; - } - } - - let notFoundPage: string | undefined; - { - const notFoundPageValue = commandLineValues['not-found-page']; - if (notFoundPageValue === null) { - // If 'spa' is provided with a null value, then the flag was provided - // with no value. Assumed to be './404.html' relative to the public directory. - notFoundPage = '[public-dir]/404.html'; - console.log('--not-found-page provided, but no value specified. Assuming ' + notFoundPage); - } else if (notFoundPageValue == null || typeof notFoundPageValue === 'string') { - notFoundPage = notFoundPageValue; - } - } - - let autoIndex: string[] | undefined; - { - const autoIndexValue = commandLineValues['auto-index']; - - const asArray = Array.isArray(autoIndexValue) ? autoIndexValue : [ autoIndexValue ]; - if (asArray.every((x: any) => typeof x === 'string')) { - - autoIndex = (asArray as string[]).reduce((acc, entry) => { - - const segments = entry - .split(',') - .map(x => x.trim()) - .filter(x => Boolean(x)); - - for (const segment of segments) { - acc.push(segment); - } - - return acc; - }, []); - - } - } - - let autoExt: string[] = []; - { - const autoExtValue = commandLineValues['auto-ext']; - - const asArray = Array.isArray(autoExtValue) ? autoExtValue : [ autoExtValue ]; - if (asArray.every((x: any) => typeof x === 'string')) { - - autoExt = (asArray as string[]).reduce((acc, entry) => { - - const segments = entry - .split(',') - .map(x => x.trim()) - .filter(x => Boolean(x)) - .map(x => !x.startsWith('.') ? '.' + x : x); - - for (const segment of segments) { - acc.push(segment); - } - - return acc; - }, []); - - } - } - - let name: string | undefined; - { - const nameValue = commandLineValues['name']; - if (nameValue == null || typeof nameValue === 'string') { - name = nameValue; - } - } - - let author: string | undefined; - { - const authorValue = commandLineValues['author']; - if (authorValue == null || typeof authorValue === 'string') { - author = authorValue; - } - } - - let description: string | undefined; - { - const descriptionValue = commandLineValues['description']; - if (descriptionValue == null || typeof descriptionValue === 'string') { - description = descriptionValue; - } - } - - let serviceId: string | undefined; - { - const serviceIdValue = commandLineValues['service-id']; - if (serviceIdValue == null || typeof serviceIdValue === 'string') { - serviceId = serviceIdValue; - } - } - - let kvStoreName: string | undefined; - { - const kvStoreNameValue = commandLineValues['kv-store-name']; - if (kvStoreNameValue == null || typeof kvStoreNameValue === 'string') { - kvStoreName = kvStoreNameValue; - } - } - - return { - rootDir, - publicDir, - staticDirs, - staticContentRootDir, - spa, - notFoundPage, - autoIndex, - autoExt, - name, - author, - description, - serviceId, - kvStoreName, - }; - -} - -function pickKeys>(keys: (keyof TModel)[], object: TModel): Partial { - - const result: Partial = {}; - - for (const key of keys) { - if(object[key] !== undefined) { - result[key] = object[key]; - } - } - - return result; - -} - -const PUBLIC_DIR_TOKEN = '[public-dir]'; -function processPublicDirToken(filepath: string, publicDir: string) { - if (!filepath.startsWith(PUBLIC_DIR_TOKEN)) { - return filepath; - } - - const processedPath = '.' + filepath.slice(PUBLIC_DIR_TOKEN.length); - const resolvedPath = path.resolve(publicDir, processedPath) - return path.relative(path.resolve(), resolvedPath); -} - -export function initApp(commandLineValues: CommandLineOptions) { - - let options: AppOptions = defaultOptions; - let preset: IPresetBase | null = null; - - const presetName = (commandLineValues['preset'] as string | null) ?? 'none'; - if(presetName !== 'none') { - const presetClass = presets[presetName]; - if(presetClass == null) { - console.error('Unknown preset name.'); - console.error("--preset must be one of: none, " + (Object.keys(presets).join(', '))); - process.exitCode = 1; - return; - } - preset = new presetClass(); - } - - let packageJson; - try { - const packageJsonText = fs.readFileSync("./package.json", "utf-8"); - packageJson = JSON.parse(packageJsonText); - } catch { - console.log("Can't read/parse package.json in current directory, making no assumptions!"); - packageJson = null; - } - - // Get the current compute js static publisher version. - let computeJsStaticPublisherVersion: string | null = null; - if (packageJson != null) { - // First try current project's package.json - computeJsStaticPublisherVersion = - packageJson.dependencies?.["@fastly/compute-js-static-publish"] ?? - packageJson.devDependencies?.["@fastly/compute-js-static-publish"]; - - // This may be a file url if during development - if (computeJsStaticPublisherVersion != null) { - - if (computeJsStaticPublisherVersion.startsWith('file:')) { - // this is a relative path from the current directory. - // we replace it with an absolute path - const relPath = computeJsStaticPublisherVersion.slice('file:'.length); - const absPath = path.resolve(relPath); - computeJsStaticPublisherVersion = 'file:' + absPath; - } - - } - } - - if (computeJsStaticPublisherVersion == null) { - // Also try package.json of the package that contains the currently running program - // This is used when the program doesn't actually install the package (running via npx). - const computeJsStaticPublishPackageJsonPath = path.resolve(__dirname, '../../../package.json'); - const computeJsStaticPublishPackageJsonText = fs.readFileSync(computeJsStaticPublishPackageJsonPath, 'utf-8'); - const computeJsStaticPublishPackageJson = JSON.parse(computeJsStaticPublishPackageJsonText); - computeJsStaticPublisherVersion = computeJsStaticPublishPackageJson?.version; - } - - if (computeJsStaticPublisherVersion == null) { - // Unexpected, but if it's still null then we go to a literal - computeJsStaticPublisherVersion = '^4.0.0'; - } - - if (!computeJsStaticPublisherVersion.startsWith('^') && - !computeJsStaticPublisherVersion.startsWith('file:') - ) { - computeJsStaticPublisherVersion = '^' + computeJsStaticPublisherVersion; - } - - const commandLineAppOptions = processCommandLineArgs(commandLineValues); - - type PackageJsonAppOptions = Pick; - - options = { - ...options, - ...(preset != null ? preset.defaultOptions : {}), - ...pickKeys(['author', 'name', 'description'], (packageJson ?? {}) as PackageJsonAppOptions), - ...pickKeys(['rootDir', 'publicDir', 'staticDirs', 'staticContentRootDir', 'spa', 'notFoundPage', 'autoIndex', 'autoExt', 'author', 'name', 'description', 'serviceId', 'kvStoreName'], commandLineAppOptions), - }; - - if(preset != null) { - if(!preset.check(packageJson, options)) { - console.log("Failed preset check."); - process.exitCode = 1; - return; - } - } - - // Webpack now optional as of v4 - const useWebpack = commandLineValues['webpack'] as boolean; - - const COMPUTE_JS_DIR = commandLineValues.output as string; - const computeJsDir = path.resolve(COMPUTE_JS_DIR); - - // Resolve the root dir, relative to current directory, and make sure it exists. - const ROOT_DIR = options.rootDir; - if (ROOT_DIR == null) { - console.error("❌ required parameter --root-dir not provided."); - process.exitCode = 1; - return; - } - const rootDir = path.resolve(ROOT_DIR); - if (!fs.existsSync(rootDir) || !fs.statSync(rootDir).isDirectory()) { - console.error(`❌ Specified root directory '${ROOT_DIR}' does not exist.`); - console.error(` * ${rootDir} must exist and be a directory.`); - process.exitCode = 1; - return; - } - - // Resolve the public dir as well. If it's not specified, we use the root directory. - const PUBLIC_DIR = options.publicDir ?? ROOT_DIR; - const publicDir = path.resolve(PUBLIC_DIR); - if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) { - console.error(`❌ Specified public directory '${PUBLIC_DIR}' does not exist.`); - console.error(` * ${publicDir} must exist and be a directory.`); - process.exitCode = 1; - return; - } - - // Public dir must be inside the root dir. - if (!publicDir.startsWith(rootDir)) { - console.error(`❌ Specified public directory '${PUBLIC_DIR}' is not under the asset root directory.`); - console.error(` * ${publicDir} must be under ${rootDir}`); - process.exitCode = 1; - return; - } - - // Static dirs must be inside the public dir. - const STATIC_DIRS = options.staticDirs; - const staticDirs: string[] = []; - for (const STATIC_DIR of STATIC_DIRS) { - // For backwards compatibility, these values can start with [public-dir] - const staticDir = path.resolve(processPublicDirToken(STATIC_DIR, publicDir)); - if (!staticDir.startsWith(publicDir)) { - console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); - console.log(` * ${staticDir} is not under ${publicDir}`); - continue; - } - if (!fs.existsSync(staticDir) || !fs.statSync(staticDir).isDirectory()) { - console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); - console.log(` * ${staticDir} does not exist or is not a directory.`); - continue; - } - staticDirs.push(staticDir); - } - - // Static Content Root Dir must be under the current dir - let staticContentRootDir = options.staticContentRootDir; - if (staticContentRootDir == null) { - staticContentRootDir = path.resolve(computeJsDir, './static-publisher'); - } - if ( - !staticContentRootDir.startsWith(computeJsDir) || - staticContentRootDir === computeJsDir || - staticContentRootDir === path.resolve(computeJsDir, './bin') || - staticContentRootDir === path.resolve(computeJsDir, './pkg') || - staticContentRootDir === path.resolve(computeJsDir, './node_modules') - ) { - console.error(`❌ Specified static content root directory '${staticContentRootDir}' must be under ${computeJsDir}.`); - console.error(` It also must not be bin, pkg, or node_modules.`); - process.exitCode = 1; - return; - } - staticContentRootDir = './' + path.relative(computeJsDir, staticContentRootDir); - - // SPA and Not Found are relative to the asset root dir. - - const SPA = options.spa; - let spaFilename: string | undefined; - if (SPA != null) { - // If it starts with [public-dir], then resolve it relative to public directory. - spaFilename = path.resolve(processPublicDirToken(SPA, publicDir)); - // At any rate it must exist under the root directory - if (!spaFilename.startsWith(rootDir)) { - console.log(`⚠️ Ignoring specified SPA file '${SPA}' as is not under the asset root directory.`); - console.log(` * ${spaFilename} is not under ${rootDir}`); - spaFilename = undefined; - } else if (!fs.existsSync(spaFilename)) { - console.log(`⚠️ Ignoring specified SPA file '${SPA}' as it does not exist.`); - console.log(` * ${spaFilename} does not exist.`); - spaFilename = undefined; - } - } - - const NOT_FOUND_PAGE = options.notFoundPage; - let notFoundPageFilename: string | undefined; - if (NOT_FOUND_PAGE != null) { - // If it starts with [public-dir], then resolve it relative to public directory. - notFoundPageFilename = path.resolve(processPublicDirToken(NOT_FOUND_PAGE, publicDir)); - // At any rate it must exist under the root directory - if (!notFoundPageFilename.startsWith(rootDir)) { - console.log(`⚠️ Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as is not under the asset root directory.`); - console.log(` * ${notFoundPageFilename} is not under ${rootDir}`); - notFoundPageFilename = undefined; - } else if (!fs.existsSync(notFoundPageFilename)) { - console.log(`⚠️ Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as it does not exist.`); - console.log(` * ${notFoundPageFilename} does not exist.`); - notFoundPageFilename = undefined; - } - } - - const autoIndex = options.autoIndex; - const autoExt = options.autoExt; - - const exists = fs.existsSync(computeJsDir); - if(exists) { - console.error(`❌ '${COMPUTE_JS_DIR}' directory already exists!`); - console.error(` You should not run this command if this directory exists.`); - console.error(` If you need to re-scaffold the static publisher, delete the following directory and then try again:`); - console.error(` ${computeJsDir}`); - process.exitCode = 1; - return; - } - - const author = options.author; - const name = options.name; - const description = options.description; - const fastlyServiceId = options.serviceId; - const kvStoreName = options.kvStoreName; - - function rootRelative(itemPath: string | null | undefined) { - if (itemPath == null) { - return null; - } - const v = path.relative(path.resolve(), itemPath); - return v.startsWith('..') ? v : './' + v; - } - - console.log(''); - console.log('Asset Root Dir :', rootRelative(rootDir)); - console.log('Public Dir :', rootRelative(publicDir)); - console.log('Static Dir :', staticDirs.length > 0 ? staticDirs.map(rootRelative) : '(None)'); - console.log('Static Content Root Dir :', staticContentRootDir); - console.log('SPA :', rootRelative(spaFilename) ?? '(None)'); - console.log('404 Page :', rootRelative(notFoundPageFilename) ?? '(None)'); - console.log('Auto-Index :', autoIndex.length > 0 ? autoIndex.map(rootRelative) : '(None)') - console.log('Auto-Ext :', autoExt.length > 0 ? autoExt.map(rootRelative) : '(None)') - console.log('name :', name); - console.log('author :', author); - console.log('description :', description); - console.log('Service ID :', fastlyServiceId ?? '(None)'); - console.log('KV Store Name :', kvStoreName ?? '(None)'); - console.log(''); - if (useWebpack) { - console.log('Creating project with webpack.'); - console.log(''); - } - - console.log("Initializing Compute Application in " + computeJsDir + "..."); - fs.mkdirSync(computeJsDir); - fs.mkdirSync(path.resolve(computeJsDir, './src')); - - const staticContentRootDirFromRoot = staticContentRootDir.slice(1); - - let staticFiles = staticContentRootDirFromRoot; - if (staticContentRootDirFromRoot === '/src') { - staticFiles = `\ -${staticContentRootDirFromRoot}/statics.js -${staticContentRootDirFromRoot}/statics.d.ts -${staticContentRootDirFromRoot}/statics-metadata.js -${staticContentRootDirFromRoot}/statics-metadata.d.ts -${staticContentRootDirFromRoot}/static-content`; - } - - // .gitignore - const gitIgnoreContent = `\ -/node_modules -/bin -/pkg -${staticFiles} -`; - const gitIgnorePath = path.resolve(computeJsDir, '.gitignore'); - fs.writeFileSync(gitIgnorePath, gitIgnoreContent, "utf-8"); - - // package.json - const packageJsonContent: Record = { - name, - version: '0.1.0', - description, - author, - type: 'module', - devDependencies: { - "@fastly/cli": "^10.14.0", - '@fastly/compute-js-static-publish': computeJsStaticPublisherVersion, - }, - dependencies: { - '@fastly/js-compute': '^3.0.0', - }, - engines: { - node: '>=20.0.0', - }, - license: 'UNLICENSED', - private: true, - scripts: { - start: "fastly compute serve", - deploy: "fastly compute publish", - prebuild: 'npx @fastly/compute-js-static-publish --build-static', - build: 'js-compute-runtime ./src/index.js ./bin/main.wasm' - }, - }; - - if (useWebpack) { - delete packageJsonContent.type; - packageJsonContent.devDependencies = { - ...packageJsonContent.devDependencies, - 'webpack': '^5.75.0', - 'webpack-cli': '^5.0.0', - }; - packageJsonContent.scripts = { - ...packageJsonContent.scripts, - prebuild: 'npx @fastly/compute-js-static-publish --build-static && webpack', - build: 'js-compute-runtime ./bin/index.js ./bin/main.wasm' - }; - } - - const packageJsonContentJson = JSON.stringify(packageJsonContent, undefined, 2); - const packageJsonPath = path.resolve(computeJsDir, 'package.json'); - fs.writeFileSync(packageJsonPath, packageJsonContentJson, "utf-8"); - - // fastly.toml - - // language=toml - const fastlyTomlContent = `\ -# This file describes a Fastly Compute package. To learn more visit: -# https://developer.fastly.com/reference/fastly-toml/ - -authors = [ "${author}" ] -description = "${description}" -language = "javascript" -manifest_version = 2 -name = "${name}" -${fastlyServiceId != null ? `service_id = "${fastlyServiceId}" -` : ''} -[scripts] - build = "npm run build" - `; - - const fastlyTomlPath = path.resolve(computeJsDir, 'fastly.toml'); - fs.writeFileSync(fastlyTomlPath, fastlyTomlContent, "utf-8"); - - // static-publish.rc.js - const rootDirRel = path.relative(computeJsDir, rootDir); - - // publicDirPrefix -- if public dir is deeper than the root dir, then - // public dir is used as a prefix to drill into asset names. - // e.g., - // root dir : /path/to/root - // public dir : /path/to/root/public - // then publicDirPrefix = /public - - // We've already established publicDir.startsWith(rootDir) - let publicDirPrefix = ''; - if (rootDir !== publicDir) { - publicDirPrefix = rootDir.slice(rootDir.length); - } - - // staticItems - specified as prefixes, relative to publicDir - const staticItems: string[] = []; - for (const staticDir of staticDirs) { - // We've already established staticDir.startsWith(publicDir) - let staticItem = staticDir.slice(publicDir.length); - if (!staticItem.endsWith('/')) { - // Ending with a slash denotes that this is a directory. - staticItem = staticItem + '/'; - } - staticItems.push(staticItem); - } - - // spaFile - asset key of spa file - let spaFile: string | false = false; - if (spaFilename != null) { - // We've already established spaFilename.startsWith(rootDir) - // and that it exists - spaFile = spaFilename.slice(rootDir.length); - } - - let notFoundPageFile: string | false = false; - if(notFoundPageFilename != null) { - // We've already established notFoundPageFilename.startsWith(rootDir) - // and that it exists - notFoundPageFile = notFoundPageFilename.slice(rootDir.length); - } - - const staticPublishJsContent = `\ -/* - * Copyright Fastly, Inc. - * Licensed under the MIT license. See LICENSE file for details. - */ - -// Commented items are defaults, feel free to modify and experiment! -// See README for a detailed explanation of the configuration options. - -/** @type {import('@fastly/compute-js-static-publish').StaticPublisherConfig} */ -const config = { - rootDir: ${JSON.stringify(rootDirRel)}, - staticContentRootDir: ${JSON.stringify(staticContentRootDir)}, - ${(kvStoreName != null ? 'kvStoreName: ' + JSON.stringify(kvStoreName) : '// kvStoreName: false')}, - // excludeDirs: [ './node_modules' ], - // excludeDotFiles: true, - // includeWellKnown: true, - // contentAssetInclusionTest: (filename) => true, - // contentCompression: [ 'br', 'gzip' ], // For this config value, default is [] if kvStoreName is null. - // moduleAssetInclusionTest: (filename) => false, - // contentTypes: [ - // { test: /.custom$/, contentType: 'application/x-custom', text: false }, - // ], - server: { - publicDirPrefix: ${JSON.stringify(publicDirPrefix)}, - staticItems: ${JSON.stringify(staticItems)}, - // compression: [ 'br', 'gzip' ], - spaFile: ${JSON.stringify(spaFile)}, - notFoundPageFile: ${JSON.stringify(notFoundPageFile)}, - autoExt: ${JSON.stringify(autoExt)}, - autoIndex: ${JSON.stringify(autoIndex)}, - }, -}; - -${useWebpack ? 'module.exports =' : 'export default'} config; -`; - - const staticPublishJsonPath = path.resolve(computeJsDir, 'static-publish.rc.js'); - fs.writeFileSync(staticPublishJsonPath, staticPublishJsContent, "utf-8"); - - // Copy resource files - - if (useWebpack) { - // webpack.config.js - const webpackConfigJsSrcPath = path.resolve(__dirname, '../../../resources/webpack.config.js'); - const webpackConfigJsPath = path.resolve(computeJsDir, 'webpack.config.js'); - fs.copyFileSync(webpackConfigJsSrcPath, webpackConfigJsPath); - } - - // src/index.js - const staticsRelativePath = staticContentRootDir !== './src' ? - path.relative('./src', staticContentRootDir) : - '.'; - const indexJsContent = /* language=JavaScript */ `\ -/// -import { getServer } from '${staticsRelativePath}/statics.js'; -const staticContentServer = getServer(); - -// eslint-disable-next-line no-restricted-globals -addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); -async function handleRequest(event) { - - const response = await staticContentServer.serveRequest(event.request); - if (response != null) { - return response; - } - - // Do custom things here! - // Handle API requests, serve non-static responses, etc. - - return new Response('Not found', { status: 404 }); -} -`; - const indexJsPath = path.resolve(computeJsDir, './src/index.js'); - fs.writeFileSync(indexJsPath, indexJsContent, 'utf-8'); - - console.log("πŸš€ Compute application created!"); - - console.log('Installing dependencies...'); - console.log(`npm --prefix ${COMPUTE_JS_DIR} install`); - child_process.spawnSync('npm', [ '--prefix', COMPUTE_JS_DIR, 'install' ], { stdio: 'inherit' }); - console.log(''); - - console.log(''); - console.log('To run your Compute application locally:'); - console.log(''); - console.log(' cd ' + COMPUTE_JS_DIR); - console.log(' npm run start'); - console.log(''); - console.log('To build and deploy to your Compute service:'); - console.log(''); - console.log(' cd ' + COMPUTE_JS_DIR); - console.log(' npm run deploy'); - console.log(''); - -} diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts new file mode 100644 index 0000000..e7e37c8 --- /dev/null +++ b/src/cli/commands/manage/clean.ts @@ -0,0 +1,360 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import path from 'node:path'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap } from '../../../models/assets/kvstore-assets.js'; +import { type IndexMetadata } from '../../../models/server/index.js'; +import { isExpired } from '../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../util/api-token.js'; +import { parseCommandLine } from '../../util/args.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; +import { readServiceId } from '../../util/fastly-toml.js'; +import { + getKvStoreEntry, + getKVStoreKeys, + kvStoreDeleteEntry, +} from '../../util/kv-store.js'; +import { doKvStoreItemsOperation } from '../../util/kv-store-items.js'; +import { + getLocalKvStoreEntry, + getLocalKVStoreKeys, + localKvStoreDeleteEntry, +} from '../../util/kv-store-local-server.js'; +import { isNodeError } from '../../util/node.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish clean [options] + +Description: + Cleans up expired or unreferenced items in the Fastly KV Store. + This can include expired collection indexes and orphaned content assets. + +Options: + --delete-expired-collections If set, expired collection index files will be deleted. + + --dry-run Show what would be deleted without performing any deletions. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'delete-expired-collections', type: Boolean }, + { name: 'dry-run', type: Boolean }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['delete-expired-collections']: deleteExpiredCollections, + ['dry-run']: dryRun, + local: localMode, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`🧹 Cleaning KV Store entries...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + // ### KVStore Keys to delete + const kvKeysToDelete = new Set(); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + let indexKeys; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Go through the index files, make lists + const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); + + // All collection names that we are keeping + const liveCollections = new Set(); + + // All asset keys (hashes) that we are keeping + const assetsIdsInUse = new Set(); + + console.log('Processing collections:'); + await doKvStoreItemsOperation( + foundCollections.map(collection => ({ + key: indexesPrefix + collection, + collection, + })), + async({collection}, indexKey) => { + console.log(`Collection: ${collection}`); + + let kvAssetsIndexResponse; + if (localMode) { + kvAssetsIndexResponse = await getLocalKvStoreEntry( + storeFile, + indexKey, + ); + } else { + kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + indexKey, + ); + } + if (!kvAssetsIndexResponse) { + throw new Error(`Can't load KV Store entry ${indexesPrefix + collection}`); + } + + let indexMetadata: IndexMetadata = {}; + if (kvAssetsIndexResponse.metadata != null) { + try { + indexMetadata = JSON.parse(kvAssetsIndexResponse.metadata) as IndexMetadata; + } catch { + } + } + + let isLive = true; + if (indexMetadata.expirationTime == null) { + console.log(' Collection has no expiration time set'); + } else { + console.log(' ⏰ Expiration: ' + new Date(indexMetadata.expirationTime * 1000)); + if (collection === defaultCollectionName) { + console.log(` βœ… Expiration time not enforced for default collection.`); + } else if (isExpired(indexMetadata.expirationTime)) { + if (!deleteExpiredCollections) { + console.log(` ⚠️ Use --delete-expired-collections to delete expired collections.`); + } else { + console.log(` πŸ—‘οΈ Marking expired collection for deletion.`); + kvKeysToDelete.add(indexKey); + isLive = false; + + } + } + } + if (!isLive) { + return; + } + + liveCollections.add(collection); + const kvAssetsIndex = (await kvAssetsIndexResponse.response.json()) as KVAssetEntryMap; + for (const [_assetKey, assetEntry] of Object.entries(kvAssetsIndex)) { + if (assetEntry.key.startsWith('sha256:')) { + assetsIdsInUse.add(`sha256_${assetEntry.key.slice('sha256:'.length)}`); + } + } + } + ) + console.log(''); + + // ### List all settings ### + console.log('Enumerating settings:'); + const settingsPrefix = publishId + '_settings_'; + let settingsKeys; + if (localMode) { + settingsKeys = await getLocalKVStoreKeys( + storeFile, + settingsPrefix, + ); + } else { + settingsKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + settingsPrefix, + ); + } + if (settingsKeys == null) { + throw new Error(`Can't query settings in KV Store`); + } + + // If a settings object is found that doesn't match a live collection name then + // mark it for deletion + const foundSettingsCollections = settingsKeys.map(key => ({ key, name: key.slice(settingsPrefix.length) })); + for (const foundSettings of foundSettingsCollections) { + console.log(`Settings: ${foundSettings.name}`); + if (!liveCollections.has(foundSettings.name)) { + console.log(` πŸ—‘οΈ Does not match a live index, marking for deletion.`); + kvKeysToDelete.add(foundSettings.key); + } + } + console.log(''); + + // ### Obtain the assets in the KV Store and find the ones that are not in use + console.log('Enumerating assets:'); + const assetPrefix = publishId + '_files_'; + let assetKeys; + if (localMode) { + assetKeys = await getLocalKVStoreKeys( + storeFile, + assetPrefix, + ); + } else { + assetKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + assetPrefix, + ); + } + if (assetKeys == null) { + throw new Error(`Can't query assets in KV Store`); + } + + for (const assetKey of assetKeys) { + let assetId = assetKey.slice(assetPrefix.length); + if (assetId.startsWith('sha256_')) { + assetId = assetId.slice(0, 'sha256_'.length + 64); + } else { + // If we don't know what the prefix is, we ignore it + continue; + } + + if (assetsIdsInUse.has(assetId)) { + console.log(` ${assetId}: in use`); + } else { + kvKeysToDelete.add(assetKey); + console.log(` ${assetId}: not in use - βœ… marking for deletion`); + } + } + console.log(''); + + // ### Delete items that have been flagged + const items = [...kvKeysToDelete].map(key => ({key})); + await doKvStoreItemsOperation( + items, + async(_, key) => { + if (dryRun) { + console.log(`[DRY RUN] Deleting item: ${key}`); + } else { + console.log(`Deleting item from KV Store: ${key}`); + if (localMode) { + await localKvStoreDeleteEntry( + storeFile, + key, + ); + } else { + await kvStoreDeleteEntry( + fastlyApiContext!, + kvStoreName, + key, + ); + } + } + } + ); + + console.log('βœ… Completed.') +} diff --git a/src/cli/commands/manage/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts new file mode 100644 index 0000000..688cb37 --- /dev/null +++ b/src/cli/commands/manage/collections/delete.ts @@ -0,0 +1,220 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import path from 'node:path'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; +import { parseCommandLine } from "../../../util/args.js"; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; +import { getKVStoreKeys, kvStoreDeleteEntry } from '../../../util/kv-store.js'; +import { doKvStoreItemsOperation } from '../../../util/kv-store-items.js'; +import { getLocalKVStoreKeys, localKvStoreDeleteEntry } from '../../../util/kv-store-local-server.js'; +import { isNodeError } from '../../../util/node.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections delete --collection-name= [options] + +Description: + Deletes a collection index from the KV Store. The content files will remain as they may still + be referenced by other collection indexes. + + Use the 'npx @fastly/compute-js-static-publish clean' command afterward to remove content + files that are no longer referenced by any collection. + +Required: + --collection-name= The name of the collection to delete + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'collection-name', type: String }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['collection-name']: collectionNameValue, + ['fastly-api-token']: fastlyApiToken, + local: localMode, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + if (collectionNameValue == null) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`🧹 Cleaning KV Store entries...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + const collectionName = collectionNameValue; + if (collectionName === defaultCollectionName) { + console.error(`❌ Cannot delete default collection: ${collectionName}`); + process.exitCode = 1; + return; + } + + console.log(`βœ”οΈ Collection to delete: ${collectionName}`); + + // ### KVStore Keys to delete + const kvKeysToDelete = new Set(); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + let indexKeys: string[] | null; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(key => ({ key, name: key.slice(indexesPrefix.length), })); + if (foundCollections.length === 0) { + console.log('No collections found.'); + } else { + console.log(`Found collections: ${foundCollections.map(x => `'${x.name}'`).join(', ')}`); + for (const collection of foundCollections) { + if (collection.name === collectionName) { + console.log(`Flagging collection '${collection.name}' for deletion: ${collection.key}`); + kvKeysToDelete.add(collection.key); + } + } + } + + // ### Delete items that have been flagged + const items = [...kvKeysToDelete].map(key => ({key})); + await doKvStoreItemsOperation( + items, + async(_, key) => { + console.log(`Deleting key from KV Store: ${key}`); + if (localMode) { + await localKvStoreDeleteEntry(storeFile, key); + } else { + await kvStoreDeleteEntry(fastlyApiContext!, kvStoreName, key); + } + } + ); + + console.log("βœ… Completed.") +} diff --git a/src/cli/commands/manage/collections/index.ts b/src/cli/commands/manage/collections/index.ts new file mode 100644 index 0000000..0e7a09f --- /dev/null +++ b/src/cli/commands/manage/collections/index.ts @@ -0,0 +1,72 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { getCommandAndArgs } from '../../../util/args.js'; +import * as deleteCommand from './delete.js'; +import * as listCommand from './list.js'; +import * as promoteCommand from './promote.js'; +import * as updateExpirationCommand from './update-expiration.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections [options] + +Description: + Manage named collections within a Compute application built with @fastly/compute-js-static-publish. + +Available Subcommands: + list List all published collections + delete Delete a specific collection index + promote Copies an existing collection (content + config) + to a new collection name + update-expiration Modify expiration time for an existing collection + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. + +Examples: + npx @fastly/compute-js-static-publish collections list + npx @fastly/compute-js-static-publish collections delete --collection-name=preview-42 +`); +} + +export async function action(actionArgs: string[]) { + + const modes = { + 'delete': deleteCommand, + 'list': listCommand, + 'promote': promoteCommand, + 'update-expiration': updateExpirationCommand, + }; + + const commandAndArgs = getCommandAndArgs(actionArgs, modes); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.error != null) { + console.error(commandAndArgs.error); + console.error(`Specify one of the following sub-commands: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); + +} diff --git a/src/cli/commands/manage/collections/list.ts b/src/cli/commands/manage/collections/list.ts new file mode 100644 index 0000000..0b3d65a --- /dev/null +++ b/src/cli/commands/manage/collections/list.ts @@ -0,0 +1,225 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import path from 'node:path'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { isExpired } from '../../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; +import { parseCommandLine } from '../../../util/args.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; +import { getKvStoreEntry, getKVStoreKeys } from '../../../util/kv-store.js'; +import { isNodeError } from '../../../util/node.js'; +import { getLocalKvStoreEntry, getLocalKVStoreKeys } from "../../../util/kv-store-local-server.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections list [options] + +Description: + Lists all collections currently published in the KV Store. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + local: localMode, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`πŸ“ƒ Listing collections...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + let indexKeys: string[] | null; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } + if (indexKeys == null) { + throw new Error(`❌ Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); + if (foundCollections.length === 0) { + console.log('No collections found.'); + } else { + console.log(`Found collections:`); + for (const collection of foundCollections) { + if (collection === defaultCollectionName) { + console.log(` ${collection} *DEFAULT*`); + } else { + console.log(` ${collection}`); + } + const indexKey = indexesPrefix + collection; + let kvAssetsIndexResponse; + if (localMode) { + kvAssetsIndexResponse = await getLocalKvStoreEntry( + storeFile, + indexKey, + ); + } else { + kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + indexKey, + ); + } + if (kvAssetsIndexResponse == null) { + throw new Error(`❌ Can't load KV Store entry ${indexesPrefix + collection}`); + } + let indexMetadata: IndexMetadata | undefined; + if (kvAssetsIndexResponse.metadata != null) { + try { + indexMetadata = JSON.parse(kvAssetsIndexResponse.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata == null) { + console.log(` No metadata found.`); + continue; + } + if (indexMetadata.publishedTime == null) { + console.log(' Published : unknown'); + } else { + console.log(` Published : ${new Date(indexMetadata.publishedTime * 1000)}`); + } + + if (indexMetadata.expirationTime == null) { + console.log(' Expiration : not set'); + } else { + console.log(` Expiration : ${new Date(indexMetadata.expirationTime * 1000)}`); + if (collection === defaultCollectionName) { + console.log(` βœ… Expiration time not enforced for default collection.`); + } else if (isExpired(indexMetadata.expirationTime)) { + console.log(` ⚠️ EXPIRED - Use 'clean --delete-expired-collections' to`); + console.log(` remove expired collections.`); + } + } + } + } + + console.log("βœ… Completed.") +} diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts new file mode 100644 index 0000000..93e5319 --- /dev/null +++ b/src/cli/commands/manage/collections/promote.ts @@ -0,0 +1,301 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { calcExpirationTime } from '../../../../models/time/index.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { + getKvStoreEntry, + kvStoreSubmitEntry, +} from '../../../util/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; +import { parseCommandLine } from '../../../util/args.js'; +import path from "node:path"; +import { readServiceId } from "../../../util/fastly-toml.js"; +import { isNodeError } from "../../../util/node.js"; +import { getLocalKvStoreEntry, localKvStoreSubmitEntry } from "../../../util/kv-store-local-server.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections promote + --collection-name= \\ + --to=name> \\ + [options] + +Description: + Copies an existing collection (content + config) to a new collection name. + +Required: + --collection-name= The name of the collection to promote + --to= The name of the new (target) collection to create or overwrite + +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify no more than one. If not provided, then the + existing expiration rule of the collection being + promoted is used. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'collection-name', type: String }, + { name: 'to', type: String }, + + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['collection-name']: collectionNameValue, + to: toCollectionNameValue, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, + ['expires-never']: expiresNever, + local: localMode, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + if (collectionNameValue == null) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + if (toCollectionNameValue == null) { + console.error("❌ Required argument '--to' not specified."); + process.exitCode = 1; + return; + } + + let expirationTime: number | null | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`πŸ“ƒ Promoting collection...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + const sourceCollectionName = collectionNameValue; + console.log(`βœ”οΈ Collection to copy: ${sourceCollectionName}`); + + const targetCollectionName = toCollectionNameValue; + console.log(`βœ”οΈ Collection to promote to: ${targetCollectionName}`) + + if (expirationTime === undefined) { + console.log(`βœ”οΈ Not updating expiration timestamp.`); + } else if (expirationTime === null) { + console.log(`βœ”οΈ Updating expiration timestamp: never`); + } else { + console.log(`βœ”οΈ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } + if (targetCollectionName === defaultCollectionName && expirationTime !== undefined) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + + const sourceCollectionIndexKey = `${publishId}_index_${sourceCollectionName}`; + const targetCollectionIndexKey = `${publishId}_index_${targetCollectionName}`; + + const sourceCollectionSettingsKey = `${publishId}_settings_${collectionNameValue}`; + const targetCollectionSettingsKey = `${publishId}_settings_${targetCollectionName}`; + + let indexEntryInfo, settingsEntryInfo; + if (localMode) { + [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ + getLocalKvStoreEntry(storeFile, sourceCollectionIndexKey), + getLocalKvStoreEntry(storeFile, sourceCollectionSettingsKey), + ]); + } else { + [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ + getKvStoreEntry(fastlyApiContext!, kvStoreName, sourceCollectionIndexKey), + getKvStoreEntry(fastlyApiContext!, kvStoreName, sourceCollectionSettingsKey), + ]); + } + if (!indexEntryInfo) { + throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); + } + if (!settingsEntryInfo) { + throw new Error(`Error querying settings for '${collectionNameValue}' in KV Store`); + } + + let indexMetadata: IndexMetadata = {}; + if (indexEntryInfo.metadata != null) { + try { + indexMetadata = JSON.parse(indexEntryInfo.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata.publishedTime == null) { + indexMetadata.publishedTime = Math.floor(Date.now() / 1000); + } + if (expirationTime !== undefined) { + if (expirationTime === null) { + delete indexMetadata.expirationTime; + } else { + indexMetadata.expirationTime = expirationTime; + } + } + + console.log(`Uploading to KV Store: '${targetCollectionName}'`); + + if (localMode) { + + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + const indexFileName = `index_${collectionNameValue}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + const indexBody = await indexEntryInfo.response.arrayBuffer(); + fs.writeFileSync(indexFilePath, Buffer.from(indexBody)); + await localKvStoreSubmitEntry( + storeFile, + targetCollectionIndexKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + + const settingsFileName = `settings_${collectionNameValue}.json`; + const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); + const settingsBody = await settingsEntryInfo.response.arrayBuffer(); + fs.writeFileSync(settingsFilePath, Buffer.from(settingsBody)); + await localKvStoreSubmitEntry( + storeFile, + targetCollectionSettingsKey, + path.relative(computeAppDir, settingsFilePath), + undefined, + ); + + } else { + await Promise.all([ + kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, targetCollectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)), + kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, targetCollectionSettingsKey, settingsEntryInfo.response.body!, undefined), + ]); + } + + console.log("βœ… Completed."); +} diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts new file mode 100644 index 0000000..10f155c --- /dev/null +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -0,0 +1,270 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { calcExpirationTime } from '../../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { parseCommandLine } from '../../../util/args.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; +import { + getKvStoreEntry, + kvStoreSubmitEntry, +} from '../../../util/kv-store.js'; +import { + getLocalKvStoreEntry, + localKvStoreSubmitEntry, +} from '../../../util/kv-store-local-server.js'; +import { isNodeError } from '../../../util/node.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections update-expiration \\ + --collection-name= \\ + [options] + +Description: + Sets or updates the expiration time of an existing collection. + +Required: + --collection-name= The name of the collection to modify + +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify exactly one. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'collection-name', type: String, }, + + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['collection-name']: collectionNameValue, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, + ['expires-never']: expiresNever, + local: localMode, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + if (collectionNameValue == null) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + let expirationTime: number | null | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + if (expirationTime === undefined) { + console.error("❌ Exactly one of '--expires-in', '--expires-at', or '--expires-never' is required."); + process.exitCode = 1; + return; + } + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`πŸ“ƒ Promoting collection...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + const collectionName = collectionNameValue; + console.log(`βœ”οΈ Collection to update: ${collectionName}`); + + if (expirationTime !== null) { + console.log(`βœ”οΈ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } else { + console.log(`βœ”οΈ Updating expiration timestamp: never`); + } + if (collectionName === defaultCollectionName && expirationTime != null) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + + const collectionIndexKey = `${publishId}_index_${collectionName}`; + + let indexEntryInfo; + if (localMode) { + indexEntryInfo = await getLocalKvStoreEntry( + storeFile, + collectionIndexKey, + ); + } else { + indexEntryInfo = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + collectionIndexKey, + ); + } + if (!indexEntryInfo) { + throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); + } + + let indexMetadata: IndexMetadata = {}; + if (indexEntryInfo.metadata != null) { + try { + indexMetadata = JSON.parse(indexEntryInfo.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata.publishedTime == null) { + indexMetadata.publishedTime = Math.floor(Date.now() / 1000); + } + if (expirationTime === null) { + delete indexMetadata.expirationTime; + } else { + indexMetadata.expirationTime = expirationTime; + } + + console.log(`Uploading to KV Store: '${collectionName}'`); + + if (localMode) { + + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + const indexFileName = `index_${collectionNameValue}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + const indexBody = await indexEntryInfo.response.arrayBuffer(); + fs.writeFileSync(indexFilePath, Buffer.from(indexBody)); + await localKvStoreSubmitEntry( + storeFile, + collectionIndexKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + + } else { + await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, collectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)); + } + + console.log("βœ… Completed."); +} diff --git a/src/cli/commands/manage/index.ts b/src/cli/commands/manage/index.ts new file mode 100644 index 0000000..34f5b6b --- /dev/null +++ b/src/cli/commands/manage/index.ts @@ -0,0 +1,78 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { getCommandAndArgs } from '../../util/args.js'; +import * as cleanCommand from './clean.js'; +import * as publishContentCommand from './publish-content.js'; +import * as collectionsCommands from './collections/index.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish [options] + +Description: + Manage and publish static content to Fastly Compute using KV Store-backed collections. + + Note: If run outside a scaffolded project, this tool will automatically enter scaffolding mode. + +Available Commands: + publish-content Publish static files to the KV Store under a named collection + clean Remove expired collections and unused KV Store content + collections list List all published collections + collections delete Delete a specific collection index + collections promote Copy a collection to another name + collections update-expiration Modify expiration time for an existing collection + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. + +Automatic Project Initialization: + If run in a directory that does not contain a \`static-publish.rc.js\` file, this tool will scaffold a new + Compute application for you, including Fastly configuration, default routes, and publishing setup. + +Examples: + npx @fastly/compute-js-static-publish publish-content --collection-name=live + npx @fastly/compute-js-static-publish collections list + npx @fastly/compute-js-static-publish clean --dry-run +`); +} + +export async function action(actionArgs: string[]) { + + const modes = { + 'clean': cleanCommand, + 'publish-content': publishContentCommand, + 'collections': collectionsCommands, + }; + + const commandAndArgs = getCommandAndArgs(actionArgs, modes); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.error != null) { + console.error(commandAndArgs.error); + console.error(`Specify one of the following commands: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); + +} diff --git a/src/cli/commands/manage/publish-content.ts b/src/cli/commands/manage/publish-content.ts new file mode 100644 index 0000000..9034fe3 --- /dev/null +++ b/src/cli/commands/manage/publish-content.ts @@ -0,0 +1,701 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap, type KVAssetVariantMetadata, isKVAssetVariantMetadata } from '../../../models/assets/kvstore-assets.js'; +import { type ContentCompressionTypes } from '../../../models/compression/index.js'; +import { type PublisherServerConfigNormalized } from '../../../models/config/publisher-server-config.js'; +import { type ContentTypeDef } from '../../../models/config/publish-content-config.js'; +import { type IndexMetadata } from '../../../models/server/index.js'; +import { calcExpirationTime } from '../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../util/api-token.js'; +import { getKvStoreEntryInfo, kvStoreSubmitEntry } from '../../util/kv-store.js'; +import { parseCommandLine } from '../../util/args.js'; +import { mergeContentTypes, testFileContentType } from '../../util/content-types.js'; +import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../../util/config.js'; +import { applyDefaults } from '../../util/data.js'; +import { readServiceId } from '../../util/fastly-toml.js'; +import { calculateFileSizeAndHash, enumerateFiles, rootRelative } from '../../util/files.js'; +import { + applyKVStoreEntriesChunks, + doKvStoreItemsOperation, + type KVStoreItemDesc, +} from '../../util/kv-store-items.js'; +import { + localKvStoreSubmitEntry, + writeKVStoreEntriesForLocal, +} from '../../util/kv-store-local-server.js'; +import { isNodeError } from '../../util/node.js'; +import { ensureVariantFileExists, type Variants } from '../../util/variants.js'; + +// KV Store key format: +// _index_.json +// _settings_.json +// _files_sha256__ + +// split large files into 20MiB chunks +const KV_STORE_CHUNK_SIZE = 1_024 * 1_024 * 20; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish publish-content [--collection-name=] [options] + +Description: + Publishes static files from your local root directory into a named collection, + either in the Fastly KV Store (default) or to a local dev directory (--local). + Files that already exist with the same hash are skipped automatically. + + After this process is complete, the PublisherServer object in the Compute application + will see the updated index of files and updated server settings from the + publish-content.config.js file. + +Optional: + --collection-name= Name of the collection to publish into. + Default: value from static-publisher.rc.js (defaultCollectionName) + + --config= Path to a publish-content.config.js file. + Default: ./publish-content.config.js + + --root-dir= Directory to publish from. Overrides the config file setting. + Default: rootDir from publish-content.config.js + + --kv-overwrite Cannot be used with --local. + When using Fastly KV Store, always overwrite + existing entries, even if unchanged. + +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify only one. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: + 1. FASTLY_API_TOKEN environment variable + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. + +Examples: + npx @fastly/compute-js-static-publish publish-content --collection-name=preview-456 + npx @fastly/compute-js-static-publish publish-content --expires-in=7d --kv-overwrite + npx @fastly/compute-js-static-publish publish-content --expires-never --local + +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + { name: 'config', type: String }, + { name: 'collection-name', type: String, }, + { name: 'root-dir', type: String, }, + { name: 'kv-overwrite', type: Boolean }, + + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, + + { name: 'local', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + config: configFilePathValue, + ['collection-name']: collectionNameValue, + ['root-dir']: rootDir, + ['kv-overwrite']: overwriteKvStoreItems, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, + ['expires-never']: expiresNever, + local: localMode, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`πŸš€ Publishing content...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`βœ”οΈ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const configFilePath = configFilePathValue ?? './publish-content.config.js'; + + let publishContentConfig; + try { + publishContentConfig = await loadPublishContentConfigFile(configFilePath); + } catch (err) { + console.error(`❌ Can't load ${configFilePath}`); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publicDirRoot = path.resolve(rootDir != null ? rootDir : publishContentConfig.rootDir); + if ((computeAppDir + '/').startsWith(publicDirRoot + '/')) { + if (verbose) { + console.log(`‼️ Public directory '${rootRelative(publicDirRoot)}' includes the Compute app directory.`); + console.log(`This may be caused by an incorrect configuration, as this could cause Compute application source`); + console.log(`files to be included in your published output.`); + } + } + + let expirationTime: number | null | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + + console.log(`βœ”οΈ Public directory '${rootRelative(publicDirRoot)}'.`); + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + + // Load content types + const contentTypes: ContentTypeDef[] = mergeContentTypes(publishContentConfig.contentTypes); + + // #### Collect all file paths + console.log(`πŸ” Scanning root directory ${rootRelative(publicDirRoot)}...`); + + const excludeDirs = publishContentConfig.excludeDirs; + if (excludeDirs.length > 0) { + console.log(`βœ”οΈ Using exclude directories: ${excludeDirs.join(', ')}`); + } else { + if (verbose) { + console.log(`βœ”οΈ No exclude directories defined.`); + } + } + + const excludeDotFiles = publishContentConfig.excludeDotFiles; + if (excludeDotFiles) { + console.log(`βœ”οΈ Files/Directories starting with . are excluded.`); + } + + const includeWellKnown = publishContentConfig.includeWellKnown; + if (includeWellKnown) { + console.log(`βœ”οΈ (.well-known is exempt from exclusion.)`); + } + + // ### Collection name that we're currently publishing + // for example, live, staging + const collectionName = collectionNameValue ?? process.env.PUBLISHER_COLLECTION_NAME ?? defaultCollectionName; + console.log(`βœ”οΈ Collection Name: ${collectionName}`); + + if (expirationTime == null) { + // includes null and undefined + console.log(`βœ”οΈ Publishing with no expiration timestamp.`); + } else { + console.log(`βœ”οΈ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } + if (collectionName === defaultCollectionName && expirationTime !== undefined) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + + // files to be included in the build/publish + const files = enumerateFiles({ + publicDirRoot, + excludeDirs, + excludeDotFiles, + includeWellKnown, + }); + + const stats = { + kvStore: 0, + }; + + // Create "KV Store content" sub dir if it doesn't exist already. + // This will be used to hold a copy of files to prepare for upload to the KV Store + // and for serving using the local development server. + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + // A list of items in the KV Store at the end of the publishing. + // Includes items that already exist as well. 'write' signifies + // that the item is to be written + const kvStoreItemDescriptions: KVStoreItemDesc[] = []; + + // Assets included in the publishing, keyed by asset key + const kvAssetsIndex: KVAssetEntryMap = {}; + + // All the metadata of the variants we know about during this publishing, keyed on the base version's hash. + type VariantMetadataEntry = KVAssetVariantMetadata & { + existsInKvStore: boolean, + }; + type VariantMetadataMap = Map; + const baseHashToVariantMetadatasMap = new Map(); + + // #### Iterate files + for (const file of files) { + // #### asset key + const assetKey = file.slice(publicDirRoot.length) + // in Windows, assetKey will otherwise end up as \path\file.html + .replace(/\\/g, '/'); + + // #### decide what the Content Type will be + let contentTypeTestResult = testFileContentType(contentTypes, assetKey); + if (contentTypeTestResult == null) { + contentTypeTestResult = { + text: false, + contentType: 'application/octet-stream', + precompressAsset: false, + }; + if (verbose) { + console.log('⚠️ Notice: Unknown file type ' + assetKey + '. Treating as binary file.'); + } + } + + const contentType = contentTypeTestResult.contentType; + const contentCompression = + contentTypeTestResult.precompressAsset ? publishContentConfig.contentCompression : []; + + // #### Are we going to include this file? + let includeAsset; + if (publishContentConfig.kvStoreAssetInclusionTest != null) { + + includeAsset = publishContentConfig.kvStoreAssetInclusionTest(assetKey, contentType); + + } else { + // If no test is set, then default to inclusion + includeAsset = true; + } + + if (!includeAsset) { + continue; + } + + // #### Base file size, hash, last modified time + const { size: baseSize, hash: baseHash } = await calculateFileSizeAndHash(file); + console.log(`πŸ“„ File '${rootRelative(file)}' - ${baseSize} bytes, sha256: ${baseHash}`); + const stats = fs.statSync(file); + const lastModifiedTime = Math.floor((stats.mtime).getTime() / 1000); + + // #### Metadata per variant + let variantMetadatas = baseHashToVariantMetadatasMap.get(baseHash); + if (variantMetadatas == null) { + variantMetadatas = new Map(); + baseHashToVariantMetadatasMap.set(baseHash, variantMetadatas); + } + + const variantsToKeep: ContentCompressionTypes[] = []; + + const variants = [ + 'original', + ...contentCompression, + ] as const; + for (const variant of variants) { + let variantKey = `${publishId}_files_sha256_${baseHash}`; + let variantFilename = `${baseHash}`; + if (variant !== 'original') { + variantKey = `${variantKey}_${variant}`; + variantFilename = `${variantFilename}_${variant}`; + } + + const variantFilePath = path.resolve(staticPublisherKvStoreContent, variantFilename); + + let variantMetadata = variantMetadatas.get(variant); + if (variantMetadata != null) { + + console.log(` πŸƒβ€β™‚οΈ Asset "${variantKey}" is identical to an item we already know about, reusing existing copy.`); + + } else { + + let kvStoreItemMetadata: KVAssetVariantMetadata | null = null; + + if (!localMode && !overwriteKvStoreItems) { + const items = [{ + key: variantKey, + }]; + + await doKvStoreItemsOperation( + items, + async(_, variantKey) => { + // fastlyApiContext is non-null if useKvStore is true + const kvStoreEntryInfo = await getKvStoreEntryInfo(fastlyApiContext!, kvStoreName, variantKey); + if (!kvStoreEntryInfo) { + return; + } + let itemMetadata; + if (kvStoreEntryInfo.metadata != null) { + try { + itemMetadata = JSON.parse(kvStoreEntryInfo.metadata); + } catch { + // if the metadata does not parse successfully as JSON, + // treat it as though it didn't exist. + } + } + if (isKVAssetVariantMetadata(itemMetadata)) { + let exists = false; + if (itemMetadata.size <= KV_STORE_CHUNK_SIZE) { + // For an item equal to or smaller than the chunk size, if it exists + // and its metadata asserts no chunk count, then we assume it exists. + if (itemMetadata.numChunks === undefined) { + exists = true; + } + } else { + // For chunked objects, if the first chunk exists, and its metadata asserts + // the same number of chunks based on size, then we assume it exists (for now). + // In the future we might actually check for the existence and sizes of + // every chunk in the KV Store. + const expectedNumChunks = Math.ceil(itemMetadata.size / KV_STORE_CHUNK_SIZE); + if (itemMetadata.numChunks === expectedNumChunks) { + exists = true; + } + } + if (exists) { + kvStoreItemMetadata = { + contentEncoding: itemMetadata.contentEncoding, + size: itemMetadata.size, + hash: itemMetadata.hash, + numChunks: itemMetadata.numChunks, + }; + } + } + } + ); + } + + if ((kvStoreItemMetadata as KVAssetVariantMetadata | null) != null) { + + console.log(` ・ Asset found in KV Store with key "${variantKey}".`); + // And we already know its hash and size. + + variantMetadata = { + contentEncoding: kvStoreItemMetadata!.contentEncoding, + size: kvStoreItemMetadata!.size, + hash: kvStoreItemMetadata!.hash, + numChunks: kvStoreItemMetadata!.numChunks, + existsInKvStore: true, + }; + + } else { + + await ensureVariantFileExists( + variantFilePath, + variant, + file, + ); + if (!localMode) { + console.log(` ・ Flagging asset for upload to KV Store with key "${variantKey}".`); + } + + let contentEncoding, hash, size; + if (variant === 'original') { + contentEncoding = undefined; + hash = baseHash; + size = baseSize; + } else { + contentEncoding = variant; + ({hash, size} = await calculateFileSizeAndHash(variantFilePath)); + } + + const numChunks = Math.ceil(size / KV_STORE_CHUNK_SIZE); + + variantMetadata = { + contentEncoding, + size, + hash, + numChunks: numChunks > 1 ? numChunks : undefined, + existsInKvStore: false, + }; + } + + variantMetadatas.set(variant, variantMetadata); + + kvStoreItemDescriptions.push({ + write: !variantMetadata.existsInKvStore, + size: variantMetadata.size, + key: variantKey, + filePath: variantFilePath, + metadataJson: { + contentEncoding: variantMetadata.contentEncoding, + size: variantMetadata.size, + hash: variantMetadata.hash, + numChunks: variantMetadata.numChunks, + }, + }); + + if (localMode) { + // Although we already know the size and hash of the variant, the local server + // needs a copy of the file so we create it if it doesn't exist. + // This may happen for example if files were uploaded to the KV Store in a previous + // publishing, but local static content files have been removed since. + await ensureVariantFileExists( + variantFilePath, + variant, + file, + ); + console.log(` ・ Prepping asset for local KV Store with key "${variantKey}".`); + } + } + + // Only keep variants whose file size actually ends up smaller than + // what we started with. + if (variant !== 'original' && variantMetadata.size < baseSize) { + variantsToKeep.push(variant); + } + } + + kvAssetsIndex[assetKey] = { + key: `sha256:${baseHash}`, + size: baseSize, + contentType: contentTypeTestResult.contentType, + lastModifiedTime, + variants: variantsToKeep, + }; + + } + console.log(`βœ… Scan complete.`) + + console.log(`πŸͺ Chunking large files...`); + await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); + console.log(`βœ… Large files have been chunked.`); + + if (localMode) { + console.log(`πŸ“ Writing local server KV Store entries.`); + writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); + console.log(`βœ… Wrote KV Store entries for local server.`); + } else { + console.log(`πŸ“€ Uploading entries to KV Store.`); + // fastlyApiContext is non-null if useKvStore is true + await doKvStoreItemsOperation( + kvStoreItemDescriptions.filter(x => x.write), + async ({filePath, metadataJson}, key) => { + const fileBytes = fs.readFileSync(filePath); + await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); + console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) + } + ); + console.log(`βœ… Uploaded entries to KV Store.`); + } + + // #### INDEX FILE + console.log(`πŸ—‚οΈ Saving Index...`); + + const indexFileKey = `${publishId}_index_${collectionName}`; + const indexMetadata: IndexMetadata = { + publishedTime: Math.floor(Date.now() / 1000), + expirationTime: expirationTime ?? undefined, + }; + + if (localMode) { + const indexFileName = `index_${collectionName}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + fs.writeFileSync(indexFilePath, JSON.stringify(kvAssetsIndex)); + await localKvStoreSubmitEntry( + storeFile, + indexFileKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + } else { + await kvStoreSubmitEntry( + fastlyApiContext!, + kvStoreName, + indexFileKey, + JSON.stringify(kvAssetsIndex), + JSON.stringify(indexMetadata), + ); + } + console.log(`βœ… Index has been saved.`) + + // #### SERVER SETTINGS + // These are saved to KV Store + console.log(`βš™οΈ Saving server settings...`); + + const server = applyDefaults(publishContentConfig.server, { + publicDirPrefix: '', + staticItems: [], + allowedEncodings: [ 'br', 'gzip' ], + spaFile: null, + notFoundPageFile: null, + autoExt: [], + autoIndex: [], + }); + + let publicDirPrefix = server.publicDirPrefix; + console.log(` βœ”οΈ Server public dir prefix '${publicDirPrefix}'.`); + + let staticItems = server.staticItems; + + let allowedEncodings = server.allowedEncodings; + + let spaFile = server.spaFile; + + if(spaFile != null) { + console.log(` βœ”οΈ Application SPA file '${spaFile}'.`); + const spaAsset = kvAssetsIndex[spaFile]; + if(spaAsset == null || spaAsset.contentType !== 'text/html') { + if (verbose) { + console.log(` ⚠️ Notice: '${spaFile}' does not exist or is not of type 'text/html'. Ignoring.`); + } + spaFile = null; + } + } else { + if (verbose) { + console.log(` βœ”οΈ Application is not a SPA.`); + } + } + + let notFoundPageFile = server.notFoundPageFile; + if(notFoundPageFile != null) { + console.log(` βœ”οΈ Application 'not found (404)' file '${notFoundPageFile}'.`); + const notFoundPageAsset = kvAssetsIndex[notFoundPageFile]; + if(notFoundPageAsset == null || notFoundPageAsset.contentType !== 'text/html') { + if (verbose) { + console.log(` ⚠️ Notice: '${notFoundPageFile}' does not exist or is not of type 'text/html'. Ignoring.`); + } + notFoundPageFile = null; + } + } else { + if (verbose) { + console.log(` βœ”οΈ Application specifies no 'not found (404)' page.`); + } + } + + let autoIndex: string[] = server.autoIndex; + let autoExt: string[] = server.autoExt; + + const serverSettings: PublisherServerConfigNormalized = { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + }; + + const settingsFileKey = `${publishId}_settings_${collectionName}`; + + if (localMode) { + const settingsFileName = `settings_${collectionName}.json`; + const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); + fs.writeFileSync(settingsFilePath, JSON.stringify(serverSettings)); + await localKvStoreSubmitEntry( + storeFile, + settingsFileKey, + path.relative(computeAppDir, settingsFilePath), + undefined, + ); + + } else { + await kvStoreSubmitEntry( + fastlyApiContext!, + kvStoreName, + settingsFileKey, + JSON.stringify(serverSettings), + undefined, + ); + } + console.log(`βœ… Settings have been saved.`); + + console.log(`πŸŽ‰ Completed.`); + +} diff --git a/src/cli/commands/scaffold/index.ts b/src/cli/commands/scaffold/index.ts new file mode 100644 index 0000000..d704956 --- /dev/null +++ b/src/cli/commands/scaffold/index.ts @@ -0,0 +1,831 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +// This program creates a Fastly Compute JavaScript application +// in a subfolder named compute-js. +// This project can be served using fastly compute serve +// or deployed to a Compute service using fastly compute publish. + +import * as child_process from 'node:child_process'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +import { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; + +import { parseCommandLine } from '../../util/args.js'; +import { dotRelative, rootRelative } from '../../util/files.js'; +import { findComputeJsStaticPublisherVersion, type PackageJson } from '../../util/package.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish [options] + +Description: + Scaffold a new Compute app configured for static publishing. + + Note: If run inside a scaffolded project, this tool will automatically enter project + management mode. + +Options: + --kv-store-name (required) KV Store name for content storage + --root-dir (required) Path to static content (e.g., ./public) + -o, --output Output directory for Compute app (default: ./compute-js) + --static-publisher-working-dir Working directory for build artifacts (default: /static-publisher) + --publish-id Advanced. Prefix for KV keys (default: "default") + (default: ./compute-js/static-publisher) + +Compute Service Metadata: + --name App name (for fastly.toml) + --description App description (for fastly.toml) + --author App author (for fastly.toml) + --service-id (optional) Fastly service ID + +Server Config: + --public-dir Base dir for content (default: root-dir) + --static-dir , Directory served with long TTL (can repeat) + --spa SPA fallback file (e.g., /index.html) + --not-found-page 404 fallback file (e.g., /404.html) + --auto-index , Index filename (e.g., index.html,index.htm) + --auto-ext , Automatic extensions (e.g., .html,.htm) + +Other: + --verbose Enable verbose output + -h, --help Show help +`); +} + +export type InitAppOptions = { + outputDir: string | undefined, + rootDir: string | undefined, + publicDir: string | undefined, + staticDirs: string[], + staticPublisherWorkingDir: string | undefined, + spa: string | undefined, + notFoundPage: string | undefined, + autoIndex: string[], + autoExt: string[], + name: string | undefined, + author: string | undefined, + description: string | undefined, + serviceId: string | undefined, + publishId: string | undefined, + kvStoreName: string | undefined, +}; + +const defaultOptions: InitAppOptions = { + outputDir: undefined, + rootDir: undefined, + publicDir: undefined, + staticDirs: [], + staticPublisherWorkingDir: undefined, + spa: undefined, + notFoundPage: '[public-dir]/404.html', + autoIndex: [ 'index.html', 'index.htm' ], + autoExt: [ '.html', '.htm' ], + author: 'you@example.com', + name: 'compute-js-static-site', + description: 'Fastly Compute static site', + serviceId: undefined, + publishId: undefined, + kvStoreName: undefined, +}; + +function buildOptions( + packageJson: PackageJson | null, + commandLineOptions: CommandLineOptions, +): InitAppOptions { + + // Applied in this order for proper overriding + // 1. defaults + // 2. package.json + // 3. command-line args + + const options = structuredClone(defaultOptions); + + if (packageJson?.name !== undefined) { + options.name = packageJson!.name; + } + if (packageJson?.author !== undefined) { + options.author = packageJson!.author; + } + if (packageJson?.description !== undefined) { + options.description = packageJson!.description; + } + + { + let outputDir: string | undefined; + const outputDirValue = commandLineOptions['output']; + if (outputDirValue == null || typeof outputDirValue === 'string') { + outputDir = outputDirValue; + } + if (outputDir !== undefined) { + options.outputDir = outputDir; + } + } + + { + let rootDir: string | undefined; + const rootDirValue = commandLineOptions['root-dir']; + if (rootDirValue == null || typeof rootDirValue === 'string') { + rootDir = rootDirValue; + } + if (rootDir !== undefined) { + options.rootDir = rootDir; + } + } + + { + let publicDir: string | undefined; + const publicDirValue = commandLineOptions['public-dir']; + if (publicDirValue == null || typeof publicDirValue === 'string') { + publicDir = publicDirValue; + } + if (publicDir !== undefined) { + options.publicDir = publicDir; + } + } + + { + let staticDirs: string[] | undefined; + const staticDirsValue = commandLineOptions['static-dir']; + + const asArray = Array.isArray(staticDirsValue) ? staticDirsValue : [ staticDirsValue ]; + if (asArray.every((x: any) => typeof x === 'string')) { + staticDirs = asArray; + } + if (staticDirs !== undefined) { + options.staticDirs = staticDirs; + } + } + + { + let staticPublisherWorkingDir: string | undefined; + const staticPublisherWorkingDirValue = commandLineOptions['static-publisher-working-dir']; + if (staticPublisherWorkingDirValue == null || typeof staticPublisherWorkingDirValue === 'string') { + staticPublisherWorkingDir = staticPublisherWorkingDirValue; + } + if (staticPublisherWorkingDir !== undefined) { + options.staticPublisherWorkingDir = staticPublisherWorkingDir; + } + } + + { + let spa: string | undefined; + const spaValue = commandLineOptions['spa']; + if (spaValue === null) { + // If 'spa' is provided with a null value, then the flag was provided + // with no value. Assumed to be './index.html' relative to the public directory. + spa = '[public-dir]/index.html'; + console.log('--spa provided, but no value specified. Assuming ' + spa); + } else if (spaValue == null || typeof spaValue === 'string') { + spa = spaValue; + } + if (spa !== undefined) { + options.spa = spa; + } + } + + { + let notFoundPage: string | undefined; + const notFoundPageValue = commandLineOptions['not-found-page']; + if (notFoundPageValue === null) { + // If 'spa' is provided with a null value, then the flag was provided + // with no value. Assumed to be './404.html' relative to the public directory. + notFoundPage = '[public-dir]/404.html'; + console.log('--not-found-page provided, but no value specified. Assuming ' + notFoundPage); + } else if (notFoundPageValue == null || typeof notFoundPageValue === 'string') { + notFoundPage = notFoundPageValue; + } + if (notFoundPage !== undefined) { + options.notFoundPage = notFoundPage; + } + } + + { + let autoIndex: string[] | undefined; + const autoIndexValue = commandLineOptions['auto-index']; + + const asArray = Array.isArray(autoIndexValue) ? autoIndexValue : [ autoIndexValue ]; + if (asArray.every((x: any) => typeof x === 'string')) { + + autoIndex = (asArray as string[]).reduce((acc, entry) => { + + const segments = entry + .split(',') + .map(x => x.trim()) + .filter(x => Boolean(x)); + + for (const segment of segments) { + acc.push(segment); + } + + return acc; + }, []); + + } + if (autoIndex !== undefined) { + options.autoIndex = autoIndex; + } + } + + { + let autoExt: string[] = []; + const autoExtValue = commandLineOptions['auto-ext']; + + const asArray = Array.isArray(autoExtValue) ? autoExtValue : [ autoExtValue ]; + if (asArray.every((x: any) => typeof x === 'string')) { + + autoExt = (asArray as string[]).reduce((acc, entry) => { + + const segments = entry + .split(',') + .map(x => x.trim()) + .filter(x => Boolean(x)) + .map(x => !x.startsWith('.') ? '.' + x : x); + + for (const segment of segments) { + acc.push(segment); + } + + return acc; + }, []); + + } + if (autoExt !== undefined) { + options.autoExt = autoExt; + } + } + + { + let name: string | undefined; + const nameValue = commandLineOptions['name']; + if (nameValue == null || typeof nameValue === 'string') { + name = nameValue; + } + if (name !== undefined) { + options.name = name; + } + } + + { + let author: string | undefined; + const authorValue = commandLineOptions['author']; + if (authorValue == null || typeof authorValue === 'string') { + author = authorValue; + } + if (author !== undefined) { + options.author = author; + } + } + + { + let description: string | undefined; + const descriptionValue = commandLineOptions['description']; + if (descriptionValue == null || typeof descriptionValue === 'string') { + description = descriptionValue; + } + if (description !== undefined) { + options.description = description; + } + } + + { + let serviceId: string | undefined; + const serviceIdValue = commandLineOptions['service-id']; + if (serviceIdValue == null || typeof serviceIdValue === 'string') { + serviceId = serviceIdValue; + } + if (serviceId !== undefined) { + options.serviceId = serviceId; + } + } + + { + let publishId: string | undefined; + const publishIdValue = commandLineOptions['publish-id']; + if (publishIdValue == null || typeof publishIdValue === 'string') { + publishId = publishIdValue; + } + if (publishId !== undefined) { + options.publishId = publishId; + } + } + + { + let kvStoreName: string | undefined; + const kvStoreNameValue = commandLineOptions['kv-store-name']; + if (kvStoreNameValue == null || typeof kvStoreNameValue === 'string') { + kvStoreName = kvStoreNameValue; + } + if (kvStoreName !== undefined) { + options.kvStoreName = kvStoreName; + } + } + + return options; + +} + +const PUBLIC_DIR_TOKEN = '[public-dir]'; +function processPublicDirToken(filepath: string, publicDir: string) { + if (!filepath.startsWith(PUBLIC_DIR_TOKEN)) { + return path.resolve(filepath); + } + + const processedPath = '.' + filepath.slice(PUBLIC_DIR_TOKEN.length); + return path.resolve(publicDir, processedPath) +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + + // Required. The name of a Fastly KV Store to hold the content assets. + // It is also added to the fastly.toml that is generated. + { name: 'kv-store-name', type: String, }, + + // Output directory. "Required" (if not specified, then defaultValue is used). + { name: 'output', alias: 'o', type: String, defaultValue: './compute-js', }, + + // Name of the application, to be inserted into fastly.toml + { name: 'name', type: String, }, + + // Description of the application, to be inserted into fastly.toml + { name: 'description', type: String, }, + + // Name of the author, to be inserted into fastly.toml + { name: 'author', type: String, }, + + // Fastly Service ID to be added to the fastly.toml that is generated. + { name: 'service-id', type: String }, + + // Required. The 'root' directory for the publishing. + // All assets are expected to exist under this root. Required. + // For backwards compatibility, if this value is not provided, + // then the value of 'public-dir' is used. + { name: 'root-dir', type: String, }, + + // Publish ID to be used as a prefix for all KV Store entries. + // If not provided, the default value of 'default' is used. + { name: 'publish-id', type: String, }, + + // The 'static publisher working directory' is the directory under the Compute + // application where asset files are written in preparation for upload to the + // KV Store and for serving for local mode. + { name: 'static-publisher-working-dir', type: String, }, + + // The 'public' directory. The Publisher Server will + // resolve requests relative to this directory. If not specified, + // defaults to the same value as 'root-dir'. See README for + // details. + { name: 'public-dir', type: String, }, + + // Directories to specify as containing 'static' files. The + // Publisher Server will serve files from these directories + // with a long TTL. + { name: 'static-dir', type: String, multiple: true, }, + + // Path to a file to be used to serve in a SPA application. + // The Publisher Server will serve this file with a 200 status code + // when the request doesn't match a known file, and the accept + // header includes text/html. You may use the '[public-dir]' token + // if you wish to specify this as a relative path from the 'public-dir'. + { name: 'spa', type: String, }, + + // Path to a file to be used to serve as a 404 not found page. + // The Publisher Server will serve this file with a 404 status code + // when the request doesn't match a known file, and the accept + // header includes text/html. You may use the '[public-dir]' token + // if you wish to specify this as a relative path from the 'public-dir'. + { name: 'not-found-page', type: String, }, + + // List of files to automatically use as index, for example, index.html,index.htm + // If a request comes in but the route does not exist, we check the route + // plus a slash plus the items in this array. + { name: 'auto-index', type: String, multiple: true, }, + + // List of extensions to apply to a path name, for example, if + // http://example.com/about is requested, we can respond with http://example.com/about.html + { name: 'auto-ext', type: String, multiple: true, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + let packageJson; + try { + const packageJsonText = fs.readFileSync("./package.json", "utf-8"); + packageJson = JSON.parse(packageJsonText) as PackageJson; + } catch { + console.log("Can't read/parse package.json in current directory, making no assumptions!"); + packageJson = null; + } + + const options = buildOptions( + packageJson, + parsed.commandLineOptions, + ); + + const COMPUTE_JS_DIR = options.outputDir; + if (COMPUTE_JS_DIR == null) { + console.error("❌ required parameter --output not provided."); + process.exitCode = 1; + return; + } + const computeJsDir = path.resolve(COMPUTE_JS_DIR); + + // Resolve the root dir, relative to current directory, and make sure it exists. + const ROOT_DIR = options.rootDir; + if (ROOT_DIR == null) { + console.error("❌ required parameter --root-dir not provided."); + process.exitCode = 1; + return; + } + const rootDir = path.resolve(ROOT_DIR); + if (!fs.existsSync(rootDir) || !fs.statSync(rootDir).isDirectory()) { + console.error(`❌ Specified root directory '${ROOT_DIR}' does not exist.`); + console.error(` * ${rootRelative(rootDir)} must exist and be a directory.`); + process.exitCode = 1; + return; + } + + // Resolve the public dir as well. If it's not specified, we use the root directory. + const PUBLIC_DIR = options.publicDir ?? ROOT_DIR; + const publicDir = path.resolve(PUBLIC_DIR); + if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) { + console.error(`❌ Specified public directory '${PUBLIC_DIR}' does not exist.`); + console.error(` * ${rootRelative(publicDir)} must exist and be a directory.`); + process.exitCode = 1; + return; + } + + // Public dir must be inside the root dir. + if (!publicDir.startsWith(rootDir)) { + console.error(`❌ Specified public directory '${PUBLIC_DIR}' is not under the asset root directory.`); + console.error(` * ${rootRelative(publicDir)} must be under ${rootRelative(rootDir)}`); + process.exitCode = 1; + return; + } + + // Static dirs must be inside the public dir. + const STATIC_DIRS = options.staticDirs; + const staticDirs: string[] = []; + for (const STATIC_DIR of STATIC_DIRS) { + // For backwards compatibility, these values can start with [public-dir] + const staticDir = processPublicDirToken(STATIC_DIR, publicDir); + if (!staticDir.startsWith(publicDir)) { + console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); + console.log(` * ${rootRelative(staticDir)} is not under ${rootRelative(publicDir)}`); + continue; + } + if (!fs.existsSync(staticDir) || !fs.statSync(staticDir).isDirectory()) { + console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); + console.log(` * ${rootRelative(staticDir)} does not exist or is not a directory.`); + continue; + } + staticDirs.push(staticDir); + } + + // Static Publisher Working Root Dir must be under the current dir + // This comes in relative to cwd. + const STATIC_PUBLISHER_WORKING_DIR = options.staticPublisherWorkingDir ?? + path.resolve(computeJsDir, './static-publisher'); + const staticPublisherWorkingDir = path.resolve(STATIC_PUBLISHER_WORKING_DIR); + if ( + !staticPublisherWorkingDir.startsWith(computeJsDir) || + staticPublisherWorkingDir === computeJsDir || + staticPublisherWorkingDir === path.resolve(computeJsDir, './bin') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './pkg') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './src') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './node_modules') + ) { + console.error(`❌ Specified static publisher working directory '${rootRelative(staticPublisherWorkingDir)}' must be under ${rootRelative(computeJsDir)}.`); + console.error(` It also must not be bin, pkg, src, or node_modules.`); + process.exitCode = 1; + return; + } + + // SPA and Not Found are relative to the asset root dir. + + const SPA = options.spa; + let spaFilename: string | undefined; + if (SPA != null) { + // If it starts with [public-dir], then resolve it relative to public directory. + spaFilename = processPublicDirToken(SPA, publicDir); + // At any rate it must exist under the root directory + if (!spaFilename.startsWith(rootDir)) { + console.log(`⚠️ Ignoring specified SPA file '${SPA}' as is not under the asset root directory.`); + console.log(` * ${rootRelative(spaFilename)} is not under ${rootRelative(rootDir)}`); + spaFilename = undefined; + } else if (!fs.existsSync(spaFilename)) { + console.log(`⚠️ Warning: Ignoring specified SPA file '${SPA}' does not exist.`); + console.log(` * ${rootRelative(spaFilename)} does not exist.`); + } + } + + const NOT_FOUND_PAGE = options.notFoundPage; + let notFoundPageFilename: string | undefined; + if (NOT_FOUND_PAGE != null) { + // If it starts with [public-dir], then resolve it relative to public directory. + notFoundPageFilename = processPublicDirToken(NOT_FOUND_PAGE, publicDir); + // At any rate it must exist under the root directory + if (!notFoundPageFilename.startsWith(rootDir)) { + console.log(`⚠️ Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as is not under the asset root directory.`); + console.log(` * ${rootRelative(notFoundPageFilename)} is not under ${rootRelative(rootDir)}`); + notFoundPageFilename = undefined; + } else if (!fs.existsSync(notFoundPageFilename)) { + console.log(`⚠️ Warning: Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as it does not exist.`); + console.log(` * ${rootRelative(notFoundPageFilename)} does not exist.`); + } + } + + const autoIndex = options.autoIndex; + const autoExt = options.autoExt; + + const exists = fs.existsSync(computeJsDir); + if(exists) { + console.error(`❌ '${COMPUTE_JS_DIR}' directory already exists!`); + console.error(` You should not run this command if this directory exists.`); + console.error(` If you need to re-scaffold the static publisher Compute App,`); + console.error(` delete the following directory and then try again:`); + console.error(` ${rootRelative(computeJsDir)}`); + process.exitCode = 1; + return; + } + + const author = options.author; + const name = options.name; + const description = options.description; + const fastlyServiceId = options.serviceId; + const kvStoreName = options.kvStoreName; + if (kvStoreName == null) { + console.error(`❌ required parameter --kv-store-name not provided.`); + process.exitCode = 1; + return; + } + let publishId = options.publishId; + if (publishId == null) { + publishId = 'default'; + } + + const defaultCollectionName = 'live'; + + console.log(''); + console.log('Compute Application Settings'); + console.log('----------------------------'); + console.log('Compute Application Output Dir :', rootRelative(computeJsDir)); + console.log('name :', name); + console.log('author :', author); + console.log('description :', description); + console.log('Service ID :', fastlyServiceId ?? '(None)'); + console.log('KV Store Name :', kvStoreName); + console.log('Default Collection Name :', defaultCollectionName); + console.log('Publish ID :', publishId); + + console.log(''); + console.log('Publish Settings'); + console.log('----------------'); + console.log('Publish Root Dir :', rootRelative(rootDir)); + console.log('Publisher Working Dir :', rootRelative(staticPublisherWorkingDir)); + + console.log(''); + console.log('Publisher Server Settings'); + console.log('-------------------------'); + console.log('Server Public Dir :', rootRelative(publicDir)); + console.log('SPA :', spaFilename != null ? rootRelative(spaFilename) : '(None)'); + console.log('404 Page :', notFoundPageFilename != null ? rootRelative(notFoundPageFilename) : '(None)'); + console.log('Auto-Index :', autoIndex.length > 0 ? autoIndex : '(None)') + console.log('Auto-Ext :', autoExt.length > 0 ? autoExt : '(None)') + console.log('Static Files Dir :', staticDirs.length > 0 ? staticDirs.map(rootRelative) : '(None)'); + + console.log(''); + + console.log("Initializing Compute Application in " + computeJsDir + "..."); + fs.mkdirSync(computeJsDir); + fs.mkdirSync(path.resolve(computeJsDir, './src')); + + const resourceFiles: Record = {}; + + // .gitignore + resourceFiles['.gitignore'] = `\ +/node_modules +/bin +/pkg +/${path.relative(computeJsDir, staticPublisherWorkingDir)} +`; + + // package.json + const computeJsStaticPublisherVersion = findComputeJsStaticPublisherVersion(packageJson); + resourceFiles['package.json'] = JSON.stringify({ + name, + version: '0.1.0', + description, + author, + type: 'module', + devDependencies: { + "@fastly/cli": "^11.2.0", + '@fastly/compute-js-static-publish': computeJsStaticPublisherVersion, + }, + dependencies: { + '@fastly/js-compute': '^3.26.0', + }, + engines: { + node: '>=20.11.0', + }, + private: true, + scripts: { + 'dev:publish': 'npx @fastly/compute-js-static-publish publish-content --local', + 'dev:start': 'fastly compute serve', + 'fastly:deploy': 'fastly compute publish', + 'fastly:publish': 'npx @fastly/compute-js-static-publish publish-content', + 'build': 'js-compute-runtime ./src/index.js ./bin/main.wasm' + }, + }, undefined, 2); + + // fastly.toml + const localServerKvStorePath = dotRelative(computeJsDir, path.resolve(staticPublisherWorkingDir, 'kvstore.json')); + resourceFiles['fastly.toml'] = /* language=text */ `\ +# This file describes a Fastly Compute package. To learn more visit: +# https://developer.fastly.com/reference/fastly-toml/ + +authors = [ "${author}" ] +description = "${description}" +language = "javascript" +manifest_version = 3 +name = "${name}" +${fastlyServiceId != null ? `service_id = "${fastlyServiceId}"\n` : ''} +[scripts] +build = "npm run build" + +[local_server.kv_stores] +${kvStoreName} = { file = "${localServerKvStorePath}", format = "json" } + +[setup.kv_stores.${kvStoreName}] +`; + + // kvstore.json + resourceFiles[localServerKvStorePath] = '{}'; + + // static-publish.rc.js + resourceFiles['static-publish.rc.js'] = `\ +/* + * Generated by @fastly/compute-js-static-publish. + */ + +/** @type {import('@fastly/compute-js-static-publish').StaticPublishRc} */ +const rc = { + kvStoreName: ${JSON.stringify(kvStoreName)}, + publishId: ${JSON.stringify(publishId)}, + defaultCollectionName: ${JSON.stringify(defaultCollectionName)}, + staticPublisherWorkingDir: ${JSON.stringify(dotRelative(computeJsDir, staticPublisherWorkingDir))}, +}; + +export default rc; +`; + + // publish-content.config.js + // publicDirPrefix -- if public dir is deeper than the root dir, then + // public dir is used as a prefix to drill into asset names. + // e.g., + // root dir : /path/to/root + // public dir : /path/to/root/public + // then publicDirPrefix = /public + + // We've already established publicDir.startsWith(rootDir) + let publicDirPrefix = ''; + if (rootDir !== publicDir) { + publicDirPrefix = rootDir.slice(rootDir.length); + } + + // staticItems - specified as prefixes, relative to publicDir + const staticItems: string[] = []; + for (const staticDir of staticDirs) { + // We've already established staticDir.startsWith(publicDir) + let staticItem = staticDir.slice(publicDir.length); + if (!staticItem.endsWith('/')) { + // Ending with a slash denotes that this is a directory. + staticItem = staticItem + '/'; + } + staticItems.push(staticItem); + } + + // spaFile - asset key of spa file + let spaFile: string | false = false; + if (spaFilename != null) { + // We've already established spaFilename.startsWith(rootDir) + // and that it exists + spaFile = spaFilename.slice(rootDir.length); + } + + let notFoundPageFile: string | false = false; + if(notFoundPageFilename != null) { + // We've already established notFoundPageFilename.startsWith(rootDir) + // and that it exists + notFoundPageFile = notFoundPageFilename.slice(rootDir.length); + } + + resourceFiles['publish-content.config.js'] = `\ +/* + * Generated by @fastly/compute-js-static-publish. + */ + +// Commented items are defaults, feel free to modify and experiment! +// See README for a detailed explanation of the configuration options. + +/** @type {import('@fastly/compute-js-static-publish').PublishContentConfig} */ +const config = { + rootDir: ${JSON.stringify(dotRelative(computeJsDir, rootDir))}, + // excludeDirs: [ './node_modules' ], + // excludeDotFiles: true, + // includeWellKnown: true, + // kvStoreAssetInclusionTest: (assetKey) => true, + // contentCompression: [ 'br', 'gzip' ], + // contentTypes: [ + // { test: /.custom$/, contentType: 'application/x-custom', text: false }, + // ], + + // Server settings are saved to the KV Store per collection + server: { + publicDirPrefix: ${JSON.stringify(publicDirPrefix)}, + staticItems: ${JSON.stringify(staticItems)}, + allowedEncodings: [ 'br', 'gzip' ], + spaFile: ${JSON.stringify(spaFile)}, + notFoundPageFile: ${JSON.stringify(notFoundPageFile)}, + autoExt: ${JSON.stringify(autoExt)}, + autoIndex: ${JSON.stringify(autoIndex)}, + }, +}; + +export default config; +`; + + // src/index.js + resourceFiles['./src/index.js'] = /* language=text */ `\ +/// +import { env } from 'fastly:env'; +import { PublisherServer } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; +const publisherServer = PublisherServer.fromStaticPublishRc(rc); + +// eslint-disable-next-line no-restricted-globals +addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); +async function handleRequest(event) { + + console.log('FASTLY_SERVICE_VERSION', env('FASTLY_SERVICE_VERSION')); + + const request = event.request; + + const response = await publisherServer.serveRequest(request); + if (response != null) { + return response; + } + + // Do custom things here! + // Handle API requests, serve non-static responses, etc. + + return new Response('Not found', { status: 404 }); +} +`; + + // Write out the files + for (const [filename, content] of Object.entries(resourceFiles)) { + const filePath = path.resolve(computeJsDir, filename); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, content, 'utf-8'); + } + + console.log("πŸš€ Compute application created!"); + + console.log('Installing dependencies...'); + console.log(`npm --prefix ${COMPUTE_JS_DIR} install`); + child_process.spawnSync('npm', [ '--prefix', COMPUTE_JS_DIR, 'install' ], { stdio: 'inherit' }); + console.log(''); + + console.log(''); + console.log('To run your Compute application locally:'); + console.log(''); + console.log(' cd ' + COMPUTE_JS_DIR); + console.log(' npm run dev:publish'); + console.log(' npm run dev:start'); + console.log(''); + console.log('To build and deploy to your Compute service:'); + console.log(''); + console.log(' cd ' + COMPUTE_JS_DIR); + console.log(' npm run fastly:deploy'); + console.log(' npm run fastly:publish'); + console.log(''); +} diff --git a/src/cli/compression/brotli.ts b/src/cli/compression/brotli.ts index 0cf9e37..34d7487 100644 --- a/src/cli/compression/brotli.ts +++ b/src/cli/compression/brotli.ts @@ -1,19 +1,17 @@ -import fs from 'fs'; -import zlib from 'zlib'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import zlib from 'node:zlib'; export const key = 'br'; -export async function compressTo(src: string, dest: string, isText: boolean): Promise { +export async function compressTo(src: string, dest: string): Promise { const buffer = fs.readFileSync(src); const resultBuffer = zlib.brotliCompressSync(buffer); - - // Don't actually create the file if it would be bigger - if (resultBuffer.length < buffer.length) { - fs.writeFileSync(dest, resultBuffer); - return true; - } else { - return false; - } + fs.writeFileSync(dest, resultBuffer); } diff --git a/src/cli/compression/gzip.ts b/src/cli/compression/gzip.ts index ab18232..16e6b26 100644 --- a/src/cli/compression/gzip.ts +++ b/src/cli/compression/gzip.ts @@ -1,19 +1,17 @@ -import fs from 'fs'; -import zlib from 'zlib'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import zlib from 'node:zlib'; export const key = 'gzip'; -export async function compressTo(src: string, dest: string, isText: boolean): Promise { +export async function compressTo(src: string, dest: string): Promise { const buffer = fs.readFileSync(src); const resultBuffer = zlib.gzipSync(buffer); - - // Don't actually create the file if it would be bigger - if (resultBuffer.length < buffer.length) { - fs.writeFileSync(dest, resultBuffer); - return true; - } else { - return false; - } + fs.writeFileSync(dest, resultBuffer); } diff --git a/src/cli/compression/index.ts b/src/cli/compression/index.ts index 936c857..e80b6f7 100644 --- a/src/cli/compression/index.ts +++ b/src/cli/compression/index.ts @@ -1,23 +1,18 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + import * as brotli from './brotli.js'; import * as gzip from './gzip.js'; -import { compressionTypes, ContentCompressionTypes } from "../../constants/compression.js"; +import { type ContentCompressionTypes } from '../../models/compression/index.js'; -export type CompressAlg = (src: string, dest: string, text: boolean) => Promise; +export type CompressAlg = (src: string, dest: string) => Promise; -export type AlgLib = { - key: string, - compressTo: CompressAlg, +const algs: Record = { + br: brotli.compressTo, + gzip: gzip.compressTo, }; -const libs: AlgLib[] = [ brotli, gzip ]; - -const algs = compressionTypes.reduce>>((obj, alg) => { - const lib = libs.find((lib: AlgLib) => lib.key === alg); - if (lib != null) { - obj[alg] = lib.compressTo; - } - return obj; -}, {}); - export { algs }; diff --git a/src/cli/index.ts b/src/cli/index.ts index 053ab73..6a6471d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,115 +1,22 @@ #!/usr/bin/env node +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ -import commandLineArgs, { OptionDefinition } from "command-line-args"; +import fs from 'node:fs'; -import { initApp } from "./commands/init-app.js"; -import { buildStaticLoader } from "./commands/build-static.js"; -import { cleanKVStore } from "./commands/clean-kv-store.js"; +import * as scaffoldCommand from './commands/scaffold/index.js'; +import * as manageCommands from './commands/manage/index.js'; -const optionDefinitions: OptionDefinition[] = [ - // (optional) Should be one of: - // - cra (or create-react-app) - // - vite - // - sveltekit - // - vue - // - next - // - astro - // - gatsby - // - docusaurus - { name: 'preset', type: String, }, +if (!fs.existsSync('./static-publish.rc.js')) { - { name: 'build-static', type: Boolean }, - { name: 'suppress-framework-warnings', type: Boolean }, - { name: 'output', alias: 'o', type: String, defaultValue: './compute-js', }, + console.log("πŸ§‘β€πŸ’»Fastly Compute JavaScript Static Publisher (Scaffolding mode)"); + await scaffoldCommand.action(process.argv); - // Whether the scaffolded project should use webpack to bundle assets. - { name: 'webpack', type: Boolean, defaultValue: false }, +} else { - // The 'root' directory for the publishing. - // All assets are expected to exist under this root. Required. - // For backwards compatibility, if this value is not provided, - // then the value of 'public-dir' is used. - { name: 'root-dir', type: String, }, + console.log("πŸ§‘β€πŸ’»Fastly Compute JavaScript Static Publisher (Management mode)"); + await manageCommands.action(process.argv); - // The 'public' directory. The Publisher Server will - // resolve requests relative to this directory. If not specified, - // defaults to the same value as 'root-dir'. See README for - // details. - { name: 'public-dir', type: String, }, - - // Directories to specify as containing 'static' files. The - // Publisher Server will serve files from these directories - // with a long TTL. - { name: 'static-dir', type: String, multiple: true, }, - - // The 'static content root directory' where the Static Publisher - // outputs its metadata files and loaders. - { name: 'static-content-root-dir', type: String, }, - - // Path to a file to be used to serve in a SPA application. - // The Publisher Server will serve this file with a 200 status code - // when the request doesn't match a known file, and the accept - // header includes text/html. You may use the '[public-dir]' token - // if you wish to specify this as a relative path from the 'public-dir'. - { name: 'spa', type: String, }, - - // Path to a file to be used to serve as a 404 not found page. - // The Publisher Server will serve this file with a 404 status code - // when the request doesn't match a known file, and the accept - // header includes text/html. You may use the '[public-dir]' token - // if you wish to specify this as a relative path from the 'public-dir'. - { name: 'not-found-page', type: String, }, - - // List of files to automatically use as index, for example, index.html,index.htm - // If a request comes in but the route does not exist, we check the route - // plus a slash plus the items in this array. - { name: 'auto-index', type: String, multiple: true, }, - - // List of extensions to apply to a path name, for example, if - // http://example.com/about is requested, we can respond with http://example.com/about.html - { name: 'auto-ext', type: String, multiple: true, }, - - // Components from fastly.toml - - // Name of the application, to be inserted into fastly.toml - { name: 'name', type: String, }, - - // Name of the author, to be inserted into fastly.toml - { name: 'author', type: String, }, - - // Description of the application, to be inserted into fastly.toml - { name: 'description', type: String, }, - - // Fastly Service ID to be added to the fastly.toml that is generated. - { name: 'service-id', type: String }, - - // The name of a Fastly KV Store to hold the content assets. - // It must be linked to the service specified by `--service-id`. - { name: 'kv-store-name', type: String }, - - // Clean KV Store mode - { name: 'clean-kv-store', type: Boolean, }, -]; - -const commandLineValues = commandLineArgs(optionDefinitions); - -console.log("Fastly Compute JavaScript Static Publisher"); - -let mode: 'init-app' | 'build-static' | 'clean-kv-store' = 'init-app'; -if(commandLineValues['build-static']) { - mode = 'build-static'; -} else if(commandLineValues['clean-kv-store']) { - mode = 'clean-kv-store'; -} - -switch(mode) { -case 'build-static': - await buildStaticLoader(commandLineValues); - break; -case 'init-app': - initApp(commandLineValues); - break; -case 'clean-kv-store': - await cleanKVStore(commandLineValues); - break; } diff --git a/src/cli/presets/implementations/astro.ts b/src/cli/presets/implementations/astro.ts deleted file mode 100644 index adf4153..0000000 --- a/src/cli/presets/implementations/astro.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class AstroPreset implements IPresetBase { - name = 'Astro'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-astro-app', - description: 'Fastly Compute static site from Astro', - }; - check(packageJson: any) { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a Astro project directory."); - return false; - } - if(packageJson?.dependencies?.['astro'] == null) { - console.error("❌ Can't find astro in dependencies"); - console.error("Run this from a Astro project directory."); - return false; - } - return true; - } -} diff --git a/src/cli/presets/implementations/create-react-app.ts b/src/cli/presets/implementations/create-react-app.ts deleted file mode 100644 index e463e20..0000000 --- a/src/cli/presets/implementations/create-react-app.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class CreateReactAppPreset implements IPresetBase { - name = 'Create React App'; - defaultOptions: Partial = { - rootDir: './build', - staticDirs: [ '[public-dir]/static' ], - name: 'my-create-react-app', - description: 'Fastly Compute static site from create-react-app', - }; - - check(packageJson: any): boolean { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a create-react-app project directory."); - return false; - } - if(packageJson?.dependencies?.['react-scripts'] == null) { - console.error("❌ Can't find react-scripts in dependencies"); - console.error("Run this from a create-react-app project directory."); - console.log("If this is a project created with create-react-app and has since been ejected, specify preset cra-eject to skip this check."); - return false; - } - return true; - } - -} - -export class CreateReactAppEjectedPreset extends CreateReactAppPreset { - name = 'Create React App (Ejected)'; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/docusaurus.ts b/src/cli/presets/implementations/docusaurus.ts deleted file mode 100644 index e383892..0000000 --- a/src/cli/presets/implementations/docusaurus.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class DocusaurusPreset implements IPresetBase { - name = 'Docusaurus'; - defaultOptions: Partial = { - rootDir: './build', - name: 'my-docusaurus-app', - description: 'Fastly Compute static site from docusaurus', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/gatsby.ts b/src/cli/presets/implementations/gatsby.ts deleted file mode 100644 index 46aea0c..0000000 --- a/src/cli/presets/implementations/gatsby.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class GatsbyPreset implements IPresetBase { - name = 'Gatsby'; - defaultOptions: Partial = { - rootDir: './public', - name: 'my-gatsby-app', - description: 'Fastly Compute static site from Gatsby', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/next.ts b/src/cli/presets/implementations/next.ts deleted file mode 100644 index c39dfd7..0000000 --- a/src/cli/presets/implementations/next.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class NextJsPreset implements IPresetBase { - name = 'Next.js'; - defaultOptions: Partial = { - rootDir: './out', - name: 'my-next-app', - description: 'Fastly Compute static site from Next.js', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/sveltekit.ts b/src/cli/presets/implementations/sveltekit.ts deleted file mode 100644 index 44f5fea..0000000 --- a/src/cli/presets/implementations/sveltekit.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class SvelteKitPreset implements IPresetBase { - name = 'SvelteKit'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-sveltekit-app', - description: 'Fastly Compute static site from SvelteKit', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/vite.ts b/src/cli/presets/implementations/vite.ts deleted file mode 100644 index 9740564..0000000 --- a/src/cli/presets/implementations/vite.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class VitePreset implements IPresetBase { - name = 'Vite'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-vite-app', - description: 'Fastly Compute static site from Vite', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/vue.ts b/src/cli/presets/implementations/vue.ts deleted file mode 100644 index 5b960ed..0000000 --- a/src/cli/presets/implementations/vue.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class VuePreset implements IPresetBase { - name = 'Vue'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-vue-app', - description: 'Fastly Compute static site from Vue (create-vue)', - }; - check(packageJson: any) { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a Vue project directory created by create-vue."); - return false; - } - if(packageJson?.devDependencies?.['vite'] == null) { - console.error("❌ Can't find vite in dependencies"); - console.error("Run this from a Vue project directory created by create-vue."); - console.log("If this is a project created with the Vue CLI, migrate it to use Vite first. Refer to the create-vue documentation at https://www.npmjs.com/package/create-vue (not authored or maintained by Fastly) for details on this process."); - return false; - } - return true; - } -} diff --git a/src/cli/presets/index.ts b/src/cli/presets/index.ts deleted file mode 100644 index 5fbf809..0000000 --- a/src/cli/presets/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { IPresetBase } from './preset-base.js'; -import { CreateReactAppPreset, CreateReactAppEjectedPreset } from './implementations/create-react-app.js'; -import { VitePreset } from './implementations/vite.js'; -import { SvelteKitPreset } from './implementations/sveltekit.js'; -import { VuePreset } from './implementations/vue.js'; -import { NextJsPreset } from './implementations/next.js'; -import { AstroPreset } from "./implementations/astro.js"; -import { GatsbyPreset } from './implementations/gatsby.js'; -import { DocusaurusPreset } from './implementations/docusaurus.js'; - -export const presets: Record IPresetBase> = { - 'cra': CreateReactAppPreset, - 'create-react-app': CreateReactAppPreset, - 'cra-eject': CreateReactAppEjectedPreset, - 'vite': VitePreset, - 'sveltekit': SvelteKitPreset, - 'vue': VuePreset, - 'next': NextJsPreset, - 'astro': AstroPreset, - 'gatsby': GatsbyPreset, - 'docusaurus': DocusaurusPreset, -}; diff --git a/src/cli/presets/preset-base.ts b/src/cli/presets/preset-base.ts deleted file mode 100644 index 098258d..0000000 --- a/src/cli/presets/preset-base.ts +++ /dev/null @@ -1,21 +0,0 @@ -export type AppOptions = { - rootDir: string | undefined, - publicDir: string | undefined, - staticDirs: string[], - staticContentRootDir: string | undefined, - spa: string | undefined, - notFoundPage: string | undefined, - autoIndex: string[], - autoExt: string[], - name: string | undefined, - author: string | undefined, - description: string | undefined, - serviceId: string | undefined, - kvStoreName: string | undefined, -}; - -export interface IPresetBase { - name: string; - defaultOptions: Partial; - check(packageJson: any | null, options: AppOptions): boolean; -} diff --git a/src/cli/util/fastly-api.ts b/src/cli/util/api-token.ts similarity index 53% rename from src/cli/util/fastly-api.ts rename to src/cli/util/api-token.ts index d52719c..7c0be5a 100644 --- a/src/cli/util/fastly-api.ts +++ b/src/cli/util/api-token.ts @@ -1,32 +1,61 @@ -import { execSync } from 'child_process'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { spawnSync } from 'node:child_process'; +import cli from '@fastly/cli'; + import { makeRetryable } from './retryable.js'; export interface FastlyApiContext { apiToken: string, -}; +} -export type LoadApiKeyResult = { +export type LoadApiTokenResult = { apiToken: string, source: string, }; -export function loadApiKey(): LoadApiKeyResult | null { +export type LoadApiParams = { + commandLine: any, +}; + +export function loadApiToken(params: LoadApiParams): LoadApiTokenResult | null { let apiToken: string | null = null; let source: string = ''; - // Try to get API key from FASTLY_API_TOKEN - apiToken = process.env.FASTLY_API_TOKEN || null; - if (apiToken != null) { - source = 'env'; + // Try command line arg + if (typeof params.commandLine === 'string') { + apiToken = params.commandLine.trim(); + source = 'commandline'; } + // Try env (FASTLY_API_TOKEN) + if (apiToken == null) { + apiToken = process.env.FASTLY_API_TOKEN || null; + if (apiToken != null) { + source = 'env'; + } + } + + // Try fastly cli if (apiToken == null) { - // Try to get API key from fastly cli try { - apiToken = execSync('fastly profile token --quiet', { + const { stdout, error, status } = spawnSync(cli, ['profile', 'token', '--quiet'], { encoding: 'utf-8', - })?.trim() || null; + }); + if (status != null && status !== 0) { + console.warn(`⚠️ Warning: 'fastly profile token' returned a non-zero status code.`); + apiToken = null; + } else if (error) { + console.warn(`⚠️ Warning: 'fastly profile token' returned an error:`); + console.warn(String(error)); + apiToken = null; + } else { + apiToken = stdout.trim(); + } } catch { apiToken = null; } @@ -63,6 +92,15 @@ export class FetchError extends Error { status: number; } +function isReadableStream(data: unknown): data is ReadableStream { + return ( + typeof data === 'object' && + data !== null && + typeof (data as ReadableStream).getReader === 'function' && + typeof (data as ReadableStream).tee === 'function' + ); +} + export async function callFastlyApi( fastlyApiContext: FastlyApiContext, endpoint: string, @@ -71,16 +109,13 @@ export async function callFastlyApi( requestInit?: RequestInit, ): Promise { - let finalEndpoint = endpoint; + const url = new URL(endpoint, 'https://api.fastly.com/'); if (queryParams != null) { - const queryString = String(queryParams); - if (queryString.length > 0) { - finalEndpoint += '?' + queryString; + for (const [key, value] of queryParams.entries()) { + url.searchParams.append(key, value); } } - const url = new URL(finalEndpoint, 'https://api.fastly.com/'); - const headers = new Headers(requestInit?.headers); headers.set('Fastly-Key', fastlyApiContext.apiToken); @@ -88,7 +123,9 @@ export async function callFastlyApi( ...requestInit, headers, redirect: 'error', + ...(isReadableStream(requestInit?.body) ? { duplex: 'half' } as RequestInit : null), }); + let response; try { response = await fetch(request); diff --git a/src/cli/util/args.ts b/src/cli/util/args.ts new file mode 100644 index 0000000..9d62999 --- /dev/null +++ b/src/cli/util/args.ts @@ -0,0 +1,137 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import commandLineArgs, { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; + +export type ModeAction = (argv: string[]) => void | Promise; +export type ActionModule = { action: ModeAction }; +export type ActionTable = Record; + +const helpOptionsDefs: OptionDefinition[] = [ + { name: 'help', alias: 'h', type: Boolean }, +]; + +export function isHelpArgs(argv: string[]) { + + const helpOptions = commandLineArgs(helpOptionsDefs, { argv, stopAtFirstUnknown: true }); + return !!helpOptions['help']; + +} + +export function findMainCommandNameAndArgs(argv: string[]): [string | null, string[]] { + + const mainDefinitions = [ + { name: 'command', type: String, defaultOption: true }, + ]; + const mainOptions = commandLineArgs(mainDefinitions, { argv, stopAtFirstUnknown: true }); + const commandArgs = mainOptions._unknown || []; + + const command = mainOptions['command']; + if (typeof command !== 'string') { + return [ null, commandArgs ]; + } + + return [ command, commandArgs ]; + +} + +export type CommandAndArgs = { + needHelp: false, + command: T, + argv: string[], +}; + +export type CommandAndArgsHelp = { + needHelp: true, + error: string | null, +} + +export type CommandAndArgsResult = CommandAndArgs | CommandAndArgsHelp; + +export function getCommandAndArgs( + argv: string[], + actions: ActionTable, +): CommandAndArgsResult { + if (isHelpArgs(argv)) { + return { + needHelp: true, + error: null, + }; + } + + const result = findMainCommandNameAndArgs(argv); + + const [ command, actionArgv ] = result; + if (command != null) { + for (const modeName of Object.keys(actions)) { + if (command === modeName) { + return { + needHelp: false, + command: command as T, + argv: actionArgv, + }; + } + } + return { + needHelp: true, + error: `Unknown command: ${command}`, + }; + } + + let error = null; + try { + commandLineArgs([], { argv: actionArgv }); + } catch(err) { + error = String(err); + } + + return { + needHelp: true, + error, + }; +} + +export type CommandLineParsed = { + needHelp: false, + commandLineOptions: CommandLineOptions, +}; + +export type CommandLineHelp = { + needHelp: true, + error: Error | null, +}; + +export type CommandLineResult = CommandLineParsed | CommandLineHelp; + +export function parseCommandLine( + argv: string[], + optionDefinitions: OptionDefinition[], +): CommandLineResult { + let commandLineOptions; + try { + commandLineOptions = commandLineArgs([ + ...helpOptionsDefs, + ...optionDefinitions, + ], { argv }); + } catch(err) { + const error = err instanceof Error ? err : new Error(String(err)); + return { + needHelp: true, + error, + } + } + + if (!!commandLineOptions['help']) { + return { + needHelp: true, + error: null, + }; + } + + return { + needHelp: false, + commandLineOptions, + }; +} diff --git a/src/cli/load-config.ts b/src/cli/util/config.ts similarity index 63% rename from src/cli/load-config.ts rename to src/cli/util/config.ts index 4dc137d..a179a9a 100644 --- a/src/cli/load-config.ts +++ b/src/cli/util/config.ts @@ -1,169 +1,179 @@ -import url from "url"; -import path from "path"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import path from 'node:path'; + import globToRegExp from 'glob-to-regexp'; +import { + type PublishContentConfigNormalized, + type ContentTypeDef, +} from '../../models/config/publish-content-config.js'; import { buildNormalizeFunctionForArray, buildNormalizeFunctionForObject, isSpecified, isStringAndNotEmpty, -} from "./util/data.js"; +} from './data.js'; +import { type PublisherServerConfigNormalized } from '../../models/config/publisher-server-config.js'; +import { type StaticPublishRc } from '../../models/config/static-publish-rc.js'; -import type { - ContentTypeDef, -} from "../types/content-types.js"; -import type { - StaticPublisherConfigNormalized, - PublisherServerConfigNormalized, -} from "../types/config-normalized.js"; +export class LoadConfigError extends Error { + errors: string[]; -const normalizeContentTypeDef = buildNormalizeFunctionForObject((config, errors): ContentTypeDef | null => { + constructor(configFilePath: string, errors: string[]) { + super(`Error loading config file ${configFilePath}`); + this.errors = errors; + } +} - let { test, contentType, text } = config; +export async function loadStaticPublisherRcFile(): Promise { - if (typeof test === 'function' || test instanceof RegExp) { - // ok - } else { - errors.push('test cannot be null.'); - } + let configRaw; - if (isStringAndNotEmpty(contentType)) { - // ok - } else { - errors.push('contentType must be a non-empty string.'); + const configFile = './static-publish.rc.js'; + + try { + const filePath = path.resolve(configFile); + configRaw = (await import(filePath)).default; + } catch { + // } - if (!isSpecified(config, 'text')) { - text = true; - } else { - if (typeof text === 'boolean') { - // ok - } else { - errors.push('text, if specified, must be a boolean value.'); - } + if (configRaw == null) { + throw new LoadConfigError(configFile, [ + 'Unable to load ' + configFile, + ]); } - if (errors.length > 0) { - return null; + const errors: string[] = []; + const config = normalizeStaticPublisherRc(configRaw, errors); + if (config == null) { + throw new LoadConfigError(configFile, errors); } - return { - test, - contentType, - text, - }; + return config; -}); +} -const normalizeContentTypeDefs = buildNormalizeFunctionForArray((config, errors) => { - return normalizeContentTypeDef(config, errors); -}); -const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { +export const normalizeStaticPublisherRc = buildNormalizeFunctionForObject((config, errors) => { - let { publicDirPrefix, staticItems, compression, spaFile, notFoundPageFile, autoExt, autoIndex } = config; + let { + kvStoreName, + publishId, + defaultCollectionName, + staticPublisherWorkingDir, + } = config; - if (!isSpecified(config, 'publicDirPrefix')) { - publicDirPrefix = ''; + if (!isSpecified(config, 'kvStoreName')) { + errors.push('kvStoreName must be specified.'); } else { - if (typeof publicDirPrefix === 'string') { + if (isStringAndNotEmpty(kvStoreName)) { // ok } else { - errors.push('publicDirPrefix, if specified, must be a string value.'); + errors.push('kvStoreName must be a non-empty string.'); } } - if (!isSpecified(config, 'staticItems')) { - staticItems = []; + if (!isSpecified(config, 'publishId')) { + errors.push('publishId must be specified.'); } else { - if (staticItems === null || staticItems === false) { - staticItems = []; - } - if (!Array.isArray(staticItems)) { - staticItems = [ staticItems ]; - } - if (staticItems.every((x: any) => typeof x === 'string')) { - staticItems = (staticItems as string[]).map((x, index) => { - if (x.includes('*')) { - let re; - try { - const regexp = globToRegExp(x, {globstar: true}); - re = 're:' + String(regexp); - } catch { - errors.push(`staticItems item at index ${index}, '${x}', cannot be parsed as glob pattern.`); - re = null; - } - return re; - } - return x; - }); + if (isStringAndNotEmpty(publishId)) { + // ok } else { - errors.push('staticItems, if specified, must be a string value, an array of string values, false, or null'); + errors.push('publishId must be a non-empty string.'); } } - if (!isSpecified(config, 'compression')) { - compression = [ 'br', 'gzip' ]; - } else if (compression === null) { - compression = [] + if (!isSpecified(config, 'defaultCollectionName')) { + errors.push('defaultCollectionName must be specified.'); } else { - if (!Array.isArray(compression)) { - compression = [ compression ]; - } - if (compression.some((x: any) => x !== 'br' && x !== 'gzip')) { - errors.push(`compression, if specified, must be null or an array and can only contain 'br' and 'gzip'.`); + if (isStringAndNotEmpty(defaultCollectionName)) { + // ok + } else { + errors.push('defaultCollectionName must be a non-empty string.'); } } - if (!isSpecified(config, 'spaFile')) { - spaFile = null; + if (!isSpecified(config, 'staticPublisherWorkingDir')) { + errors.push('staticPublisherWorkingDir must be specified.'); } else { - if (typeof spaFile === 'string' || spaFile === null) { + if ( + staticPublisherWorkingDir.startsWith('./') && + staticPublisherWorkingDir !== './' && + !staticPublisherWorkingDir.includes('//') + ) { // ok - } else if (spaFile === false) { - spaFile = null; } else { - errors.push('spaFile, if specified, must be a string value, false, or null'); + errors.push('staticPublisherWorkingDir must be a relative subdirectory.'); + } + + while (staticPublisherWorkingDir.endsWith('/')) { + staticPublisherWorkingDir = staticPublisherWorkingDir.slice(0, -1); } } - if (!isSpecified(config, 'notFoundPageFile')) { - notFoundPageFile = null; + return { + kvStoreName, + publishId, + defaultCollectionName, + staticPublisherWorkingDir, + }; +}); + +export async function loadPublishContentConfigFile(configFile: string): Promise { + + let configRaw; + + try { + const filePath = path.resolve(configFile); + configRaw = (await import(filePath)).default; + } catch { + // + } + + if (configRaw == null) { + throw new LoadConfigError(configFile, [ + 'Unable to load ' + configFile, + ]); + } + + const errors: string[] = []; + const config = normalizePublishContentConfig(configRaw, errors); + if (config == null) { + throw new LoadConfigError(configFile, errors); + } + + return config; + +} + +const normalizeContentTypeDef = buildNormalizeFunctionForObject((config, errors): ContentTypeDef | null => { + + let { test, contentType, text } = config; + + if (typeof test === 'function' || test instanceof RegExp) { + // ok } else { - if (typeof notFoundPageFile === 'string' || notFoundPageFile === null) { - // ok - } else if (notFoundPageFile === false) { - notFoundPageFile = null; - } else { - errors.push('notFoundPageFile, if specified, must be a string value, false, or null'); - } + errors.push('test cannot be null.'); } - if (!isSpecified(config, 'autoExt')) { - autoExt = []; + if (isStringAndNotEmpty(contentType)) { + // ok } else { - if (Array.isArray(autoExt) && autoExt.every(e => typeof e === 'string')) { - // ok - } else if (autoExt === false || autoExt === null) { - autoExt = []; - } else if (typeof autoExt === 'string') { - autoExt = [ autoExt ]; - } else { - errors.push('autoExt, if specified, must be an array of string values, a string value, false, or null'); - } + errors.push('contentType must be a non-empty string.'); } - if (!isSpecified(config, 'autoIndex')) { - autoIndex = []; + if (!isSpecified(config, 'text')) { + text = true; } else { - if (Array.isArray(autoIndex) && autoIndex.every(e => typeof e === 'string')) { + if (typeof text === 'boolean') { // ok - } else if (autoIndex === false || autoIndex === null) { - autoIndex = []; - } else if (typeof autoIndex === 'string') { - autoIndex = [ autoIndex ]; } else { - errors.push('autoIndex, if specified, must be an array of string values, a string value, false, or null'); + errors.push('text, if specified, must be a boolean value.'); } } @@ -172,29 +182,26 @@ const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { +const normalizeContentTypeDefs = buildNormalizeFunctionForArray((config, errors) => { + return normalizeContentTypeDef(config, errors); +}); + +export const normalizePublishContentConfig = buildNormalizeFunctionForObject((config, errors) => { let { rootDir, - staticContentRootDir, - kvStoreName, excludeDirs, excludeDotFiles, includeWellKnown, - contentAssetInclusionTest, + kvStoreAssetInclusionTest, contentCompression, - moduleAssetInclusionTest, contentTypes, server, } = config; @@ -209,36 +216,6 @@ const normalizeConfig = buildNormalizeFunctionForObject { +export const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { - let raw: any = undefined; - const staticPublishRcPath = path.resolve('./static-publish.rc.js'); - const staticPublishRcFileURL = String(url.pathToFileURL(staticPublishRcPath)); - try { - raw = (await import(staticPublishRcFileURL)).default; - } catch { - errors.push('Unable to load ' + staticPublishRcFileURL); + let { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + } = config; + + if (!isSpecified(config, 'publicDirPrefix')) { + publicDirPrefix = ''; + } else { + if (typeof publicDirPrefix === 'string') { + // ok + } else { + errors.push('publicDirPrefix, if specified, must be a string value.'); + } } - let normalized: any = undefined; - if (raw != null) { - normalized = normalizeConfig(raw, errors); + if (!isSpecified(config, 'staticItems')) { + staticItems = []; + } else { + if (staticItems === null || staticItems === false) { + staticItems = []; + } + if (!Array.isArray(staticItems)) { + staticItems = [ staticItems ]; + } + if (staticItems.every((x: any) => typeof x === 'string')) { + staticItems = (staticItems as string[]).map((x, index) => { + if (x.includes('*')) { + let re; + try { + const regexp = globToRegExp(x, {globstar: true}); + re = 're:' + String(regexp); + } catch { + errors.push(`staticItems item at index ${index}, '${x}', cannot be parsed as glob pattern.`); + re = null; + } + return re; + } + return x; + }); + } else { + errors.push('staticItems, if specified, must be a string value, an array of string values, false, or null'); + } } - return { normalized: normalized ?? null, raw }; + if (!isSpecified(config, 'allowedEncodings')) { + allowedEncodings = [ 'br', 'gzip' ]; + } else if (allowedEncodings === null) { + allowedEncodings = [] + } else { + if (!Array.isArray(allowedEncodings)) { + allowedEncodings = [ allowedEncodings ]; + } + if (allowedEncodings.some((x: any) => x !== 'br' && x !== 'gzip')) { + errors.push(`allowedEncodings, if specified, must be null or an array and can only contain 'br' and 'gzip'.`); + } + } -} + if (!isSpecified(config, 'spaFile')) { + spaFile = null; + } else { + if (typeof spaFile === 'string' || spaFile === null) { + // ok + } else if (spaFile === false) { + spaFile = null; + } else { + errors.push('spaFile, if specified, must be a string value, false, or null'); + } + } + + if (!isSpecified(config, 'notFoundPageFile')) { + notFoundPageFile = null; + } else { + if (typeof notFoundPageFile === 'string' || notFoundPageFile === null) { + // ok + } else if (notFoundPageFile === false) { + notFoundPageFile = null; + } else { + errors.push('notFoundPageFile, if specified, must be a string value, false, or null'); + } + } + + if (!isSpecified(config, 'autoExt')) { + autoExt = []; + } else { + if (Array.isArray(autoExt) && autoExt.every(e => typeof e === 'string')) { + // ok + } else if (autoExt === false || autoExt === null) { + autoExt = []; + } else if (typeof autoExt === 'string') { + autoExt = [ autoExt ]; + } else { + errors.push('autoExt, if specified, must be an array of string values, a string value, false, or null'); + } + } + + if (!isSpecified(config, 'autoIndex')) { + autoIndex = []; + } else { + if (Array.isArray(autoIndex) && autoIndex.every(e => typeof e === 'string')) { + // ok + } else if (autoIndex === false || autoIndex === null) { + autoIndex = []; + } else if (typeof autoIndex === 'string') { + autoIndex = [ autoIndex ]; + } else { + errors.push('autoIndex, if specified, must be an array of string values, a string value, false, or null'); + } + } + + if (errors.length > 0) { + return null; + } + + return { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + }; + +}); diff --git a/src/util/content-types.ts b/src/cli/util/content-types.ts similarity index 95% rename from src/util/content-types.ts rename to src/cli/util/content-types.ts index 9777c0b..73b8fda 100644 --- a/src/util/content-types.ts +++ b/src/cli/util/content-types.ts @@ -1,7 +1,12 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + import { - ContentTypeDef, - ContentTypeTestResult, -} from "../types/content-types.js"; + type ContentTypeDef, + type ContentTypeTestResult, +} from '../../models/config/publish-content-config.js'; const defaultContentTypes: ContentTypeDef[] = [ // Text formats @@ -38,10 +43,6 @@ const defaultContentTypes: ContentTypeDef[] = [ { test: /\.woff2$/, contentType: 'font/woff2', text: false }, ]; -export function getDefaultContentTypes() { - return defaultContentTypes; -} - export function mergeContentTypes(contentTypes: ContentTypeDef[]) { const finalContentTypes: ContentTypeDef[] = []; diff --git a/src/cli/util/data.ts b/src/cli/util/data.ts index a30b0b5..0433bdf 100644 --- a/src/cli/util/data.ts +++ b/src/cli/util/data.ts @@ -1,11 +1,16 @@ -export type NormalizeAction = (value: any, errors: string[]) => T | null; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type NormalizeAction = (value: Record, errors: string[]) => T | null; export function buildNormalizeFunctionForObject(action: NormalizeAction) { - return (obj: any, errCtx: string[] = []): TObject | null => { + return (obj: unknown, errCtx: string[] = []): TObject | null => { - if (obj == null) { - errCtx.push('obj cannot be null.'); + if (obj == null || typeof obj !== 'object') { + errCtx.push('obj must be an object.'); return null; } @@ -25,7 +30,7 @@ export function buildNormalizeFunctionForObject(action: NormalizeAction export function buildNormalizeFunctionForArray(action: NormalizeAction) { - return (obj: any, errCtx: string[] = []): TEntry[] | null => { + return (obj: unknown, errCtx: string[] = []): TEntry[] | null => { if (!Array.isArray(obj)) { errCtx.push('obj must be array.'); diff --git a/src/cli/util/fastly-toml.ts b/src/cli/util/fastly-toml.ts new file mode 100644 index 0000000..7bf8a11 --- /dev/null +++ b/src/cli/util/fastly-toml.ts @@ -0,0 +1,17 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'fs'; +import toml from 'toml'; + +export function readServiceId(filePath: string) { + const text = fs.readFileSync(filePath, 'utf8'); + const parsed = toml.parse(text); + const serviceId = parsed.service_id; + if (typeof serviceId === 'string') { + return serviceId; + } + return undefined; +} diff --git a/src/cli/util/files.ts b/src/cli/util/files.ts index 62760f2..24ee2fd 100644 --- a/src/cli/util/files.ts +++ b/src/cli/util/files.ts @@ -1,24 +1,30 @@ -import fs from "fs"; -import path from "path"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; export interface FilenameTest { test(name: string): boolean; } -export type GetFilesOpts = { +export type EnumerateFilesOpts = { publicDirRoot: string, excludeDirs: FilenameTest[], excludeDotFiles: boolean, includeWellKnown: boolean, }; -export function getFiles(dir: string, opts: GetFilesOpts) { +export function enumerateFiles(opts: EnumerateFilesOpts) { const results: string[] = []; - getFilesWorker(results, dir, opts); + enumerateFilesWorker(results, opts.publicDirRoot, opts); return results; } -function getFilesWorker(results: string[], dir: string, opts: GetFilesOpts) { +function enumerateFilesWorker(results: string[], dir: string, opts: EnumerateFilesOpts) { const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { const { name } = entry; @@ -45,9 +51,62 @@ function getFilesWorker(results: string[], dir: string, opts: GetFilesOpts) { } if (entry.isDirectory()) { - getFilesWorker(results, fullpath, opts); + enumerateFilesWorker(results, fullpath, opts); } else { results.push(fullpath); } } } + +export function getFileSize(filePath: string) { + const stats = fs.statSync(filePath); + return stats.size; +} + +export function directoryExists(dirPath: string) { + try { + const stats = fs.statSync(dirPath); + return stats.isDirectory(); + } catch (err) { + if ((err as any).code === 'ENOENT') { + return false; // Path doesn't exist + } + throw err; // Other errors, rethrow + } +} + +export async function calculateFileSizeAndHash(filePath: string) { + + const stream = fs.createReadStream(filePath); + + // Use createHash, rather than subtle, since it supports streaming. + const hash = crypto.createHash('sha256'); + let size = 0; + + return new Promise<{ size: number, hash: string }>((resolve, reject) => { + stream.on('data', chunk => { + hash.update(chunk); + size += chunk.length; + }); + + stream.on('end', () => { + resolve({ + size, + hash: hash.digest('hex'), + }); + }); + + stream.on('error', reject); + }); + +} + +export function rootRelative(itemPath: string) { + return dotRelative(null, itemPath); +} + +export function dotRelative(from: string | null, to: string) { + const relPath = path.relative(from ?? path.resolve(), to); + return relPath.startsWith('..') ? relPath : './' + relPath; +} + diff --git a/src/cli/util/hash.ts b/src/cli/util/hash.ts deleted file mode 100644 index e12e782..0000000 --- a/src/cli/util/hash.ts +++ /dev/null @@ -1,14 +0,0 @@ -import crypto from 'crypto'; -import fs from 'fs'; - -export function calculateFileSizeAndHash(filename: string) { - - const fileBuffer = fs.readFileSync(filename); - const size = fileBuffer.length; - const hash = crypto.createHash('sha256'); - - hash.update(fileBuffer); - - return { size, hash: hash.digest('hex') }; - -} diff --git a/src/cli/util/id.ts b/src/cli/util/id.ts deleted file mode 100644 index d9e64b3..0000000 --- a/src/cli/util/id.ts +++ /dev/null @@ -1,22 +0,0 @@ -import crypto from "crypto"; - -const ID_SYMBOLS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; -const ID_SYMBOLS_BASE = BigInt(ID_SYMBOLS.length); - -export function createStringId() { - return uuidToString(crypto.randomUUID()); -} - -export function uuidToString(uuid: string) { - const uuidWithoutHyphens = uuid.replace(/-/g, ''); - - let number = BigInt('0x' + uuidWithoutHyphens); - let result = ''; - do { - const [ quotient, remainder ] = [ number / ID_SYMBOLS_BASE, Number(number % ID_SYMBOLS_BASE) ]; - result = ID_SYMBOLS.charAt( remainder ) + result; - number = quotient; - } while(number > 0n); - - return result; -} diff --git a/src/cli/util/kv-store-items.ts b/src/cli/util/kv-store-items.ts new file mode 100644 index 0000000..7622a07 --- /dev/null +++ b/src/cli/util/kv-store-items.ts @@ -0,0 +1,205 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import { directoryExists, getFileSize } from './files.js'; +import { attemptWithRetries } from './retryable.js'; +import { FetchError } from './api-token.js'; + +export async function doKvStoreItemsOperation( + objects: TObject[], + fn: (obj: TObject, key: string, index: number) => Promise, + maxConcurrent: number = 12, +) { + + let index = 0; // Shared among workers + + async function worker() { + while (index < objects.length) { + const currentIndex = index; + index = index + 1; + + const object = objects[currentIndex]; + const { key } = object; + + try { + await attemptWithRetries( + async() => { + await fn(object, key, currentIndex); + }, + { + onAttempt(attempt) { + if (attempt > 0) { + console.log(` Attempt ${attempt + 1} for: ${key}`); + } + }, + onRetry(attempt, err, delay) { + let statusMessage = 'unknown'; + if (err instanceof FetchError) { + statusMessage = `HTTP ${err.status}`; + } else if (err instanceof TypeError) { + statusMessage = 'transport'; + } + console.log(` ‼️ Attempt ${attempt + 1} for ${key} gave retryable error (${statusMessage}), delaying ${delay} ms`); + }, + } + ); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + console.error(` ❌ Failed: ${key} β†’ ${e.message}`); + console.error(e.stack); + } + } + } + + const workers = Array.from({ length: maxConcurrent }, () => worker()); + await Promise.all(workers); +} + +export type KVStoreItemDesc = { + write: boolean, + size: number, + key: string, + filePath: string, + metadataJson?: Record, +}; + +export function shouldRecreateChunks(chunksDir: string, numChunks: number, item: KVStoreItemDesc, chunkSize: number) { + console.log(` πŸ“„ '${item.key}' - ${item.size} bytes β†’ ${numChunks} chunks`) + if (!directoryExists(chunksDir)) { + console.log(` ・ Creating chunks for '${item.key}' - found no existing chunks`); + return true; + } + + const existingChunks = fs.readdirSync(chunksDir).length; + if (existingChunks !== numChunks) { + console.log(` ・ Recreating chunks for '${item.key}' - found ${existingChunks} existing chunk(s), expected ${numChunks}`); + return true; + } + + const finalChunkSize = item.size % chunkSize; + for (let chunk = 0; chunk < numChunks; chunk++) { + const chunkFileName = `${chunksDir}/${chunk}`; + // If the chunk does not exist + if (!fs.existsSync(chunkFileName)) { + console.log(` ・ Recreating chunks for '${item.key}' - chunk ${chunk} does not exist`); + return true; + } + + // or if the chunk isn't the expected size + const expectedFileSize = chunk === numChunks - 1 ? finalChunkSize : chunkSize; + + const existingFileSize = getFileSize(chunkFileName); + if (existingFileSize !== expectedFileSize) { + console.log(` ・ Recreating chunks for '${item.key}' - chunk ${chunk} existing file size ${existingFileSize} does not match expected size ${expectedFileSize}`); + return true; + } + } + + console.log(` ・ Existing chunks for '${item.key}' look good.`); + return false; +} + +export async function applyKVStoreEntriesChunks(kvStoreItemDescriptions: KVStoreItemDesc[], chunkSize: number) { + + const items = kvStoreItemDescriptions.splice(0); + // kvStoreItemDescriptions is now empty. + + for (const item of items) { + if (item.size <= chunkSize) { + // If file is not over the chunk size, use it as it is. + kvStoreItemDescriptions.push(item); + continue; + } + + // Check if the chunks exist on disk, and if they don't, recreate them. + + const chunksDir = `${item.filePath}_chunks`; + const numChunks = Math.ceil(item.size / chunkSize); + + if (shouldRecreateChunks(chunksDir, numChunks, item, chunkSize)) { + // Recreate chunks + fs.rmSync(chunksDir, { recursive: true, force: true }); + fs.mkdirSync(chunksDir, { recursive: true }); + + const numWrittenChunks = await new Promise((resolve, reject) => { + const chunkPaths: string[] = []; + let chunkIndex = 0; + let bytesWritten = 0; + let currentStream: fs.WriteStream | null = null; + + const inputStream = fs.createReadStream(item.filePath); + inputStream.on('error', reject); + + inputStream.on('data', (chunk) => { + let offset = 0; + + while (offset < chunk.length) { + if (!currentStream || bytesWritten >= chunkSize) { + if (currentStream != null) { + currentStream.end(); + } + const chunkFileName = `${chunksDir}/${chunkIndex}`; + currentStream = fs.createWriteStream(chunkFileName); + chunkPaths.push(chunkFileName); + chunkIndex++; + bytesWritten = 0; + } + + const bytesLeftInChunk = chunkSize - bytesWritten; + const bytesLeftInBuffer = chunk.length - offset; + const bytesToWrite = Math.min(bytesLeftInChunk, bytesLeftInBuffer); + const slice = chunk.slice(offset, offset + bytesToWrite); + + currentStream.write(slice); + bytesWritten += bytesToWrite; + offset += bytesToWrite; + } + }); + + inputStream.on('end', () => { + if (currentStream != null) { + currentStream.end(); + } + resolve(chunkPaths.length); + }); + }); + + if (numWrittenChunks !== numChunks) { + throw new Error(`numWrittenChunks (${numWrittenChunks}) does not equal numChunks (${numChunks})`); + } + } + + for (let chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) { + + const kvItem = structuredClone(item); + + // Source the chunk from the chunk directory + kvItem.filePath = `${chunksDir}/${chunkIndex}`; + + // Add chunk ID to metadata + if (kvItem.metadataJson == null) { + kvItem.metadataJson = {}; + } + kvItem.metadataJson.chunkIndex = chunkIndex; + + if (chunkIndex !== 0) { + + // For additional chunks (all but the first): + // add suffix to key + kvItem.key = item.key + '_' + chunkIndex; + // remove other metadata + delete kvItem.metadataJson.hash; + delete kvItem.metadataJson.size; + delete kvItem.metadataJson.numChunks; + + } + + kvStoreItemDescriptions.push(kvItem); + + } + } + +} diff --git a/src/cli/util/kv-store-local-server.ts b/src/cli/util/kv-store-local-server.ts new file mode 100644 index 0000000..758463b --- /dev/null +++ b/src/cli/util/kv-store-local-server.ts @@ -0,0 +1,156 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { type KVStoreItemDesc } from './kv-store-items.js'; + +type KVStoreLocalServerEntry = ({ data: string, } | { file: string }) & { metadata?:string }; +type KVStoreLocalServerData = Record; + +export async function getLocalKVStoreKeys( + storeFile: string, + prefix?: string, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + let keys = Object.keys(store); + + if (prefix != null) { + keys = keys.filter(key => key.startsWith(prefix)); + } + + return keys; + +} + +export async function getLocalKvStoreEntry( + storeFile: string, + key: string, + metadataOnly?: boolean, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + const obj = store[key]; + if (obj == null) { + return null; + } + + let response; + if (metadataOnly) { + response = new Response(null); + } else if ('data' in obj) { + response = new Response(obj.data); + } else { + const fileData = fs.readFileSync(obj.file); + response = new Response(fileData); + } + return { + metadata: obj.metadata ?? null, + generation: null, + response, + }; + +} + +export async function localKvStoreSubmitEntry( + storeFile: string, + key: string, + file: string, + metadata: string | undefined, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + store[key] = { + file, + metadata, + }; + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} + +export async function localKvStoreDeleteEntry( + storeFile: string, + key: string, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + delete store[key]; + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} + +export function writeKVStoreEntriesForLocal( + storeFile: string, + computeAppDir: string, + kvStoreItemDescriptions: KVStoreItemDesc[], +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + for (const kvStoreItemDescription of kvStoreItemDescriptions) { + store[kvStoreItemDescription.key] = { + file: path.relative(computeAppDir, kvStoreItemDescription.filePath), + metadata: kvStoreItemDescription.metadataJson != null ? JSON.stringify(kvStoreItemDescription.metadataJson) : undefined, + }; + } + + // Delete any keys that point to items that do not exist in the file system + const keysToDelete = new Set(); + for (const [key, value] of Object.entries(store)) { + if (!("file" in value) || fs.existsSync(path.resolve(computeAppDir, value.file))) { + continue; + } + keysToDelete.add(key); + } + for (const key of keysToDelete) { + delete store[key]; + } + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} diff --git a/src/cli/util/kv-store.ts b/src/cli/util/kv-store.ts index 3eb2733..cb9c0c4 100644 --- a/src/cli/util/kv-store.ts +++ b/src/cli/util/kv-store.ts @@ -1,4 +1,9 @@ -import { callFastlyApi, FastlyApiContext, FetchError } from "./fastly-api.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { callFastlyApi, type FastlyApiContext, FetchError } from './api-token.js'; type KVStoreInfo = { id: string, @@ -100,44 +105,85 @@ export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext) { return kvStoreInfos; } -export const _getKVStoreKeys = createArrayGetter()((kvStoreId: string) => `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys`); +export const _getKVStoreKeys = createArrayGetter()( + (kvStoreId: string, prefix?: string) => { + let endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys`; + if (prefix != null) { + endpoint += '?prefix=' + encodeURIComponent(prefix); + } + return endpoint; + } +); -export async function getKVStoreKeys(fastlyApiContext: FastlyApiContext, kvStoreName: string) { +export async function getKVStoreKeys( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + prefix?: string, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { return null; } - return await _getKVStoreKeys(fastlyApiContext, `Listing Keys for KV Store [${kvStoreId}] ${kvStoreName}`, kvStoreId); + return await _getKVStoreKeys( + fastlyApiContext, + `Listing Keys for KV Store [${kvStoreId}] ${kvStoreName}${prefix != null ? ` (prefix '${prefix}')` : ''}`, + kvStoreId, + prefix, + ); } -export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { +export async function getKvStoreEntryInfo(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { + + return getKvStoreEntry(fastlyApiContext, kvStoreName, key, true); + +} + +export async function getKvStoreEntry( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + key: string, + metadataOnly?: boolean, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { - return false; + return null; } const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; + let response: Response; try { - await callFastlyApi(fastlyApiContext, endpoint, `Checking existence of [${key}]`, null, { method: 'HEAD' }); + response = await callFastlyApi(fastlyApiContext, endpoint, `Checking existence of [${key}]`, null, { method: metadataOnly ? 'HEAD' : 'GET' }); } catch(err) { if (err instanceof FetchError && err.status === 404) { - return false; + return null; } throw err; } - return true; + const metadata = response.headers.get('metadata'); + const generation = response.headers.get('generation'); + return { + metadata, + generation, + response, + }; } const encoder = new TextEncoder(); -export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string) { +export async function kvStoreSubmitEntry( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + key: string, + data: ReadableStream | Uint8Array | string, + metadata: string | undefined, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -151,13 +197,14 @@ export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvSt method: 'PUT', headers: { 'content-type': 'application/octet-stream', + ...(metadata != null ? { metadata } : null) }, body, }); } -export async function kvStoreDeleteFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { +export async function kvStoreDeleteEntry(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { diff --git a/src/cli/util/node.ts b/src/cli/util/node.ts new file mode 100644 index 0000000..2d279be --- /dev/null +++ b/src/cli/util/node.ts @@ -0,0 +1,11 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export function isNodeError(err: unknown): err is NodeJS.ErrnoException { + if (err == null || typeof err !== 'object' || !('code' in err)) { + return false; + } + return true; +} diff --git a/src/cli/util/package.ts b/src/cli/util/package.ts new file mode 100644 index 0000000..6b86dfb --- /dev/null +++ b/src/cli/util/package.ts @@ -0,0 +1,63 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +export type PackageJson = { + author?: string, + name?: string, + description?: string, + dependencies?: Record, + devDependencies?: Record, +}; + +export function findComputeJsStaticPublisherVersion(packageJson: PackageJson | null) { + + // Get the current compute js static publisher version. + let computeJsStaticPublisherVersion: string | null = null; + if (packageJson != null) { + // First try current project's package.json + computeJsStaticPublisherVersion = + packageJson.dependencies?.["@fastly/compute-js-static-publish"] ?? + packageJson.devDependencies?.["@fastly/compute-js-static-publish"] ?? + null; + + // This may be a file url if during development + if (computeJsStaticPublisherVersion != null) { + + if (computeJsStaticPublisherVersion.startsWith('file:')) { + // this is a relative path from the current directory. + // we replace it with an absolute path + const relPath = computeJsStaticPublisherVersion.slice('file:'.length); + const absPath = path.resolve(relPath); + computeJsStaticPublisherVersion = 'file:' + absPath; + } + + } + } + + if (computeJsStaticPublisherVersion == null) { + // Also try package.json of the package that contains the currently running program + // This is used when the program doesn't actually install the package (running via npx). + const computeJsStaticPublishPackageJsonPath = path.resolve(import.meta.dirname, '../../../package.json'); + const computeJsStaticPublishPackageJsonText = fs.readFileSync(computeJsStaticPublishPackageJsonPath, 'utf-8'); + const computeJsStaticPublishPackageJson = JSON.parse(computeJsStaticPublishPackageJsonText); + computeJsStaticPublisherVersion = computeJsStaticPublishPackageJson?.version; + } + + if (computeJsStaticPublisherVersion == null) { + // Unexpected, but if it's still null then we go to a literal + computeJsStaticPublisherVersion = '7.0.0'; + } + + if (!computeJsStaticPublisherVersion.startsWith('^') && + !computeJsStaticPublisherVersion.startsWith('file:') + ) { + computeJsStaticPublisherVersion = '^' + computeJsStaticPublisherVersion; + } + + return computeJsStaticPublisherVersion; +} diff --git a/src/cli/util/publish-id.ts b/src/cli/util/publish-id.ts deleted file mode 100644 index b83233e..0000000 --- a/src/cli/util/publish-id.ts +++ /dev/null @@ -1,34 +0,0 @@ -import fs from "fs"; -import { createStringId } from "./id.js"; - -export function generateOrLoadPublishId() { - - let created = false; - - const filename = './.publish-id'; - let contents: string | null; - try { - contents = fs.readFileSync(filename, 'utf-8'); - } catch { - contents = null; - } - - let publishId: string | null = null; - if (contents != null) { - publishId = contents - .split('\n') - .find(line => line.length > 0 && !line.startsWith('#')) ?? null; - } - - if (publishId == null) { - - publishId = createStringId(); - created = true; - const fileContents = `# Generated by @fastly/compute-js-static-publish.\n${publishId}\n`; - fs.writeFileSync(filename, fileContents, 'utf-8'); - - } - - return { publishId, created }; - -} diff --git a/src/cli/util/retryable.ts b/src/cli/util/retryable.ts index fe4e543..90a2cbc 100644 --- a/src/cli/util/retryable.ts +++ b/src/cli/util/retryable.ts @@ -1,3 +1,8 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + let _globalBackoffUntil = 0; const retryableSymbol = Symbol(); diff --git a/src/cli/util/variants.ts b/src/cli/util/variants.ts new file mode 100644 index 0000000..b114102 --- /dev/null +++ b/src/cli/util/variants.ts @@ -0,0 +1,41 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import { type ContentCompressionTypes } from '../../models/compression/index.js'; +import { algs } from '../compression/index.js'; +import { rootRelative } from './files.js'; + +export type Variants = 'original' | ContentCompressionTypes; + +export async function ensureVariantFileExists( + variantFilePath: string, + variant: Variants, + file: string, +) { + + // Compress/prepare the asset if it doesn't already exist + // We disregard chunked copies of the file and only check for the existence + // of the main copy of the file at this point. + if (!fs.existsSync(variantFilePath)) { + if (variant === 'original') { + + fs.cpSync(file, variantFilePath); + console.log(` πŸ“„β†’πŸ“„ Copied file '${rootRelative(file)}' to '${rootRelative(variantFilePath)}'.`); + + } else { + + const compressTo = algs[variant]; + await compressTo(file, variantFilePath); + console.log(` πŸ“„β†’πŸ—„οΈ Compressed file '${rootRelative(file)}' to '${rootRelative(variantFilePath)}' [${variant}].`); + + } + + // However, if we did just create the file, + // we delete any chunked copies that may exist as they can be out of date. + // (They will be recreated in a later step) + fs.rmSync(`${variantFilePath}_chunks`, { recursive: true, force: true }); + } +} diff --git a/src/compute-js.ts b/src/compute-js.ts deleted file mode 100644 index b07fa74..0000000 --- a/src/compute-js.ts +++ /dev/null @@ -1,2 +0,0 @@ -import './server/assets/content-asset-wasm-inline.js'; -import './server/assets/content-asset-kv-store.js'; diff --git a/src/constants/compression.ts b/src/constants/compression.ts deleted file mode 100644 index d959bc4..0000000 --- a/src/constants/compression.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const compressionTypes = [ - 'br', - 'gzip', -] as const; - -export type ContentCompressionTypes = typeof compressionTypes[number]; diff --git a/src/index.ts b/src/index.ts index 9b7352c..ad8dfb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,13 @@ -/// +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ -export * from './types/index.js'; -export * from './util/index.js'; - -export { ContentAssets } from './server/assets/content-assets.js'; -export { ModuleAssets } from './server/assets/module-assets.js'; -export { PublisherServer } from './server/publisher-server.js'; +export { type StaticPublishRc } from './models/config/static-publish-rc.js'; +export { type PublishContentConfig } from './models/config/publish-content-config.js'; +export * as collectionSelector from './server/collection-selector/index.js'; +export { + CookieCollectionSelector, + type CookieCollectionSelectorOpts, +} from './server/collection-selector/from-cookie.js'; +export { PublisherServer } from './server/publisher-server/index.js'; diff --git a/src/models/assets/kvstore-assets.ts b/src/models/assets/kvstore-assets.ts new file mode 100644 index 0000000..194f612 --- /dev/null +++ b/src/models/assets/kvstore-assets.ts @@ -0,0 +1,50 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type ContentCompressionTypes, isCompressionType } from '../compression/index.js'; + +export type KVAssetEntry = { + key: string, + size: number, + contentType: string, + lastModifiedTime: number, + variants: ContentCompressionTypes[], +}; + +export type KVAssetEntryMap = Record; + +export type KVAssetVariantMetadata = { + contentEncoding?: ContentCompressionTypes, + size: number, + hash: string, + numChunks?: number, +}; + +export function isKVAssetVariantMetadata(obj: unknown): obj is KVAssetVariantMetadata { + if (obj == null || typeof obj !== 'object') { + return false; + } + if ('contentEncoding' in obj) { + if ( + typeof obj.contentEncoding !== 'string' || + !isCompressionType(obj.contentEncoding) + ) { + return false; + } + } + if (!('size' in obj) || typeof obj.size !== 'number') { + return false; + } + if (!('hash' in obj) || typeof obj.hash !== 'string') { + return false; + } + if ('numChunks' in obj) { + if (typeof obj.numChunks !== 'number') { + return false; + } + } + + return true; +} diff --git a/src/models/compression/index.ts b/src/models/compression/index.ts new file mode 100644 index 0000000..0eb8d67 --- /dev/null +++ b/src/models/compression/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export const compressionTypes = [ + 'br', + 'gzip', +] as const; + +export type ContentCompressionTypes = typeof compressionTypes[number]; + +export function isCompressionType(value: string): value is ContentCompressionTypes { + return (compressionTypes as Readonly).includes(value); +} diff --git a/src/models/config/publish-content-config.ts b/src/models/config/publish-content-config.ts new file mode 100644 index 0000000..2e069e2 --- /dev/null +++ b/src/models/config/publish-content-config.ts @@ -0,0 +1,86 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { + type PublisherServerConfig, + type PublisherServerConfigNormalized, +} from './publisher-server-config.js'; +import { + type ContentCompressionTypes, +} from '../compression/index.js'; + +export type ContentTypeTest = (name: string) => boolean; + +// Content Type definition +export type ContentTypeDef = { + // A test on the asset key to perform on this content type. + test: RegExp | ContentTypeTest, + + // The Content-Type header value to provide for this content type. + contentType: string, + + // Whether this content type represents a text value encoded in utf-8. + // If so, conveniences can be provided. + text?: boolean, + + // Binary formats are usually not candidates for compression, but certain + // types can benefit from it. + precompressAsset?: boolean, +}; + +export type ContentTypeTestResult = { + contentType: string, + text: boolean, + precompressAsset: boolean, +}; + +export interface ExcludeDirTest { + test(name: string): boolean; +} + +export type KVStoreAssetInclusionTest = (assetKey: string, contentType?: string) => boolean; + +export type PublishContentConfig = { + // Set to a directory that acts as the root of all files that will be included in this publish. + rootDir: string, + + // An array of values used to exclude files and directories (as well as files within those directories) from being + // included in this publish. Each entry in the array can be a string or a RegExp and will be tested against the relative + // path from 'rootDir' of each file or directory. + // Defaults to [ './node_modules' ]. Set to an empty array or specifically to null to include all files. + excludeDirs?: (string | ExcludeDirTest)[] | string | ExcludeDirTest | null, + + // If true, then files whose names begin with a dot, as well as files in directories whose names begin with a .dot, + // are excluded from this publish. Defaults to true. + excludeDotFiles?: boolean, + + // If true, include .well-known even if excludeDotFiles is true. + // Defaults to true. + includeWellKnown?: boolean, + + // A test to run on each asset key to determine whether and how to include the file. + kvStoreAssetInclusionTest?: KVStoreAssetInclusionTest | null, + + // Pre-generate content in these formats as well and serve them in tandem with the + // allowedEncodings setting in the server settings. Default value is [ 'br' | 'gzip' ]. + contentCompression?: ('br' | 'gzip')[], + + // Additional / override content types. + contentTypes?: ContentTypeDef[], + + // Server settings + server?: PublisherServerConfig | null, +}; + +export type PublishContentConfigNormalized = { + rootDir: string, + excludeDirs: ExcludeDirTest[], + excludeDotFiles: boolean, + includeWellKnown: boolean, + kvStoreAssetInclusionTest: KVStoreAssetInclusionTest | null, + contentCompression: ContentCompressionTypes[], + contentTypes: ContentTypeDef[], + server: PublisherServerConfigNormalized | null, +}; diff --git a/src/models/config/publisher-server-config.ts b/src/models/config/publisher-server-config.ts new file mode 100644 index 0000000..8be9d4a --- /dev/null +++ b/src/models/config/publisher-server-config.ts @@ -0,0 +1,52 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type ContentCompressionTypes } from '../compression/index.js'; + +// Publisher Server configuration +export type PublisherServerConfig = { + // Prefix to apply to web requests. Effectively, a directory within rootDir that is used + // by the web server to determine the asset to respond with. Defaults to the empty string. + publicDirPrefix?: string, + + // A test to apply to item names to decide whether to serve them as "static" files, in other + // words, with a long TTL. These are used for files that are not expected to change. + // They can be provided as a string or array of strings. + // Items that contain asterisks, are interpreted as glob patterns. + // Items that end with a trailing slash are interpreted as directory names, + // Items that don't contain asterisks and that do not end in slash are checked for exact match. + staticItems?: string[] | string | false | null, + + // Allowed encodings. If the request contains an Accept-Encoding header, they are checked in the order listed + // in the header for the values listed here. The compression algorithm represented by the first match is applied. + // Default value is [ 'br', 'gzip' ]. + allowedEncodings?: ContentCompressionTypes[], + + // Set to the asset key of a content item to serve this when a GET request comes in for an unknown asset, and + // the Accept header includes text/html. + spaFile?: string | false | null, + + // Set to the asset key of a content item to serve this when a request comes in for an unknown asset, and + // the Accept header includes text/html. + notFoundPageFile?: string | false | null, + + // When a file is not found, and it doesn't end in a slash, then try auto-ext: try to serve a file with the same name + // postfixed with the specified strings, tested in the order listed. + autoExt?: string[] | string | false | null, + + // When a file is not found, then try auto-index: treat it as a directory, then try to serve a file that has the + // specified strings, tested in the order listed. + autoIndex?: string[] | string | false | null, +}; + +export type PublisherServerConfigNormalized = { + publicDirPrefix: string, + staticItems: string[], + allowedEncodings: ContentCompressionTypes[], + spaFile: string | null, + notFoundPageFile: string | null, + autoExt: string[], + autoIndex: string[], +}; diff --git a/src/models/config/static-publish-rc.ts b/src/models/config/static-publish-rc.ts new file mode 100644 index 0000000..2cfa001 --- /dev/null +++ b/src/models/config/static-publish-rc.ts @@ -0,0 +1,21 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type StaticPublishRc = { + // Name of KV Store to use. + kvStoreName: string, + + // Publish ID. This is a unique string "prefix" that identifies your assets in the + // KV Store. This is generated by init-app and is usually not to be changed. + publishId: string; + + // Default collection name. + defaultCollectionName: string, + + // A directory that holds the working files built by compute-js-static-publish, + // relative to the Compute application. + // These files should not be committed to source control. + staticPublisherWorkingDir: string, +}; diff --git a/src/models/server/index.ts b/src/models/server/index.ts new file mode 100644 index 0000000..db6add5 --- /dev/null +++ b/src/models/server/index.ts @@ -0,0 +1,4 @@ +export type IndexMetadata = { + publishedTime?: number, + expirationTime?: number, +}; diff --git a/src/models/time/index.ts b/src/models/time/index.ts new file mode 100644 index 0000000..d3c9a76 --- /dev/null +++ b/src/models/time/index.ts @@ -0,0 +1,71 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +const DURATION_MS: Record = { + w: 604_800_000, + d: 86_400_000, + h: 3_600_000, + m: 60_000, + s: 1_000, + ms: 1, +}; + +// Parses something like "1d2h30m" or "5m15s" or "150ms" +function parseDuration(durationStr: string) { + const regex = /(\d+(?:\.\d+)?)(ms|[dhms])/g; + let totalMs = 0; + let matchedLength = 0; + + for (const match of durationStr.matchAll(regex)) { + const [res, valueStr, unit] = match; + const value = parseFloat(valueStr); + totalMs += value * DURATION_MS[unit]; + matchedLength += res.length; + } + + if (matchedLength !== durationStr.length) { + throw new Error(`Invalid duration format: '${durationStr}'`); + } + + return totalMs; +} + +export type CalcExpirationTimeArg = { + expiresIn?: string, + expiresAt?: string, + expiresNever?: boolean, +}; + +export function calcExpirationTime({ expiresIn, expiresAt, expiresNever }: CalcExpirationTimeArg) { + if ([expiresIn, expiresAt, expiresNever].filter(x => x !== undefined).length > 1) { + throw new Error('Only one of expiresIn or expiresAt or expiresNever may be provided'); + } + + if (expiresNever) { + return null; + } + + if (expiresIn) { + const deltaMs = parseDuration(expiresIn); + console.log(` ⏳ Expiration duration '${expiresIn}' = ${deltaMs} ms`); + return Math.floor((Date.now() + deltaMs) / 1000); // return UNIX timestamp + } + + if (expiresAt) { + const time = Date.parse(expiresAt); + if (isNaN(time)) { + throw new Error(`Invalid expiresAt value: '${expiresAt}'`); + } + console.log(` ⏰️ Expiration timestamp '${expiresAt}' = '${new Date(time)}'`); + console.log(` β†’ ${new Date(time).toISOString()}`); + return Math.floor(time / 1000); + } + + return undefined; +} + +export function isExpired(unixTime: number) { + return unixTime < Math.floor(Date.now() / 1000); +} diff --git a/src/server/assets/asset-manager.ts b/src/server/assets/asset-manager.ts deleted file mode 100644 index 35b676c..0000000 --- a/src/server/assets/asset-manager.ts +++ /dev/null @@ -1,21 +0,0 @@ -export abstract class AssetManager { - - protected readonly assets: Record; - - protected constructor() { - this.assets = {}; - } - - protected initAsset(assetKey: string, asset: TAsset) { - this.assets[assetKey] = asset; - } - - getAsset(assetKey: string): TAsset | null { - return this.assets[assetKey] ?? null; - } - - getAssetKeys(): string[] { - return Object.keys(this.assets); - } - -} diff --git a/src/server/assets/content-asset-kv-store.ts b/src/server/assets/content-asset-kv-store.ts deleted file mode 100644 index 6ebb5b7..0000000 --- a/src/server/assets/content-asset-kv-store.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { KVStore } from "fastly:kv-store"; -import { - ContentAsset, - ContentAssetMetadataMapEntry, - StoreEntry, -} from "../../types/index.js"; -import { - CompressedFileInfos, - ContentAssetMetadataMapEntryKVStore, -} from "../../types/content-assets.js"; -import { ContentCompressionTypes } from "../../constants/compression.js"; -import { ContentAssets, findMatchingSourceAndInfo, SourceAndInfo } from "./content-assets.js"; - -export class ContentKVStoreAsset implements ContentAsset { - readonly type = 'kv-store'; - - private readonly metadata: ContentAssetMetadataMapEntryKVStore; - - private readonly kvStoreName: string; - - private readonly sourceAndInfo: SourceAndInfo; - private readonly compressedSourcesAndInfo: CompressedFileInfos>; - - constructor(metadata: ContentAssetMetadataMapEntryKVStore, kvStoreName: string) { - this.metadata = metadata; - this.kvStoreName = kvStoreName; - - this.sourceAndInfo = { - source: metadata.fileInfo.kvStoreKey, - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }; - - this.compressedSourcesAndInfo = Object.entries(metadata.compressedFileInfos) - .reduce>>((obj, [key, value]) => { - obj[key as ContentCompressionTypes] = { - source: value.kvStoreKey, - hash: value.hash, - size: value.size, - }; - return obj; - }, {}); - } - - get isLocal() { - return false; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { - - const { sourceAndInfo, contentEncoding } = findMatchingSourceAndInfo(acceptEncodingsGroups, this.sourceAndInfo, encoding => this.compressedSourcesAndInfo[encoding]); - - const kvStore = new KVStore(this.kvStoreName); - let retries = 3; - while (retries > 0) { - const storeEntry = await kvStore.get(sourceAndInfo.source); - if (storeEntry != null) { - const { hash, size } = sourceAndInfo; - return Object.assign(storeEntry, { contentEncoding, hash, size }); - } - - // Note the null does NOT mean 404. The fact that we are here means - // metadata exists for this path, and we're just trying to get the data from - // the KV Store. - - // So if we're here then the data is either not available yet (in which case - // we can wait just a bit and try again), or the data was deleted. - retries--; - - // We're going to wait 250ms/500ms/750ms and try again. - const delay = (3-retries) * 250; - await new Promise(resolve => setTimeout(resolve, delay)); - } - throw new Error("Asset could not be retrieved from the KV Store."); - } - - getBytes(): Uint8Array { - throw new Error("Can't getBytes() for KV Store asset"); - } - - getText(): string { - throw new Error("Can't getText() for KV Store asset"); - } - - getJson(): T { - throw new Error("Can't getJson() for KV Store asset"); - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } - -} - -ContentAssets.registerAssetBuilder('kv-store', (metadata, context) => { - const { kvStoreName } = context; - - if (kvStoreName == null) { - throw new Error("Unexpected! KV Store name should be specified!!"); - } - - return new ContentKVStoreAsset(metadata, kvStoreName); -}); diff --git a/src/server/assets/content-asset-wasm-inline.ts b/src/server/assets/content-asset-wasm-inline.ts deleted file mode 100644 index 1a05fb3..0000000 --- a/src/server/assets/content-asset-wasm-inline.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { includeBytes } from "fastly:experimental"; -import { - ContentAsset, - ContentAssetMetadataMapEntry, - StoreEntry, -} from "../../types/index.js"; -import { - CompressedFileInfos, - ContentAssetMetadataMapEntryWasmInline, -} from "../../types/content-assets.js"; -import { ContentCompressionTypes } from "../../constants/compression.js"; -import { InlineStoreEntry } from "../kv-store/inline-store-entry.js"; -import { ContentAssets, findMatchingSourceAndInfo, SourceAndInfo } from "./content-assets.js"; - -const decoder = new TextDecoder(); - -export class ContentWasmInlineAsset implements ContentAsset { - readonly type = 'wasm-inline'; - - private readonly metadata: ContentAssetMetadataMapEntryWasmInline; - - private readonly sourceAndInfo: SourceAndInfo; - private readonly compressedSourcesAndInfo: CompressedFileInfos>; - - constructor(metadata: ContentAssetMetadataMapEntryWasmInline) { - this.metadata = metadata; - - this.sourceAndInfo = { - source: includeBytes(metadata.fileInfo.staticFilePath), - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }; - - this.compressedSourcesAndInfo = Object.entries(metadata.compressedFileInfos) - .reduce>>((obj, [key, value]) => { - obj[key as ContentCompressionTypes] = { - source: includeBytes(value.staticFilePath), - hash: value.hash, - size: value.size, - }; - return obj; - }, {}); - } - - get isLocal() { - return true; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { - const { sourceAndInfo, contentEncoding } = findMatchingSourceAndInfo(acceptEncodingsGroups, this.sourceAndInfo, encoding => this.compressedSourcesAndInfo[encoding]); - const { source, hash, size } = sourceAndInfo; - return new InlineStoreEntry(source, contentEncoding, hash, size); - } - - getBytes(): Uint8Array { - return this.sourceAndInfo.source; - } - - getText(): string { - if (!this.metadata.text) { - throw new Error("Can't getText() for non-text content"); - } - return decoder.decode(this.sourceAndInfo.source); - } - - getJson(): T { - const text = this.getText(); - return JSON.parse(text) as T; - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } -} - -ContentAssets.registerAssetBuilder('wasm-inline', metadata => new ContentWasmInlineAsset(metadata)); diff --git a/src/server/assets/content-assets.ts b/src/server/assets/content-assets.ts deleted file mode 100644 index 18b4548..0000000 --- a/src/server/assets/content-assets.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { AssetManager } from "./asset-manager.js"; -import { InlineStoreEntry } from "../kv-store/inline-store-entry.js"; - -import type { - ContentAsset, - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, - ContentAssetMetadataMapEntryBytes, - ContentAssetMetadataMapEntryString, -} from "../../types/content-assets.js"; -import type { StoreEntry } from "../../types/compute.js"; -import type { ContentCompressionTypes } from "../../constants/compression.js"; - -const decoder = new TextDecoder(); - -export type SourceAndInfo = { - source: TSource, - hash: string, - size: number, -}; - -type SourceAndInfoForEncodingFn = (contentEncoding: ContentCompressionTypes) => SourceAndInfo | undefined; -export function findMatchingSourceAndInfo(acceptEncodingsGroups: ContentCompressionTypes[][], defaultSourceAndInfo: SourceAndInfo, sourceAndInfoForEncodingFn: SourceAndInfoForEncodingFn) { - - let sourceAndInfo: SourceAndInfo | undefined; - let contentEncoding: ContentCompressionTypes | null = null; - if (acceptEncodingsGroups != null) { - for(const encodingGroup of acceptEncodingsGroups) { - const sourceAndInfosForEncodingsGroup = encodingGroup - .map(encoding => ({ encoding, sourceAndInfo: sourceAndInfoForEncodingFn(encoding) })) - .filter(x => x.sourceAndInfo != null) as {encoding: ContentCompressionTypes, sourceAndInfo: SourceAndInfo}[]; - - if (sourceAndInfosForEncodingsGroup.length === 0) { - // If no encoding in this group is available then move to next group - continue; - } - - // Sort the items, putting the smallest size first - sourceAndInfosForEncodingsGroup - .sort((a, b) => a.sourceAndInfo.size - b.sourceAndInfo.size); - - // The first item is the one we want. - sourceAndInfo = sourceAndInfosForEncodingsGroup[0].sourceAndInfo; - contentEncoding = sourceAndInfosForEncodingsGroup[0].encoding; - break; - } - } - - sourceAndInfo ??= defaultSourceAndInfo; - - return { sourceAndInfo, contentEncoding }; -} - -abstract class ContentRuntimeAsset { - protected readonly metadata: TMetadataMapEntry; - protected sourceAndInfo: SourceAndInfo; - protected constructor(metadata: TMetadataMapEntry, sourceAndInfo: SourceAndInfo) { - this.metadata = metadata; - this.sourceAndInfo = sourceAndInfo; - } - - get isLocal() { - return true; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(): Promise { - const { source, hash, size } = this.sourceAndInfo; - return new InlineStoreEntry(source, null, hash, size); - } - - getBytes(): Uint8Array { - return this.sourceAndInfo.source; - } - - getText(): string { - if (!this.metadata.text) { - throw new Error("Can't getText() for non-text content"); - } - return decoder.decode(this.sourceAndInfo.source); - } - - getJson(): T { - const text = this.getText(); - return JSON.parse(text) as T; - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } -} - -export class ContentBytesAsset - extends ContentRuntimeAsset - implements ContentAsset { - readonly type = 'bytes'; - - constructor(metadata: ContentAssetMetadataMapEntryBytes) { - super(metadata, { - source: metadata.fileInfo.bytes, - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }); - } -} - -export class ContentStringAsset - extends ContentRuntimeAsset - implements ContentAsset { - readonly type = 'string'; - - static encoder = new TextEncoder(); - constructor(metadata: ContentAssetMetadataMapEntryString) { - super(metadata, { - source: ContentStringAsset.encoder.encode(metadata.fileInfo.content), - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }); - } -} - -export type AssetBuilder = (metadata: ContentAssetMetadataMapEntry & { type: TType }, assetBuilderContext: AssetBuilderContext) => ContentAsset; -type AssetBuilderContext = any; - -export class ContentAssets extends AssetManager { - - static builders: Record> = {}; - - constructor(contentAssetMetadataMap: ContentAssetMetadataMap, assetBuilderContext: AssetBuilderContext = {}) { - super(); - - for (const [assetKey, metadata] of Object.entries(contentAssetMetadataMap)) { - - if (!(metadata.type in ContentAssets.builders)) { - throw new Error(`Unknown content asset type '${metadata.type}'`); - } - - const asset = ContentAssets.builders[metadata.type](metadata, assetBuilderContext); - this.initAsset(assetKey, asset); - } - } - - static registerAssetBuilder(type: TType, builder: AssetBuilder) { - this.builders[type] = builder; - } -} - -ContentAssets.registerAssetBuilder('bytes', metadata => new ContentBytesAsset(metadata)); -ContentAssets.registerAssetBuilder('string', metadata => new ContentStringAsset(metadata)); diff --git a/src/server/assets/module-assets.ts b/src/server/assets/module-assets.ts deleted file mode 100644 index 1baf699..0000000 --- a/src/server/assets/module-assets.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { AssetManager } from "./asset-manager.js"; - -import type { ModuleAsset, ModuleAssetMap } from "../../types/module-assets.js"; - -export class ModuleAssetDynamic implements ModuleAsset { - readonly assetKey: string; - readonly isStaticImport: boolean = false; - - private readonly loadModule: () => Promise; - - constructor(assetKey: string, loadModule: () => Promise) { - this.assetKey = assetKey; - this.loadModule = loadModule; - } - - private _modulePromise: Promise | undefined; - getModule(): Promise { - if (this._modulePromise === undefined) { - this._modulePromise = this.loadModule(); - } - return this._modulePromise; - } - - getStaticModule(): any { - return null; - } - -} - -export class ModuleAssetStatic implements ModuleAsset { - readonly assetKey: string; - readonly isStaticImport: boolean = true; - - private readonly module: any; - constructor(assetKey: string, module: any) { - this.assetKey = assetKey; - this.module = module; - } - - getModule(): Promise { - return Promise.resolve(this.module); - } - - getStaticModule(): any { - return this.module; - } - -} - -export class ModuleAssets extends AssetManager { - - constructor(moduleAssetMap: ModuleAssetMap) { - super(); - - for (const [assetKey, moduleEntry] of Object.entries(moduleAssetMap)) { - - let asset: ModuleAsset; - if (moduleEntry.isStaticImport) { - - asset = new ModuleAssetStatic(assetKey, moduleEntry.module); - - } else { - - asset = new ModuleAssetDynamic(assetKey, moduleEntry.loadModule); - - } - - this.initAsset(assetKey, asset); - } - } -} diff --git a/src/server/collection-selector/from-config-store.ts b/src/server/collection-selector/from-config-store.ts new file mode 100644 index 0000000..a87b11a --- /dev/null +++ b/src/server/collection-selector/from-config-store.ts @@ -0,0 +1,18 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { ConfigStore } from 'fastly:config-store'; + +export function fromConfigStore(configStoreName: string, key: string) { + + try { + const configStore = new ConfigStore(configStoreName); + return configStore.get(key); + } catch { + // If there is no config store + return null; + } + +} diff --git a/src/server/collection-selector/from-cookie.ts b/src/server/collection-selector/from-cookie.ts new file mode 100644 index 0000000..34873b0 --- /dev/null +++ b/src/server/collection-selector/from-cookie.ts @@ -0,0 +1,136 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { getCookieValue } from '../util/cookies.js'; +import { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; + +export type CookieCollectionSelectorOpts = { + cookieName: string, + activatePath: string, + resetPath: string, + cookieHttpOnly: boolean, + cookieMaxAge: number | undefined, + cookiePath: string, +}; + +export class CookieCollectionSelector implements RequestCollectionNameSelector { + public constructor(); + public constructor(cookieName: string); + public constructor(opts: Partial); + public constructor(optsOrCookieName?: string | Partial) { + let opts = optsOrCookieName; + if (opts === undefined) { + opts = {}; + } else if (typeof opts === 'string') { + opts = { + cookieName: opts, + }; + } + + this.opts = { + cookieName: opts.cookieName ?? 'publisher-collection', + activatePath: opts.activatePath ?? '/activate', + resetPath: opts.resetPath ?? '/reset', + cookieHttpOnly: opts.cookieHttpOnly ?? true, + cookieMaxAge: opts.cookieMaxAge, + cookiePath: opts.cookiePath ?? '/', + }; + } + + public opts: CookieCollectionSelectorOpts; + + public getCollectionName(request: Request): CollectionNameSelectorResult { + return getCookieValue(request, this.opts.cookieName); + } + + public applySelector(request: Request) { + + const url = new URL(request.url); + + if (url.pathname === this.opts.activatePath) { + const collectionName = url.searchParams.get('collection'); + if (collectionName == null) { + return new Response(`Missing required 'collection' query parameter.`, { status: 400 }); + } + const redirectTo = url.searchParams.get('redirectTo') ?? '/'; + + const cookieSegments = [ + `${this.opts.cookieName}=${collectionName}`, + ]; + cookieSegments.push(`Path=${this.opts.cookiePath}`); + if (this.opts.cookieMaxAge != null) { + cookieSegments.push(`Max-Age=${this.opts.cookieMaxAge}`); + } + if (this.opts.cookieHttpOnly) { + cookieSegments.push('HttpOnly'); + } + if (url.protocol === 'https:') { + cookieSegments.push('Secure'); + } + cookieSegments.push(`SameSite=Lax`); + + const headers = new Headers(); + headers.append("Set-Cookie", cookieSegments.join(';')); + headers.set("Location", redirectTo); + + return new Response(null, { + status: 302, + headers, + }); + } + + if (url.pathname === this.opts.resetPath) { + const redirectTo = url.searchParams.get('redirectTo') ?? '/'; + + const cookieSegments = [ + `${this.opts.cookieName}=`, + ]; + cookieSegments.push(`Path=${this.opts.cookiePath}`); + cookieSegments.push(`Expires=Thu, 01 Jan 1970 00:00:00 GMT`); + if (this.opts.cookieHttpOnly) { + cookieSegments.push('HttpOnly'); + } + if (url.protocol === 'https:') { + cookieSegments.push('Secure'); + } + + const headers = new Headers(); + headers.append("Set-Cookie", cookieSegments.join(';')); + headers.set("Location", redirectTo); + + return new Response(null, { + status: 302, + headers, + }); + } + + return null; + } +} + +export interface FromCookieResult { + collectionName: CollectionNameSelectorResult, + redirectResponse: Response | null, +} + +export function fromCookie(request: Request, optsOrCookieName?: string | CookieCollectionSelectorOpts): FromCookieResult { + + let selector; + if (optsOrCookieName === undefined) { + selector = new CookieCollectionSelector(); + } else if (typeof optsOrCookieName === 'string') { + selector = new CookieCollectionSelector(optsOrCookieName); + } else { + selector = new CookieCollectionSelector(optsOrCookieName); + } + + return { + collectionName: selector.getCollectionName(request), + redirectResponse: selector.applySelector(request), + }; +} diff --git a/src/server/collection-selector/from-request.ts b/src/server/collection-selector/from-request.ts new file mode 100644 index 0000000..5b13454 --- /dev/null +++ b/src/server/collection-selector/from-request.ts @@ -0,0 +1,47 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; + +export type RequestToCollectionNameFunc = (request: Request) => CollectionNameSelectorResult; +type UrlToCollectionNameFunc = (url: URL) => string | null; + + +export class RequestCollectionNameFuncSelector implements RequestCollectionNameSelector { + public constructor( + func: RequestToCollectionNameFunc + ) { + this.func = func; + } + + public func: RequestToCollectionNameFunc; + + public getCollectionName(request: Request) { + return this.func(request); + } +} + +export function fromRequest(request: Request, func: RequestToCollectionNameFunc): CollectionNameSelectorResult { + + const selector = new RequestCollectionNameFuncSelector(func); + return selector.getCollectionName(request); + +} + +export function fromRequestUrl(request: Request, func: UrlToCollectionNameFunc): CollectionNameSelectorResult { + + return fromRequest(request, (req) => func(new URL(req.url))); + +} + +export function fromHostDomain(request: Request, hostRegex: RegExp): CollectionNameSelectorResult { + return fromRequestUrl( + request, + (url) => url.host.match(hostRegex)?.[1] ?? null, + ); +} diff --git a/src/server/collection-selector/index.ts b/src/server/collection-selector/index.ts new file mode 100644 index 0000000..444eba4 --- /dev/null +++ b/src/server/collection-selector/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; +export { + RequestCollectionNameFuncSelector, + fromRequest, + fromRequestUrl, + fromHostDomain, +} from './from-request.js'; +export { + CookieCollectionSelector, + fromCookie, + type CookieCollectionSelectorOpts, + type FromCookieResult, +} from './from-cookie.js'; +export { + fromConfigStore, +} from './from-config-store.js'; diff --git a/src/server/collection-selector/request-collection-selector.ts b/src/server/collection-selector/request-collection-selector.ts new file mode 100644 index 0000000..b19ab70 --- /dev/null +++ b/src/server/collection-selector/request-collection-selector.ts @@ -0,0 +1,10 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type CollectionNameSelectorResult = string | null; + +export interface RequestCollectionNameSelector { + getCollectionName(request: Request): CollectionNameSelectorResult; +} diff --git a/src/server/kv-store/inline-store-entry.ts b/src/server/kv-store/inline-store-entry.ts deleted file mode 100644 index f406f58..0000000 --- a/src/server/kv-store/inline-store-entry.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { StoreEntry } from "../../types/compute.js"; - -// A class designed to give a Body-style interface to Uint8Array. -// The intended use case is arrays brought in from includeBytes(). - -export class InlineStoreEntry implements StoreEntry { - _consumed: boolean; - - _body: ReadableStreamForBytes; - - _contentEncoding: string | null; - - _hash: string; - - _size: number; - - constructor(array: Uint8Array, contentEncoding: string | null, hash: string, size: number) { - this._body = new ReadableStreamForBytes(array); - this._consumed = false; - this._contentEncoding = contentEncoding; - this._hash = hash; - this._size = size; - } - - get body(): ReadableStream { - return this._body; - } - - get bodyUsed(): boolean { - return this._consumed; - } - - async arrayBuffer(): Promise { - if (this._consumed) { - throw new Error('Body has already been consumed'); - } - if (this._body.locked) { - throw new Error('The ReadableStream body is already locked and can\'t be consumed'); - } - if (this._body._disturbed) { - throw new Error('Body object should not be disturbed or locked'); - } - this._consumed = true; - - let result = new Uint8Array(0); - const reader = this._body.getReader(); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - const newResult = new Uint8Array(result.length + value.length); - newResult.set(result); - newResult.set(value, result.length); - result = newResult; - } - - return result; - } finally { - reader.releaseLock(); - } - } - - static decoder = new TextDecoder(); - async text(): Promise { - const data = await this.arrayBuffer(); - return InlineStoreEntry.decoder.decode(data); - } - - async json(): Promise { - const text = await this.text(); - return JSON.parse(text); - } - - get contentEncoding() { - return this._contentEncoding; - } - - get hash() { - return this._hash; - } - - get size() { - return this._size; - } -} - -class ReadableStreamForBytes extends ReadableStream { - - // Closest we can get to the "disturbed" flag - _disturbed = false; - - constructor(array: Uint8Array) { - super({ - async start(controller) { - controller.enqueue(array); - controller.close(); - }, - }) - } - - override getReader(): ReadableStreamDefaultReader { - - const reader = super.getReader(); - - const stream = this; - - // Monkey-patch read - const _read = reader.read; - reader.read = async () => { - const result = await _read.call(reader); - if (result.done) { - // NOTE: C@E Request body does not seem to get marked as "disturbed" until - // end of stream is reached either... - stream._disturbed = true; - } - return result; - }; - - // Monkey-patch cancel - const _cancel = reader.cancel; - reader.cancel = async (reason?: any) => { - await _cancel.call(reader, reason); - stream._disturbed = true; - }; - - return reader; - - } - -} diff --git a/src/server/publisher-server.ts b/src/server/publisher-server.ts deleted file mode 100644 index 1c19444..0000000 --- a/src/server/publisher-server.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { ContentAssets } from "./assets/content-assets.js"; -import { checkIfModifiedSince, getIfModifiedSinceHeader } from "./serve-preconditions/if-modified-since.js"; -import { checkIfNoneMatch, getIfNoneMatchHeader } from "./serve-preconditions/if-none-match.js"; -import { buildHeadersSubset } from "./util/http.js"; - -import type { PublisherServerConfigNormalized } from "../types/config-normalized.js"; -import type { ContentAsset } from "../types/content-assets.js"; -import type { ContentCompressionTypes } from "../constants/compression.js"; - -// https://httpwg.org/specs/rfc9110.html#rfc.section.15.4.5 -// The server generating a 304 response MUST generate any of the following header fields that would have been sent in -// a 200 (OK) response to the same request: -// * Content-Location, Date, ETag, and Vary -// * Cache-Control and Expires -const headersToPreserveForUnmodified = ['Content-Location', 'ETag', 'Vary', 'Cache-Control', 'Expires'] as const; - -function requestAcceptsTextHtml(req: Request) { - const accept = (req.headers.get('Accept') ?? '') - .split(',') - .map(x => x.split(';')[0]); - if(!accept.includes('text/html') && !accept.includes('*/*') && accept.includes('*')) { - return false; - } - return true; -} - -type AssetInit = { - status?: number, - headers?: Record, - cache?: 'extended' | 'never' | null, -}; - -export class PublisherServer { - private readonly serverConfig: PublisherServerConfigNormalized; - private readonly staticItems: (string|RegExp)[]; - private readonly contentAssets: ContentAssets; - - public constructor( - serverConfig: PublisherServerConfigNormalized, - contentAssets: ContentAssets, - ) { - this.serverConfig = serverConfig; - this.contentAssets = contentAssets; - - this.staticItems = serverConfig.staticItems - .map((x, i) => { - if (x.startsWith('re:')) { - const fragments = x.slice(3).match(/\/(.*?)\/([a-z]*)?$/i); - if (fragments == null) { - console.warn(`Cannot parse staticItems item index ${i}: '${x}', skipping...`); - return ''; - } - return new RegExp(fragments[1], fragments[2] || ''); - } - return x; - }) - .filter(x => Boolean(x)); - } - - getMatchingAsset(path: string): ContentAsset | null { - const assetKey = this.serverConfig.publicDirPrefix + path; - - if(!assetKey.endsWith('/')) { - // A path that does not end in a slash can match an asset directly - const asset = this.contentAssets.getAsset(assetKey); - if (asset != null) { - return asset; - } - - // ... or, we can try auto-ext: - // looks for an asset that has the specified suffix (usually extension, such as .html) - for (const extEntry of this.serverConfig.autoExt) { - let assetKeyWithExt = assetKey + extEntry; - const asset = this.contentAssets.getAsset(assetKeyWithExt); - if (asset != null) { - return asset; - } - } - } - - if(this.serverConfig.autoIndex.length > 0) { - // try auto-index: - // treats the path as a directory, and looks for an asset with the specified - // suffix (usually an index file, such as index.html) - let assetNameAsDir = assetKey; - // remove all slashes from end, and add one trailing slash - while(assetNameAsDir.endsWith('/')) { - assetNameAsDir = assetNameAsDir.slice(0, -1); - } - assetNameAsDir = assetNameAsDir + '/'; - for (const indexEntry of this.serverConfig.autoIndex) { - let assetKeyIndex = assetNameAsDir + indexEntry; - const asset = this.contentAssets.getAsset(assetKeyIndex); - if (asset != null) { - return asset; - } - } - } - - return null; - } - - findAcceptEncodings(request: Request): ContentCompressionTypes[][] { - if (this.serverConfig.compression.length === 0) { - return []; - } - const found = (request.headers.get('accept-encoding') ?? '') - .split(',') - .map(x => { - let [encodingValue, qValueStr] = x.trim().split(';'); - let qValue; // q value multiplied by 1000 - if (qValueStr == null || !qValueStr.startsWith('q=')) { - // use default of 1.0 - qValue = 1000; - } else { - qValueStr = qValueStr.slice(2); // remove the q= - qValue = parseFloat(qValueStr); - if (Number.isNaN(qValue) || qValue > 1) { - qValue = 1; - } - if (qValue < 0) { - qValue = 0; - } - // q values can have up to 3 decimal digits - qValue = Math.floor(qValue * 1000); - } - return [encodingValue.trim(), qValue] as const; - }) - .filter(([encoding]) => - this.serverConfig.compression.includes(encoding as ContentCompressionTypes) - ); - - const priorityMap = new Map; - for (const [encoding, qValue] of found) { - let typesForQValue = priorityMap.get(qValue); - if (typesForQValue == null) { - typesForQValue = []; - priorityMap.set(qValue, typesForQValue); - } - typesForQValue.push(encoding as ContentCompressionTypes); - } - - // Sort keys, larger numbers to come first - const keysSorted = [...priorityMap.keys()] - .sort((qValueA, qValueB) => qValueB - qValueA); - - return keysSorted - .map(qValue => priorityMap.get(qValue)!); - } - - testExtendedCache(pathname: string) { - return this.staticItems - .some(x => { - if (x instanceof RegExp) { - return x.test(pathname); - } - if (x.endsWith('/')) { - return pathname.startsWith(x); - } - return x === pathname; - }); - } - - handlePreconditions(request: Request, asset: ContentAsset, responseHeaders: Record): Response | null { - // Handle preconditions according to https://httpwg.org/specs/rfc9110.html#rfc.section.13.2.2 - - // A recipient cache or origin server MUST evaluate the request preconditions defined by this specification in the following order: - // 1. When recipient is the origin server and If-Match is present, evaluate the If-Match precondition: - // - if true, continue to step 3 - // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.1) - - // 2. When recipient is the origin server, If-Match is not present, and If-Unmodified-Since is present, evaluate the If-Unmodified-Since precondition: - // - if true, continue to step 3 - // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.4) - - // 3. When If-None-Match is present, evaluate the If-None-Match precondition: - // - if true, continue to step 5 - // - if false for GET/HEAD, respond 304 (Not Modified) - // - if false for other methods, respond 412 (Precondition Failed) - - let skipIfNoneMatch = false; - { - const headerValue = getIfNoneMatchHeader(request); - if (headerValue.length > 0) { - const result = checkIfNoneMatch(responseHeaders['ETag'], headerValue); - if (result) { - skipIfNoneMatch = true; - } else { - return new Response(null, { - status: 304, - headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), - }); - } - } - } - - // 4. When the method is GET or HEAD, If-None-Match is not present, and If-Modified-Since is present, evaluate the - // If-Modified-Since precondition: - // - if true, continue to step 5 - // - if false, respond 304 (Not Modified) - - if (!skipIfNoneMatch) { - // For us, method is always GET or HEAD here. - const headerValue = getIfModifiedSinceHeader(request); - if (headerValue != null) { - const result = checkIfModifiedSince(asset, headerValue); - if (!result) { - return new Response(null, { - status: 304, - headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), - }); - } - } - } - - // 5. When the method is GET and both Range and If-Range are present, evaluate the If-Range precondition: - // - if true and the Range is applicable to the selected representation, respond 206 (Partial Content) - // - otherwise, ignore the Range header field and respond 200 (OK) - - // 6. Otherwise, - // - perform the requested method and respond according to its success or failure. - return null; - } - - async serveAsset(request: Request, asset: ContentAsset, init?: AssetInit): Promise { - const metadata = asset.getMetadata(); - const headers: Record = { - 'Content-Type': metadata.contentType, - }; - Object.assign(headers, init?.headers); - - if (init?.cache != null) { - let cacheControlValue; - switch(init.cache) { - case 'extended': - cacheControlValue = 'max-age=31536000'; - break; - case 'never': - cacheControlValue = 'no-store'; - break; - } - headers['Cache-Control'] = cacheControlValue; - } - - const acceptEncodings = this.findAcceptEncodings(request); - const storeEntry = await asset.getStoreEntry(acceptEncodings); - if (storeEntry.contentEncoding != null) { - headers['Content-Encoding'] = storeEntry.contentEncoding; - } - - headers['ETag'] = `"${storeEntry.hash}"`; - if (metadata.lastModifiedTime !== 0) { - headers['Last-Modified'] = (new Date( metadata.lastModifiedTime * 1000 )).toUTCString(); - } - - const preconditionResponse = this.handlePreconditions(request, asset, headers); - if (preconditionResponse != null) { - return preconditionResponse; - } - - return new Response(storeEntry.body, { - status: init?.status ?? 200, - headers, - }); - } - - async serveRequest(request: Request): Promise { - - // Only handle GET and HEAD - if (request.method !== 'GET' && request.method !== 'HEAD') { - return null; - } - - const url = new URL(request.url); - const pathname = decodeURI(url.pathname); - - const asset = this.getMatchingAsset(pathname); - if (asset != null) { - return this.serveAsset(request, asset, { - cache: this.testExtendedCache(pathname) ? 'extended' : null, - }); - } - - // fallback HTML responses, like SPA and "not found" pages - if (requestAcceptsTextHtml(request)) { - - // These are raw asset paths, not relative to public path - const { spaFile } = this.serverConfig; - - if (spaFile != null) { - const asset = this.contentAssets.getAsset(spaFile); - if (asset != null) { - return this.serveAsset(request, asset, { - cache: 'never', - }); - } - } - - const { notFoundPageFile } = this.serverConfig; - if (notFoundPageFile != null) { - const asset = this.contentAssets.getAsset(notFoundPageFile); - if (asset != null) { - return this.serveAsset(request, asset, { - status: 404, - cache: 'never', - }); - } - } - } - - return null; - } -} diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts new file mode 100644 index 0000000..6072586 --- /dev/null +++ b/src/server/publisher-server/index.ts @@ -0,0 +1,566 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +/// +import { KVStore, type KVStoreEntry } from 'fastly:kv-store'; + +import { + type StaticPublishRc, +} from '../../models/config/static-publish-rc.js'; +import { + type PublisherServerConfigNormalized, +} from '../../models/config/publisher-server-config.js'; +import { + type ContentCompressionTypes, +} from '../../models/compression/index.js'; +import { + isKVAssetVariantMetadata, + type KVAssetEntry, + type KVAssetEntryMap, + type KVAssetVariantMetadata, +} from '../../models/assets/kvstore-assets.js'; +import { type IndexMetadata } from '../../models/server/index.js'; +import { getKVStoreEntry } from '../util/kv-store.js'; +import { checkIfModifiedSince, getIfModifiedSinceHeader } from './serve-preconditions/if-modified-since.js'; +import { checkIfNoneMatch, getIfNoneMatchHeader } from './serve-preconditions/if-none-match.js'; +import { isExpired } from "../../models/time/index.js"; + +type KVAssetVariant = { + kvStoreEntry: KVStoreEntry, +} & KVAssetVariantMetadata; + +export function buildHeadersSubset(responseHeaders: Headers, keys: Readonly) { + const resultHeaders = new Headers(); + for (const value of keys) { + if (value in responseHeaders) { + const responseHeaderValue = responseHeaders.get(value); + if (responseHeaderValue != null) { + resultHeaders.set(value, responseHeaderValue); + } + } + } + return resultHeaders; +} + +// https://httpwg.org/specs/rfc9110.html#rfc.section.15.4.5 +// The server generating a 304 response MUST generate any of the following header fields that would have been sent in +// a 200 (OK) response to the same request: +// * Content-Location, Date, ETag, and Vary +// * Cache-Control and Expires +const headersToPreserveForUnmodified = ['Content-Location', 'ETag', 'Vary', 'Cache-Control', 'Expires'] as const; + +function requestAcceptsTextHtml(req: Request) { + const accept = (req.headers.get('Accept') ?? '') + .split(',') + .map(x => x.split(';')[0]); + if(!accept.includes('text/html') && !accept.includes('*/*') && accept.includes('*')) { + return false; + } + return true; +} + +type AssetInit = { + status?: number, + headers?: Record, + cache?: 'extended' | 'never' | null, +}; + +export class PublisherServer { + public constructor( + publishId: string, + kvStoreName: string, + defaultCollectionName: string, + ) { + this.publishId = publishId; + this.kvStoreName = kvStoreName; + this.defaultCollectionName = defaultCollectionName; + this.activeCollectionName = this.defaultCollectionName; + this.collectionNameHeader = 'X-Publisher-Server-Collection'; + } + + static fromStaticPublishRc(config: StaticPublishRc) { + return new PublisherServer( + config.publishId, + config.kvStoreName, + config.defaultCollectionName, + ); + } + + publishId: string; + kvStoreName: string; + defaultCollectionName: string; + activeCollectionName: string; + collectionNameHeader: string | null; + + // Cached settings + settingsCached: PublisherServerConfigNormalized | null | undefined; + + // Cached index + kvAssetsIndex: KVAssetEntryMap | null | undefined; + + setActiveCollectionName(collectionName: string) { + this.activeCollectionName = collectionName; + this.settingsCached = undefined; + this.kvAssetsIndex = undefined; + } + + setCollectionNameHeader(collectionHeader: string | null) { + this.collectionNameHeader = collectionHeader; + } + + // Server config is obtained from the KV Store, and cached for the duration of this object. + async getServerConfig() { + if (this.settingsCached !== undefined) { + return this.settingsCached; + } + const settingsFileKey = `${this.publishId}_settings_${this.activeCollectionName}`; + const kvStore = new KVStore(this.kvStoreName); + const settingsFile = await getKVStoreEntry(kvStore, settingsFileKey); + if (settingsFile == null) { + console.error(`Settings File not found at ${settingsFileKey}.`); + console.error(`You may need to publish your application.`); + this.settingsCached = null; + } else { + this.settingsCached = (await settingsFile.json()) as PublisherServerConfigNormalized; + } + return this.settingsCached; + } + + async getStaticItems() { + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return []; + } + return serverConfig.staticItems + .map((x, i) => { + if (x.startsWith('re:')) { + const fragments = x.slice(3).match(/\/(.*?)\/([a-z]*)?$/i); + if (fragments == null) { + console.warn(`Cannot parse staticItems item index ${i}: '${x}', skipping...`); + return ''; + } + return new RegExp(fragments[1], fragments[2] || ''); + } + return x; + }) + .filter(x => Boolean(x)); + } + + async getKvAssetsIndex() { + if (this.kvAssetsIndex !== undefined) { + return this.kvAssetsIndex; + } + const indexFileKey = `${this.publishId}_index_${this.activeCollectionName}`; + const kvStore = new KVStore(this.kvStoreName); + const indexFile = await getKVStoreEntry(kvStore, indexFileKey); + if (indexFile == null) { + console.error(`Index File not found at ${indexFileKey}.`); + console.error(`You may need to publish your application.`); + this.kvAssetsIndex = null; + return null; + } + + let collectionIsExpired = false; + if (this.activeCollectionName !== this.defaultCollectionName) { + const metadataText = indexFile.metadataText() + if (metadataText != null) { + try { + const metadata = JSON.parse(metadataText) as IndexMetadata; + collectionIsExpired = metadata.expirationTime != null && isExpired(metadata.expirationTime); + } catch { + } + } + } + + if (collectionIsExpired) { + console.error(`Requested collection expired at ${indexFileKey}.`); + this.kvAssetsIndex = null; + return null; + } + + this.kvAssetsIndex = (await indexFile.json()) as KVAssetEntryMap; + return this.kvAssetsIndex; + } + + async getMatchingAsset(assetKey: string, applyAuto: boolean = false): Promise { + + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return null; + } + const kvAssetsIndex = await this.getKvAssetsIndex(); + if (kvAssetsIndex == null) { + return null; + } + + if(!assetKey.endsWith('/')) { + // A path that does not end in a slash can match an asset directly + const asset = kvAssetsIndex[assetKey]; + if (asset != null) { + return asset; + } + + if (applyAuto) { + // ... or, we can try auto-ext: + // looks for an asset that has the specified suffix (usually extension, such as .html) + for (const extEntry of serverConfig.autoExt) { + let assetKeyWithExt = assetKey + extEntry; + const asset = kvAssetsIndex[assetKeyWithExt]; + if (asset != null) { + return asset; + } + } + } + } + + if (applyAuto) { + if (serverConfig.autoIndex.length > 0) { + // try auto-index: + // treats the path as a directory, and looks for an asset with the specified + // suffix (usually an index file, such as index.html) + let assetNameAsDir = assetKey; + // remove all slashes from end, and add one trailing slash + while(assetNameAsDir.endsWith('/')) { + assetNameAsDir = assetNameAsDir.slice(0, -1); + } + assetNameAsDir = assetNameAsDir + '/'; + for (const indexEntry of serverConfig.autoIndex) { + let assetKeyIndex = assetNameAsDir + indexEntry; + const asset = kvAssetsIndex[assetKeyIndex]; + if (asset != null) { + return asset; + } + } + } + } + + return null; + } + + // A pedantic function that returns all content types that are requested for in the accept-encoding header that are + // accepted by the server config, grouped by descending order of q values. + // For example, if accept-encoding had br;q=1,gzip;q=0.5, and the server accepts both br and gzip, + // the result would be [['br'], ['gzip']] + async findAcceptEncodingsGroups(request: Request): Promise { + const serverConfig = await this.getServerConfig(); + if (serverConfig == null || serverConfig.allowedEncodings.length === 0) { + return []; + } + + const acceptEncodingHeader = request.headers.get('accept-encoding')?.trim() ?? ''; + if (acceptEncodingHeader == '') { + return []; + } + + const priorityMap = new Map; + + for (const headerValue of acceptEncodingHeader.trim().split(',')) { + let [encodingValue, qValueStr] = headerValue.trim().split(';'); + encodingValue = encodingValue.trim(); + if (!serverConfig.allowedEncodings.includes(encodingValue as ContentCompressionTypes)) { + continue; + } + let qValue; // q value multiplied by 1000 + if (qValueStr == null || !qValueStr.startsWith('q=')) { + // use default of 1.0 + qValue = 1000; + } else { + qValueStr = qValueStr.slice(2); // remove the q= + qValue = parseFloat(qValueStr); + if (Number.isNaN(qValue) || qValue > 1) { + qValue = 1; + } + if (qValue < 0) { + qValue = 0; + } + // q values can have up to 3 decimal digits + qValue = Math.floor(qValue * 1000); + } + + let typesForQValue = priorityMap.get(qValue); + if (typesForQValue == null) { + typesForQValue = []; + priorityMap.set(qValue, typesForQValue); + } + typesForQValue.push(encodingValue as ContentCompressionTypes); + } + + // Sort keys, larger numbers to come first + const keysSorted = [...priorityMap.keys()] + .sort((qValueA, qValueB) => qValueB - qValueA); + + return keysSorted + .map(qValue => priorityMap.get(qValue)!); + } + + async testExtendedCache(pathname: string) { + const staticItems = await this.getStaticItems(); + return staticItems + .some(x => { + if (x instanceof RegExp) { + return x.test(pathname); + } + if (x.endsWith('/')) { + return pathname.startsWith(x); + } + return x === pathname; + }); + } + + handlePreconditions(request: Request, asset: KVAssetEntry, responseHeaders: Headers): Response | null { + // Handle preconditions according to https://httpwg.org/specs/rfc9110.html#rfc.section.13.2.2 + + // A recipient cache or origin server MUST evaluate the request preconditions defined by this specification in the following order: + // 1. When recipient is the origin server and If-Match is present, evaluate the If-Match precondition: + // - if true, continue to step 3 + // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.1) + + // 2. When recipient is the origin server, If-Match is not present, and If-Unmodified-Since is present, evaluate the If-Unmodified-Since precondition: + // - if true, continue to step 3 + // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.4) + + // 3. When If-None-Match is present, evaluate the If-None-Match precondition: + // - if true, continue to step 5 + // - if false for GET/HEAD, respond 304 (Not Modified) + // - if false for other methods, respond 412 (Precondition Failed) + + let skipIfNoneMatch = false; + { + const headerValue = getIfNoneMatchHeader(request); + if (headerValue.length > 0) { + const result = checkIfNoneMatch(responseHeaders.get('ETag')!, headerValue); + if (result) { + skipIfNoneMatch = true; + } else { + return new Response(null, { + status: 304, + headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), + }); + } + } + } + + // 4. When the method is GET or HEAD, If-None-Match is not present, and If-Modified-Since is present, evaluate the + // If-Modified-Since precondition: + // - if true, continue to step 5 + // - if false, respond 304 (Not Modified) + + if (!skipIfNoneMatch) { + // For us, method is always GET or HEAD here. + const headerValue = getIfModifiedSinceHeader(request); + if (headerValue != null) { + const result = checkIfModifiedSince(asset.lastModifiedTime, headerValue); + if (!result) { + return new Response(null, { + status: 304, + headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), + }); + } + } + } + + // 5. When the method is GET and both Range and If-Range are present, evaluate the If-Range precondition: + // - if true and the Range is applicable to the selected representation, respond 206 (Partial Content) + // - otherwise, ignore the Range header field and respond 200 (OK) + + // 6. Otherwise, + // - perform the requested method and respond according to its success or failure. + return null; + } + + public async loadKvAssetVariant(entry: KVAssetEntry, variant: ContentCompressionTypes | null): Promise { + + const kvStore = new KVStore(this.kvStoreName); + + const baseHash = entry.key.slice(7); + const baseKey = `${this.publishId}_files_sha256_${baseHash}`; + const variantKey = variant != null ? `${baseKey}_${variant}` : baseKey; + + const kvStoreEntry = await getKVStoreEntry(kvStore, variantKey); + if (kvStoreEntry == null) { + return null; + } + const metadataText = kvStoreEntry.metadataText() ?? ''; + if (metadataText === '') { + return null; + } + let metadata; + try { + metadata = JSON.parse(metadataText); + } catch { + return null; + } + if (!isKVAssetVariantMetadata(metadata)) { + return null; + } + if (metadata.size == null) { + return null; + } + return { + kvStoreEntry, + ...metadata, + }; + } + + private async findKVAssetVariantForAcceptEncodingsGroups(entry: KVAssetEntry, acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { + + if (!entry.key.startsWith('sha256:')) { + throw new TypeError(`Key must start with 'sha256:': ${entry.key}`); + } + + // Each encodingGroup is an array of Accept-Encodings that have the same q value, + // with the highest first + for (const encodingGroup of acceptEncodingsGroups) { + + let smallestSize: number | undefined = undefined; + let smallestEntry: KVAssetVariant | undefined = undefined; + + for (const encoding of encodingGroup) { + if (!entry.variants.includes(encoding)) { + continue; + } + + const variantKvStoreEntry = await this.loadKvAssetVariant(entry, encoding); + if (variantKvStoreEntry == null) { + continue; + } + if (smallestSize == null || variantKvStoreEntry.size < smallestSize) { + smallestSize = variantKvStoreEntry.size; + smallestEntry = variantKvStoreEntry; + } + } + + if (smallestEntry != null) { + return smallestEntry; + } + } + + const baseKvStoreEntry = await this.loadKvAssetVariant(entry, null); + if (baseKvStoreEntry == null) { + throw new TypeError('Key not found: ' + entry.key); + } + + return baseKvStoreEntry; + } + + async serveAsset(request: Request, asset: KVAssetEntry, init?: AssetInit): Promise { + + const headers = new Headers(init?.headers); + headers.set('Content-Type', asset.contentType); + + if (this.collectionNameHeader) { + headers.set(this.collectionNameHeader, this.activeCollectionName); + headers.append('Vary', this.collectionNameHeader); + } + + if (init?.cache != null) { + let cacheControlValue; + switch(init.cache) { + case 'extended': + cacheControlValue = 'max-age=31536000'; + break; + case 'never': + cacheControlValue = 'no-store'; + break; + } + headers.append('Cache-Control', cacheControlValue); + } + + const acceptEncodings = await this.findAcceptEncodingsGroups(request); + const kvAssetVariant = await this.findKVAssetVariantForAcceptEncodingsGroups(asset, acceptEncodings); + if (kvAssetVariant.contentEncoding != null) { + headers.append('Content-Encoding', kvAssetVariant.contentEncoding); + } + + headers.set('ETag', `"${kvAssetVariant.hash}"`); + if (asset.lastModifiedTime !== 0) { + headers.set('Last-Modified', (new Date( asset.lastModifiedTime * 1000 )).toUTCString()); + } + + const preconditionResponse = this.handlePreconditions(request, asset, headers); + if (preconditionResponse != null) { + return preconditionResponse; + } + + const kvStoreEntry = kvAssetVariant.kvStoreEntry; + return new Response( + kvStoreEntry.body, + { + status: init?.status ?? 200, + headers, + } + ); + } + + async serveRequest(request: Request): Promise { + + // Only handle GET and HEAD + if (request.method !== 'GET' && request.method !== 'HEAD') { + return null; + } + + const url = new URL(request.url); + const pathname = decodeURI(url.pathname); + + // Custom health check route + if (pathname === '/healthz') { + return new Response("OK", { status: 200 }); + } + + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return new Response( + `Settings not found. You may need to publish your application.`, + { + status: 500, + headers: { + 'content-type': 'text/plain', + }, + }, + ); + } + + const asset = await this.getMatchingAsset(serverConfig.publicDirPrefix + pathname, true); + if (asset != null) { + return this.serveAsset(request, asset, { + cache: await this.testExtendedCache(pathname) ? 'extended' : null, + }); + } + + // fallback HTML responses, like SPA and "not found" pages + if (requestAcceptsTextHtml(request)) { + + const kvAssetsIndex = await this.getKvAssetsIndex(); + if (kvAssetsIndex == null) { + return null; + } + + // These are raw asset paths, not relative to public path + const { spaFile } = serverConfig; + + if (spaFile != null) { + const asset = kvAssetsIndex[spaFile]; + if (asset != null) { + return this.serveAsset(request, asset, { + cache: 'never', + }); + } + } + + const { notFoundPageFile } = serverConfig; + if (notFoundPageFile != null) { + const asset = kvAssetsIndex[notFoundPageFile]; + if (asset != null) { + return this.serveAsset(request, asset, { + status: 404, + cache: 'never', + }); + } + } + } + + return null; + } +} diff --git a/src/server/serve-preconditions/if-modified-since.ts b/src/server/publisher-server/serve-preconditions/if-modified-since.ts similarity index 80% rename from src/server/serve-preconditions/if-modified-since.ts rename to src/server/publisher-server/serve-preconditions/if-modified-since.ts index 2639d1c..029d2fb 100644 --- a/src/server/serve-preconditions/if-modified-since.ts +++ b/src/server/publisher-server/serve-preconditions/if-modified-since.ts @@ -1,6 +1,10 @@ -import { ContentAsset } from "../../types/index.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ // https://httpwg.org/specs/rfc9110.html#field.if-modified-since + export function getIfModifiedSinceHeader(request: Request): number | null { // A recipient MUST ignore the If-Modified-Since header field if the received // field value is not a valid HTTP-date, the field value has more than one @@ -19,11 +23,11 @@ export function getIfModifiedSinceHeader(request: Request): number | null { return Math.floor(dateValueMs / 1000); } -export function checkIfModifiedSince(asset: ContentAsset, ifModifiedSince: number): boolean { +export function checkIfModifiedSince(lastModifiedTime: number, ifModifiedSince: number): boolean { // 1. If the selected representation's last modification date is earlier or equal to the // date provided in the field value, the condition is false. - if (asset.getMetadata().lastModifiedTime <= ifModifiedSince) { + if (lastModifiedTime <= ifModifiedSince) { return false; } diff --git a/src/server/serve-preconditions/if-none-match.ts b/src/server/publisher-server/serve-preconditions/if-none-match.ts similarity index 91% rename from src/server/serve-preconditions/if-none-match.ts rename to src/server/publisher-server/serve-preconditions/if-none-match.ts index d92d7f2..2b11e35 100644 --- a/src/server/serve-preconditions/if-none-match.ts +++ b/src/server/publisher-server/serve-preconditions/if-none-match.ts @@ -1,4 +1,7 @@ -import { ContentAsset } from "../../types/index.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ // https://httpwg.org/specs/rfc9110.html#field.if-none-match diff --git a/src/server/util/cookies.ts b/src/server/util/cookies.ts new file mode 100644 index 0000000..34e5c75 --- /dev/null +++ b/src/server/util/cookies.ts @@ -0,0 +1,16 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export function getCookieValue(request: Request, name: string) { + const cookieHeader = request.headers.get('Cookie'); + if (!cookieHeader) { + return null; + } + + return cookieHeader + .split(';') + .map(v => v.trim().split('=')) + .find(([key]) => key === name)?.[1] ?? null; +} diff --git a/src/server/util/http.ts b/src/server/util/http.ts deleted file mode 100644 index 466ceff..0000000 --- a/src/server/util/http.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function buildHeadersSubset(responseHeaders: Record, keys: Readonly) { - const resultHeaders: Record = {}; - for (const value of keys) { - if (value in responseHeaders) { - resultHeaders[value] = responseHeaders[value]; - } - } - return resultHeaders; -} diff --git a/src/server/util/kv-store.ts b/src/server/util/kv-store.ts new file mode 100644 index 0000000..43e625b --- /dev/null +++ b/src/server/util/kv-store.ts @@ -0,0 +1,118 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +/// +import { KVStore, type KVStoreEntry } from 'fastly:kv-store'; + +// This is a custom KVStoreEntry implementation that is built by using a body and +// metadataText passed in through the constructor. We piggyback off Response +// for the utility functions such as json() and text(). +class CustomKVStoreEntry extends Response implements KVStoreEntry { + constructor(body: BodyInit, metadataText: string) { + super(body); + this.metadataTextValue = metadataText; + } + + private readonly metadataTextValue: string; + get body(): ReadableStream { + return super.body!; + } + + metadata(): ArrayBuffer | null { + return new TextEncoder().encode(this.metadataTextValue); + } + metadataText(): string | null { + return this.metadataTextValue; + } +} + +export async function getKVStoreEntry( + kvStore: KVStore, + key: string, +): Promise { + + const entry = await kvStore.get(key); + if (entry == null) { + return null; + } + + const metadataText = entry.metadataText() ?? ''; + if (metadataText === '') { + return entry; + } + + let metadata; + try { + metadata = JSON.parse(metadataText); + } catch { + return entry; + } + + if (!('numChunks' in metadata) || typeof metadata.numChunks !== 'number') { + return entry; + } + + if (metadata.numChunks < 2) { + return entry; + } + + const streams = [ + entry.body, + ]; + + for (let chunkIndex = 1; chunkIndex < metadata.numChunks; chunkIndex++) { + const chunkKey = `${key}_${chunkIndex}`; + const chunkEntry = await kvStore.get(chunkKey); + if (chunkEntry == null) { + throw new Error(`Missing chunk ${chunkKey}`); + } + streams.push(chunkEntry?.body); + } + + const combinedStream = concatReadableStreams(streams); + + return new CustomKVStoreEntry(combinedStream, metadataText); +} + +function concatReadableStreams(streams: ReadableStream[]): ReadableStream { + let currentStreamIndex = 0; + let currentReader: ReadableStreamDefaultReader | null = null; + + return new ReadableStream({ + async pull(controller) { + while (true) { + // If no current reader, get one from the next stream + if (!currentReader) { + if (currentStreamIndex >= streams.length) { + controller.close(); + return; + } + currentReader = streams[currentStreamIndex].getReader(); + } + + const { value, done } = await currentReader.read(); + + if (done) { + currentReader.releaseLock(); + currentReader = null; + currentStreamIndex++; + continue; // Go to next stream + } + + controller.enqueue(value); + return; // Let pull() be called again + } + }, + async cancel(reason) { + if (currentReader) { + try { + await currentReader.cancel(reason); + } catch { + // swallow + } + } + } + }); +} diff --git a/src/types/compute.ts b/src/types/compute.ts deleted file mode 100644 index da4d095..0000000 --- a/src/types/compute.ts +++ /dev/null @@ -1,12 +0,0 @@ -export interface StoreEntry { - readonly body: ReadableStream | null; - readonly bodyUsed: boolean; - arrayBuffer(): Promise; - // blob(): Promise; - // formData(): Promise; - json(): Promise; - text(): Promise; - readonly contentEncoding: string | null; - readonly hash: string; - readonly size: number; -} diff --git a/src/types/config-normalized.ts b/src/types/config-normalized.ts deleted file mode 100644 index 822012f..0000000 --- a/src/types/config-normalized.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { ContentAssetInclusionTest, ExcludeDirTest, ModuleAssetInclusionTest } from "./config.js"; -import type { ContentTypeDef } from "./content-types.js"; -import type { ContentCompressionTypes } from "../constants/compression.js"; - -export type ContentAssetInclusionResultNormalized = { - includeContent: boolean, - inline: boolean, -}; - -export type ModuleAssetInclusionResultNormalized = { - includeModule: boolean, - useStaticImport: boolean, -}; - -export type StaticPublisherConfigNormalized = { - rootDir: string, - staticContentRootDir: string, - kvStoreName: string | null, - excludeDirs: ExcludeDirTest[], - excludeDotFiles: boolean, - includeWellKnown: boolean, - contentAssetInclusionTest: ContentAssetInclusionTest | null; - contentCompression: ContentCompressionTypes[], - moduleAssetInclusionTest: ModuleAssetInclusionTest | null; - contentTypes: ContentTypeDef[], - server: PublisherServerConfigNormalized | null, -}; - -export type PublisherServerConfigNormalized = { - publicDirPrefix: string, - staticItems: string[], - compression: ContentCompressionTypes[], - spaFile: string | null, - notFoundPageFile: string | null, - autoExt: string[], - autoIndex: string[], - // modifyResponse: ModifyResponseFunction | null, -}; diff --git a/src/types/config.ts b/src/types/config.ts deleted file mode 100644 index a9b0381..0000000 --- a/src/types/config.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { ContentTypeDef } from "./content-types.js"; - -export interface ExcludeDirTest { - test(name: string): boolean; -} - -export type ContentAssetInclusionResult = { - // Content asset inclusion mode: - - // includeContent: - // true - Include in Wasm binary and serve using includeBytes() function, or - // serve from KV Store if enabled. Default. - // false - Exclude from this publish. - includeContent?: boolean, - - // inline: - // true - Use includeBytes() for this asset, even if KV Store is enabled. - // false - If KV Store is enabled, serve the file from it. Default. - inline?: boolean, -}; - -// Asset Inclusion test -export type ContentAssetInclusionTest = (assetKey: string, contentType?: string) => ContentAssetInclusionResult | true | 'inline' | false | null; - -export type ModuleAssetInclusionResult = { - // Module asset inclusion mode: - - // includeModule: - // true - Include in Wasm binary as a dynamically loaded module. - // false - Exclude from this publish. Default. - includeModule?: boolean, - - // useStaticImport: - // true - If isModule is true, then import the module statically instead of dynamically. - // The dynamic loader getModule() function will simply return a reference to this - // instance, in this case. - // false - Don't load the module statically. Default. - useStaticImport?: boolean, -}; - -// Asset Inclusion test -export type ModuleAssetInclusionTest = (assetKey: string, contentType?: string) => ModuleAssetInclusionResult | true | false | 'static-import' | null; - -export type StaticPublisherConfig = { - // Set to a directory that acts as the root of all files that will be included in this publish. - rootDir: string, - - // Set to a directory that will hold the static content built by Static Publisher - // These files should not be committed to source control. './src' if not specified. - staticContentRootDir: string, - - // Set to a non-null string equal to the _name_ of an existing KV Store to enable "KV Store mode" for this publish. - // Service ID must also be specified in fastly.toml, or this will be an error. - kvStoreName?: string | false | null, - - // An array of values used to exclude files and directories (as well as files within those directories) from being - // included in this publish. Each entry in the array can be a string or a RegExp and will be tested against the relative - // path from 'rootDir' of each file or directory. - // Defaults to [ './node_modules' ]. Set to an empty array or specifically to null to include all files. - excludeDirs?: (string | ExcludeDirTest)[] | string | ExcludeDirTest | null, - - // If true, then files whose names begin with a dot, as well as files in directories whose names begin with a .dot, - // are excluded from this publish. Defaults to true. - excludeDotFiles?: boolean, - - // If true, include .well-known even if excludeDotFiles is true. - // Defaults to true. - includeWellKnown?: boolean, - - // A test to run on each asset key to determine whether and how to include the file as a content asset and/or module asset. - contentAssetInclusionTest?: ContentAssetInclusionTest | null; - - // Pre-generate content in these formats as well and serve them in tandem with the - // compression setting in the server settings. Default value is [ 'br' | 'gzip' ]. - contentCompression?: ('br' | 'gzip')[], - - // A test to run on each asset key to determine whether and how to include the file as a content asset and/or module asset. - moduleAssetInclusionTest?: ModuleAssetInclusionTest | null; - - // Additional / override content types. - contentTypes?: ContentTypeDef[], - - // Server settings - server?: PublisherServerConfig | null, -}; - -// Modify response -// export type ModifyResponseFunction = (response: Response, assetKey: string) => Response; - -// Publisher Server configuration - -export type PublisherServerConfig = { - // Prefix to apply to web requests. Effectively, a directory within rootDir that is used - // by the web server to determine the asset to respond with. Defaults to the empty string. - publicDirPrefix?: string, - - // A test to apply to item names to decide whether to serve them as "static" files, in other - // words, with a long TTL. These are used for files that are not expected to change. - // They can be provided as a string or array of strings. - // Items that contain asterisks, are interpreted as glob patterns. - // Items that end with a trailing slash are interpreted as directory names, - // Items that don't contain asterisks and that do not end in slash are checked for exact match. - staticItems?: string[] | string | false | null, - - // Compression. If the request contains an Accept-Encoding header, they are checked in the order listed - // in the header for the values listed here. The compression algorithm represented by the first match is applied. - // Default value is [ 'br', 'gzip' ]. - compression?: ('br' | 'gzip')[], - - // Set to the asset key of a content item to serve this when a GET request comes in for an unknown asset, and - // the Accept header includes text/html. - spaFile?: string | false | null, - - // Set to the asset key of a content item to serve this when a request comes in for an unknown asset, and - // the Accept header includes text/html. - notFoundPageFile?: string | false | null, - - // When a file is not found, and it doesn't end in a slash, then try auto-ext: try to serve a file with the same name - // postfixed with the specified strings, tested in the order listed. - autoExt?: string[] | string | false | null, - - // When a file is not found, then try auto-index: treat it as a directory, then try to serve a file that has the - // specified strings, tested in the order listed. - autoIndex?: string[] | string | false | null, - - // Modify Response before it is served - // modifyResponse?: ModifyResponseFunction | null, -}; diff --git a/src/types/content-assets.ts b/src/types/content-assets.ts deleted file mode 100644 index 16cd310..0000000 --- a/src/types/content-assets.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { ContentCompressionTypes } from "../constants/compression.js"; -import type { StoreEntry } from "../types/compute.js"; - -export type CompressedFileInfos = Partial>; - -export type ContentFileInfo = { - hash: string, // same as hash of file - size: number, -}; - -// For runtime use e.g., testing -export type ContentFileInfoForBytes = ContentFileInfo & { - bytes: Uint8Array, -}; -export type ContentFileInfoForString = ContentFileInfo & { - content: string, -}; - -// For static publishing -export type ContentFileInfoForStaticPublishing = ContentFileInfo & { - staticFilePath: string, -}; - -export type ContentFileInfoForWasmInline = ContentFileInfoForStaticPublishing; - -export type ContentFileInfoForKVStore = ContentFileInfoForStaticPublishing & { - kvStoreKey: string, -}; - -export type ContentAssetMetadataMapEntryBase = { - assetKey: string, - contentType: string, - text: boolean, - lastModifiedTime: number, // as unix time - fileInfo: TFileInfo, - compressedFileInfos: CompressedFileInfos -}; - -export type ContentAssetMetadataMapEntryBytes = { - type: 'bytes', -} & ContentAssetMetadataMapEntryBase; -export type ContentAssetMetadataMapEntryString = { - type: 'string', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntryWasmInline = { - type: 'wasm-inline', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntryKVStore = { - type: 'kv-store', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntry = - | ContentAssetMetadataMapEntryBytes - | ContentAssetMetadataMapEntryString - | ContentAssetMetadataMapEntryWasmInline - | ContentAssetMetadataMapEntryKVStore; - -export type ContentAssetMetadataMap = { - [assetKey: string]: ContentAssetMetadataMapEntry, -}; - -export interface ContentAsset { - readonly type: string; - readonly isLocal: boolean; - readonly assetKey: string; - getMetadata(): ContentAssetMetadataMapEntry; - getStoreEntry(acceptEncodingsGroups?: ContentCompressionTypes[][]): Promise; - getBytes(): Uint8Array; - getText(): string; - getJson(): T; -} diff --git a/src/types/content-types.ts b/src/types/content-types.ts deleted file mode 100644 index c813c07..0000000 --- a/src/types/content-types.ts +++ /dev/null @@ -1,24 +0,0 @@ -export type ContentTypeTest = (name: string) => boolean; - -// Content Type definition -export type ContentTypeDef = { - // A test on the asset key to perform on this content type. - test: RegExp | ContentTypeTest, - - // The Content-Type header value to provide for this content type. - contentType: string, - - // Whether this content type represents a text value encoded in utf-8. - // If so, conveniences can be provided. - text?: boolean, - - // Binary formats are usually not candidates for compression, but certain - // types can benefit from it. - precompressAsset?: boolean, -}; - -export type ContentTypeTestResult = { - contentType: string, - text: boolean, - precompressAsset: boolean, -}; diff --git a/src/types/index.ts b/src/types/index.ts deleted file mode 100644 index e538be3..0000000 --- a/src/types/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -export type { - StoreEntry, -} from './compute.js'; -export type { - StaticPublisherConfig, - ContentAssetInclusionTest, - ContentAssetInclusionResult, - ModuleAssetInclusionTest, - ModuleAssetInclusionResult, -} from './config.js'; -export type { - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, - ContentAsset, -} from './content-assets.js'; -export type { - ModuleAssetMap, - ModuleAssetMapEntry, - ModuleAsset, -} from './module-assets.js'; -export type { - ContentTypeTest, - ContentTypeDef, - ContentTypeTestResult, -} from './content-types.js'; -export type { - PublisherServerConfigNormalized, -} from './config-normalized.js'; diff --git a/src/types/module-assets.ts b/src/types/module-assets.ts deleted file mode 100644 index db4be9f..0000000 --- a/src/types/module-assets.ts +++ /dev/null @@ -1,15 +0,0 @@ -export type ModuleAssetMapEntry = { - isStaticImport: boolean; - module: any | null, - loadModule: () => Promise, -}; -export type ModuleAssetMap = { - [assetKey: string]: ModuleAssetMapEntry, -}; - -export interface ModuleAsset { - readonly assetKey: string; - getModule(): Promise; - getStaticModule(): any | null; - readonly isStaticImport: boolean; -} diff --git a/src/types/util.ts b/src/types/util.ts deleted file mode 100644 index d7cd5fa..0000000 --- a/src/types/util.ts +++ /dev/null @@ -1,7 +0,0 @@ -export type Merge = { - [TKey in keyof T]: TKey extends keyof U ? (T[TKey] | U[TKey]) : T[TKey]; -}; - -export type Unmerge = Required<{ - [TKey in keyof T]: TKey extends keyof U ? (Exclude) : T[TKey]; -}>; diff --git a/src/util/index.ts b/src/util/index.ts deleted file mode 100644 index 81f416b..0000000 --- a/src/util/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './content-types.js'; -export * from './metadata.js'; diff --git a/src/util/metadata.ts b/src/util/metadata.ts deleted file mode 100644 index 6cdd47d..0000000 --- a/src/util/metadata.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ContentAssetMetadataMap } from "../types/content-assets.js"; - -export function getKVStoreKeysFromMetadata(contentAssetMetadataMap: ContentAssetMetadataMap): Set { - - const results = new Set(); - - for (const metadata of Object.values(contentAssetMetadataMap)) { - if (metadata.type === 'kv-store') { - results.add(metadata.fileInfo.kvStoreKey); - for (const fileInfo of Object.values(metadata.compressedFileInfos)) { - results.add(fileInfo.kvStoreKey); - } - } - } - - return results; - -}