From 746555548820cbab9d582d227ee36330ad01abaa Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Mon, 31 Mar 2025 01:26:05 +0900 Subject: [PATCH 01/20] V7 features --- LICENSE | 2 +- MIGRATING.md | 4 + README.md | 159 ++-- package-lock.json | 251 +++++- package.json | 26 +- resources/README.md | 3 - resources/statics-metadata.d.ts | 10 - resources/statics.d.ts | 19 - resources/webpack.config.js | 41 - src/cli/commands/build-static.ts | 741 ------------------ src/cli/commands/clean-kv-store.ts | 82 -- src/cli/commands/clean.ts | 161 ++++ src/cli/commands/collections/delete.ts | 111 +++ src/cli/commands/collections/index.ts | 66 ++ src/cli/commands/collections/list.ts | 80 ++ src/cli/commands/index.ts | 85 ++ src/cli/commands/init-app.ts | 635 ++++++++------- src/cli/commands/publish-content.ts | 585 ++++++++++++++ src/cli/compression/brotli.ts | 20 +- src/cli/compression/gzip.ts | 20 +- src/cli/compression/index.ts | 25 +- .../fastly-api.ts => fastly-api/api-token.ts} | 51 +- src/cli/{util => fastly-api}/kv-store.ts | 57 +- src/cli/index.ts | 118 +-- src/cli/presets/implementations/astro.ts | 23 - .../implementations/create-react-app.ts | 34 - src/cli/presets/implementations/docusaurus.ts | 13 - src/cli/presets/implementations/gatsby.ts | 13 - src/cli/presets/implementations/next.ts | 13 - src/cli/presets/implementations/sveltekit.ts | 13 - src/cli/presets/implementations/vite.ts | 13 - src/cli/presets/implementations/vue.ts | 24 - src/cli/presets/index.ts | 22 - src/cli/presets/preset-base.ts | 21 - src/cli/util/args.ts | 86 ++ src/cli/{load-config.ts => util/config.ts} | 442 ++++++----- src/{ => cli}/util/content-types.ts | 15 +- src/cli/util/data.ts | 15 +- src/cli/util/files.ts | 73 +- src/cli/util/hash.ts | 14 - src/cli/util/id.ts | 22 - src/cli/util/kv-store-items.ts | 206 +++++ src/cli/util/kv-store-local-server.ts | 46 ++ src/cli/util/package.ts | 63 ++ src/cli/util/publish-id.ts | 34 - src/cli/util/retryable.ts | 5 + src/cli/util/variants.ts | 41 + src/compute-js.ts | 2 - src/constants/compression.ts | 6 - src/index.ts | 19 +- src/models/assets/kvstore-assets.ts | 50 ++ src/models/compression/index.ts | 15 + src/models/config/publish-content-config.ts | 91 +++ src/models/config/publisher-server-config.ts | 52 ++ src/models/config/static-publish-rc.ts | 16 + src/server/assets/asset-manager.ts | 21 - src/server/assets/content-asset-kv-store.ts | 107 --- .../assets/content-asset-wasm-inline.ts | 80 -- src/server/assets/content-assets.ts | 152 ---- src/server/assets/module-assets.ts | 71 -- .../collection-selector/from-config-store.ts | 18 + src/server/collection-selector/from-cookie.ts | 136 ++++ .../collection-selector/from-request.ts | 47 ++ src/server/collection-selector/index.ts | 24 + .../request-collection-selector.ts | 10 + src/server/kv-store/inline-store-entry.ts | 133 ---- src/server/publisher-server.ts | 313 -------- src/server/publisher-server/index.ts | 522 ++++++++++++ .../serve-preconditions/if-modified-since.ts | 10 +- .../serve-preconditions/if-none-match.ts | 5 +- src/server/util/cookies.ts | 16 + src/server/util/http.ts | 9 - src/server/util/kv-store.ts | 118 +++ src/types/compute.ts | 12 - src/types/config-normalized.ts | 38 - src/types/config.ts | 128 --- src/types/content-assets.ts | 73 -- src/types/content-types.ts | 24 - src/types/index.ts | 28 - src/types/module-assets.ts | 15 - src/types/util.ts | 7 - src/util/index.ts | 2 - src/util/metadata.ts | 18 - 83 files changed, 3791 insertions(+), 3200 deletions(-) delete mode 100644 resources/README.md delete mode 100644 resources/statics-metadata.d.ts delete mode 100644 resources/statics.d.ts delete mode 100644 resources/webpack.config.js delete mode 100644 src/cli/commands/build-static.ts delete mode 100644 src/cli/commands/clean-kv-store.ts create mode 100644 src/cli/commands/clean.ts create mode 100644 src/cli/commands/collections/delete.ts create mode 100644 src/cli/commands/collections/index.ts create mode 100644 src/cli/commands/collections/list.ts create mode 100644 src/cli/commands/index.ts create mode 100644 src/cli/commands/publish-content.ts rename src/cli/{util/fastly-api.ts => fastly-api/api-token.ts} (64%) rename src/cli/{util => fastly-api}/kv-store.ts (73%) delete mode 100644 src/cli/presets/implementations/astro.ts delete mode 100644 src/cli/presets/implementations/create-react-app.ts delete mode 100644 src/cli/presets/implementations/docusaurus.ts delete mode 100644 src/cli/presets/implementations/gatsby.ts delete mode 100644 src/cli/presets/implementations/next.ts delete mode 100644 src/cli/presets/implementations/sveltekit.ts delete mode 100644 src/cli/presets/implementations/vite.ts delete mode 100644 src/cli/presets/implementations/vue.ts delete mode 100644 src/cli/presets/index.ts delete mode 100644 src/cli/presets/preset-base.ts create mode 100644 src/cli/util/args.ts rename src/cli/{load-config.ts => util/config.ts} (63%) rename src/{ => cli}/util/content-types.ts (95%) delete mode 100644 src/cli/util/hash.ts delete mode 100644 src/cli/util/id.ts create mode 100644 src/cli/util/kv-store-items.ts create mode 100644 src/cli/util/kv-store-local-server.ts create mode 100644 src/cli/util/package.ts delete mode 100644 src/cli/util/publish-id.ts create mode 100644 src/cli/util/variants.ts delete mode 100644 src/compute-js.ts delete mode 100644 src/constants/compression.ts create mode 100644 src/models/assets/kvstore-assets.ts create mode 100644 src/models/compression/index.ts create mode 100644 src/models/config/publish-content-config.ts create mode 100644 src/models/config/publisher-server-config.ts create mode 100644 src/models/config/static-publish-rc.ts delete mode 100644 src/server/assets/asset-manager.ts delete mode 100644 src/server/assets/content-asset-kv-store.ts delete mode 100644 src/server/assets/content-asset-wasm-inline.ts delete mode 100644 src/server/assets/content-assets.ts delete mode 100644 src/server/assets/module-assets.ts create mode 100644 src/server/collection-selector/from-config-store.ts create mode 100644 src/server/collection-selector/from-cookie.ts create mode 100644 src/server/collection-selector/from-request.ts create mode 100644 src/server/collection-selector/index.ts create mode 100644 src/server/collection-selector/request-collection-selector.ts delete mode 100644 src/server/kv-store/inline-store-entry.ts delete mode 100644 src/server/publisher-server.ts create mode 100644 src/server/publisher-server/index.ts rename src/server/{ => publisher-server}/serve-preconditions/if-modified-since.ts (80%) rename src/server/{ => publisher-server}/serve-preconditions/if-none-match.ts (91%) create mode 100644 src/server/util/cookies.ts delete mode 100644 src/server/util/http.ts create mode 100644 src/server/util/kv-store.ts delete mode 100644 src/types/compute.ts delete mode 100644 src/types/config-normalized.ts delete mode 100644 src/types/config.ts delete mode 100644 src/types/content-assets.ts delete mode 100644 src/types/content-types.ts delete mode 100644 src/types/index.ts delete mode 100644 src/types/module-assets.ts delete mode 100644 src/types/util.ts delete mode 100644 src/util/index.ts delete mode 100644 src/util/metadata.ts diff --git a/LICENSE b/LICENSE index cb8d11d..56ccc75 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Fastly, Inc. +Copyright (c) 2025 Fastly, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MIGRATING.md b/MIGRATING.md index 795c4d1..700d342 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -7,6 +7,8 @@ This is straightforward if you're using `compute-js-static-publisher` out-of-the ## KV Store +KV Store is no longer optional as of v7 + Starting with `v5.0.0`, this tool refers to the KV Store using its finalized product name, "KV Store". References in code and in configuration that used the previous "Object Store" name have been changed to the new name. If you have been using the feature, you can take the following steps: @@ -37,6 +39,8 @@ been using the feature, you can take the following steps: ## Webpack +Webpack is no longer enabled for new projects + Starting with `v4.0.0` of this tool, webpack is no longer required and is disabled by default for new applications. This can simplify development and result in shorter build times. You may still wish to use webpack if you need some of the features it provides, e.g., the ability to use loaders, asset modules, module replacement, dynamic imports, etc. diff --git a/README.md b/README.md index 08bf715..9402aba 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,38 @@ # Static Publisher for JavaScript on Fastly Compute -Using a static site generator to build your website? Do you simply need to serve some static files? With `compute-js-static-publish`, now you can deploy and serve everything from Fastly's blazing-fast [Compute](https://developer.fastly.com/learning/compute/). - -## Prerequisites - -Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20 or newer. +Using a static site generator to build your website? Do you simply need to serve some static files? With `compute-js-static-publish`, now you can deploy and serve everything from Fastly's [Compute](https://developer.fastly.com/learning/compute/) platform. ## How it works -You have some HTML files, along with some accompanying CSS, JavaScript, image, and font files in a directory. Perhaps you've used a framework or static site generator to build these files. +You have some HTML files, along with some accompanying CSS, JavaScript, image, and/or font files in a directory. Perhaps you've used a framework or static site generator to build these files. -Assuming the root directory that contains your static files is `./public`, +Assuming the root directory that contains your static files is `./public`: ### 1. Run `compute-js-static-publish` ```shell -npx @fastly/compute-js-static-publish@latest --root-dir=./public +npx @fastly/compute-js-static-publish@latest --root-dir=./public --kv-store-name=my-store ``` -This will generate a Compute application at `./compute-js`. It will add a default `./compute-js/src/index.js` file that instantiates the [`PublisherServer`](#publisherserver) class and runs it to serve the static files from your project. +This will scaffold a Compute application at `./compute-js`. It will add a default `./compute-js/src/index.js` file that instantiates the [`PublisherServer`](#publisherserver) class and runs it to serve the static files from your project. + +You must also specify the name of a Fastly KV Store that you will be using when you deploy your application. The KV Store doesn't have to exist yet, we'll be doing that in the next step. > [!TIP] > This process creates a `./compute-js/static-publish.rc.js` to hold your configuration. This, as well as the other files created in your new Compute program at `./compute-js`, can be committed to source control (except for the ones we specify in `.gitignore`!) -> [!IMPORTANT] -> This step generates an application that includes your files and a program that serves them, and needs to be run only once. To make modifications to your application, simply make changes to your static files and rebuild it. Read the rest of this section for more details. +### 2. Set up your Fastly Service and KV Store + + + -### 2. Test your application locally + +The above step of generating your application that includes your files and a program that serves them, and needs to be run only once. To make modifications to your application, simply make changes to your static files and rebuild it. Read the rest of this section for more details. + + +### 3. Test your application locally + +Once your application has been generated and your KV Store is ready, it's ready to be run! The `start` script builds and runs your application using [Fastly's local development server](https://developer.fastly.com/learning/compute/testing/#running-a-local-testing-server). @@ -57,6 +63,10 @@ The `deploy` script builds and [publishes your application to a Compute service npm run deploy ``` +## Prerequisites + +Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20.11 or newer. + ## Features - Simple to set up, with a built-in server module. @@ -65,7 +75,8 @@ npm run deploy - Brotli and Gzip compression. - Support for `If-None-Match` and `If-Modified-Since` request headers. - Optionally use webpack as a module bundler. -- Selectively serve files from Fastly's [KV Store](#kv-store), or embedded into your Wasm module. +- Files are kept in Fastly's [KV Store](#kv-store). It is also possible to selectively serve some files + embedded into your Wasm module. - Supports loading JavaScript files as code into your Compute application. - Presets for several static site generators. @@ -76,8 +87,8 @@ Some of these features are new! If you wish to update to this version, you may n Once your application is scaffolded, `@fastly/compute-js-static-publish` integrates into your development process by running as part of your build process. -The files you have configured to be included (`--root-dir`) are enumerated and prepared. Their contents are included into -your Wasm binary (or made available via [KV Store](#kv-store), if so configured). This process is called "publishing". +The files you have configured to be included (`--root-dir`) are enumerated and prepared. Their contents are uploaded into +Fastly's [KV Store](#kv-store). This process is called "publishing". Once the files are published, they are available to the other source files in the Compute application. For example, the stock application runs the [PublisherServer](#publisherserver) class to serve these files. @@ -131,14 +142,19 @@ reading from stored configuration, then scanning the `--public-dir` directory to Any relative file and directory paths passed at the command line are handled as relative to the current directory. +### Required options: + +| Option | Description | +|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| `--root-dir` | The root of the directory that contains the files to include in the publishing. All files you wish to include must reside under this root. | +| `--kv-store-name` | The name of a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) to hold your assets. | + ### Publishing options: -| Option | Default | Description | -|-----------------------------|------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--preset` | (None) | Apply default options from a specified preset. See ["Frameworks and Static Site Generators"](#usage-with-frameworks-and-static-site-generators). | -| `--output` | `./compute-js` | The directory in which to create the Compute application. | -| `--static-content-root-dir` | (output directory) + `/static-publisher` | The directory under the Compute application where static asset and metadata are written. | -| `--root-dir` | (None) | **Required**. The root of the directory that contains the files to include in the publishing. All files you wish to include must reside under this root. | +| Option | Default | Description | +|----------------------------------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `--output` | `./compute-js` | The directory in which to create the Compute application. | +| `--static-publisher-working-dir` | (output directory) + `/static-publisher` | The directory under the Compute application where asset files are written in preparation for upload to the KV Store and for serving for local mode. | ### Server options: @@ -169,30 +185,6 @@ These arguments are used to populate the `fastly.toml` and `package.json` files | `--description` | `description` from `package.json`, or `Compute static site` | The description of your Compute application. | | `--author` | `author` from `package.json`, or `you@example.com` | The author of your Compute application. | | `--service-id` | (None) | The ID of an existing Fastly WASM service for your Compute application. | -| `--kv-store-name` | (None) | The name of an existing [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) to hold the content assets. In addition, a Resource Link to this KV Store must already exist on the service specified by `--service-id` and have the same name as the KV Store. | - -## Usage with frameworks and static site generators - -`compute-js-static-publish` supports preset defaults for a number of frameworks and static site generators: - -| `--preset` | `--root-dir` | `--static-dir` | Notes | -|-------------------------------|--------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------| -| `cra` (or `create-react-app`) | `./build` | `./build/static` | For apps written using [Create React App](https://create-react-app.dev). Checks for a dependency on `react-scripts`. | -| `cra-eject` | `./build` | `./build/static` | For apps written using Create React App, but which have since been ejected via `npm run eject`. Does not check for `react-scripts`. | -| `vite` | `./dist` | (None) | For apps written using [Vite](https://vitejs.dev). | -| `sveltekit` | `./dist` | (None) | For apps written using [SvelteKit](https://kit.svelte.dev). | -| `vue` | `./dist` | (None) | For apps written using [Vue](https://vuejs.org), and that were created using [create-vue](https://github.com/vuejs/create-vue). | -| `next` | `./out` | (None) | For apps written using [Next.js](https://nextjs.org), using `npm run export`. *1 | -| `astro` | `./dist` | (None) | For apps written using [Astro](https://astro.build) (static apps only). *2 | -| `gatsby` | `./public` | (None) | For apps written using [Gatsby](https://www.gatsbyjs.com). | -| `docusaurus` | `./build` | (None) | For apps written using [Docusaurus](https://docusaurus.io) | - -You may still override any of these options individually. - -*1 - For Next.js, consider using `@fastly/next-compute-js`, a Next.js server implementation that allows you to run - your Next.js application on Compute. - -*2 - Astro support does not support SSR. ## PublisherServer @@ -210,7 +202,7 @@ This server handles the following automatically: * Can be configured to serve a 404 not found file. * Returns `null` if nothing matches, so that you can add your own handling if necessary. -During initial scaffolding, the configuration based on the command-line parameters and preset are written to your `./static-publisher.rc.js` file under the `server` key. +During initial scaffolding, the configuration based on the command-line parameters is written to your `./static-publisher.rc.js` file under the `server` key. ### Configuring PublisherServer @@ -227,7 +219,8 @@ You can further configure the server by making modifications to the `server` key | `notFoundPageFile` | `null` | Asset key of a content item to serve with a status code of `404` when a GET request comes arrives for an unknown asset, and the Accept header includes text/html. | For `staticItems`: -* Items that contain asterisks are interpreted as glob patterns (for example, `/public/static/**/*.js`) +* The item name that is tested is the path of the request, without applying the publicDirPrefix. +* Items that contain asterisks are interpreted as glob patterns (for example, `/static/**/*.js`) * Items that end with a trailing slash are interpreted as a directory name. * Items that don't contain asterisks and that do not end in slash are checked for exact match. @@ -256,20 +249,24 @@ ID of the service. The Fastly CLI will deploy to the service identified by this To specify the service at the time you are scaffolding the project (for example, if you are running this tool and deploying as part of a CI process), specify the `--service-id` command line argument to populate `fastly.toml` with this value. -## Using the KV Store (BETA) +## Using the KV Store
-Starting with v4, it's now possible to upload assets to and serve them from a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores). +// Also, allow populating kv-store and publish id from command line -When this mode is enabled, you build your application as normal, and as a step during the build, your files -are uploaded to the Fastly KV Store, and metadata in the application is marked to source them from there instead -of from bytes in the Wasm binary. +Starting with v7, assets are uploaded to a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) +during the publishing process. In addition, the index file is also saved to the KV store. As a result, building this way +requires no deploy of your application to release a new version. + +// You build your application as normal, and as a step during the build, your files +// are uploaded to the Fastly KV Store, and metadata in the application is marked to source them from there instead +// of from bytes in the Wasm binary. You can enable the use of KV Store with `@fastly/compute-js-static-publish` as you scaffold your application, or at any later time. -At the time you enable the use of KV Store: +At the time you perform a publish: * Your Fastly service must already exist. See [Associating your project with a Fastly Service](#associating-your-project-with-a-fastly-service) above. @@ -354,12 +351,11 @@ And that's it! It should be possible to run this task to clean up once in a whil ### The `static-publish.rc.js` config file -* `rootDir` - All files under this root directory will be included by default in the publishing, +* `rootDir` - _Required._ All files under this root directory will be included by default in the publishing, except for those that are excluded using some of the following features. Files outside this root cannot be included in the publishing. -* `staticContentRootDir` - Static asset loader and metadata files are created under this directory. - For legacy compatibility, if not provided, defaults to `'./src'`. +* `staticPublisherWorkingDir` - _Required._ Static asset loader and metadata files are created under this directory. * `kvStoreName` - Set this value to the _name_ of an existing KV Store to enable uploading of content assets to Fastly KV Store. See [Using the KV Store](#kv-store) for more information. @@ -379,34 +375,18 @@ And that's it! It should be possible to run this task to clean up once in a whil * `includeWellKnown` - Unless disabled, will include a file or directory called `.well-known` even if `excludeDotfiles` would normally exclude it. This is `true` by default. -* `contentAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during - the publishing, to determine whether to include the asset as a content asset. For every file, this function is passed +* `kvStoreAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during + the publishing, to determine whether to include the asset. For every file, this function is passed the [asset key](#asset-keys), as well as its content type (MIME type string). You may return one of three values from this function: - * Boolean `true` - Include the file as a content asset in this publishing. Upload the file to and serve it from the - KV Store if KV Store mode is enabled, or include the contents of the file in the Wasm binary if KV Store - mode is not enabled. - * String `"inline"` - Include the file as a content asset in this publishing. Include the contents of the file in the - Wasm binary, regardless of whether KV Store mode is enabled. - * Boolean `false` - Do not include this file as a content asset in this publishing. - - If you do not provide a function, then every file will be included in this publishing as a content asset, and their - contents will be uploaded to and served from the KV Store if KV Store mode is enabled, or included in the Wasm - binary if KV Store mode is not enabled. + * Boolean `true` - Include the file. It is uploaded to the KV Store. + * Boolean `false` - exclude the file. + * Object. Include the file. It is uploaded to the KV Store. This object may specify, optionally: + * `contentType` to override the Content type + * `contentCompression` an array of strings to override the content compression types. Specify an empty array for no compression. * `contentCompression` - During the publishing, the tool will pre-generate compressed versions of content assets in these - formats and make them available to the Publisher Server or your application. Default value is [ 'br' | 'gzip' ] if - KV Store is enabled, or [] if KV Store is not enabled. - -* `moduleAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during - the publishing, to determine whether to include the asset as a module asset. For every file, this function is passed - the [asset key](#asset-keys), as well as its content type (MIME type string). You may return one of three values from this function: - * `true` (boolean) - Include the file as a module asset in this publishing. - * `"static-import"` (string) - Include the file as a module asset in this publishing, and statically import it. This causes - any top-level code in these modules to run at application initialization time. - * `false` (boolean) - Do not include this file as a module asset in this publishing. - - If you do not provide a function, then no module assets will be included in this publishing. + formats and make them available to the Publisher Server or your application. Default value is [ 'br' | 'gzip' ]. * `contentTypes` - Provide custom content types and/or override them. @@ -453,13 +433,13 @@ For example, if the `PublisherServer` is unable to formulate a response to the r add your own code to handle these cases, such as to provide custom responses. ```js -import { getServer } from './statics.js'; -const staticContentServer = getServer(); +import { getPublisherServer } from './statics.js'; +const publisherServer = getPublisherServer(); addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); async function handleRequest(event) { - const response = await staticContentServer.serveRequest(event.request); + const response = await publisherServer.serveRequest(event.request); if (response != null) { return response; } @@ -477,6 +457,9 @@ Publishing, as described earlier, is the process of preparing files for inclusio This process also makes metadata available about each of the files that are included, such as its content type, the last modified date, the file hash, and so on. +Publishing can also set up to embed content and modules into the Wasm binary of your Compute app if so configured, +although this usage will require your Wasm binary to be rebuilt and deployed to your service. + The [`PublisherServer` class](#publisherserver) used by the default scaffolded application is a simple application of this content and metadata. By importing `./statics.js` into your Compute application, you can just as easily access this information about the assets that were included during publishing. @@ -527,8 +510,6 @@ asset.type; const storeEntry = await asset.getStoreEntry(); storeEntry.contentEncoding; // null, 'br', 'gzip' -storeEntry.hash; // SHA256 of the contents of the file -storeEntry.size; // Size of file in bytes ``` Regardless of which store these objects come from, they implement the `Body` interface as defined by `@fastly/js-compute`. @@ -585,7 +566,7 @@ export function hello() { import { moduleAssets } from './statics'; // Obtain a module asset named '/module/hello.js' -const asset = contentAssets.getAsset('/module/hello.js'); +const asset = moduleAssets.getAsset('/module/hello.js'); // Load the module const helloModule = await asset.getModule(); @@ -606,12 +587,6 @@ publishing event. See the definition of `ContentAssetMetadataMapEntry` in the [`types/content-assets` file](./src/types/content-assets.ts) for more details. -### Using webpack - -As of v4, webpack is no longer required, and is no longer part of the default scaffolded application. -If you wish to use some features of webpack, you may include webpack in your generated application by specifying -`--webpack` at the command line. - ## Migrating See [MIGRATING.md](./MIGRATING.md). diff --git a/package-lock.json b/package-lock.json index 0d34eb7..116acb8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "6.3.0", "license": "MIT", "dependencies": { + "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", "glob-to-regexp": "^0.4.1" }, @@ -16,18 +17,18 @@ "compute-js-static-publish": "build/cli/index.js" }, "devDependencies": { - "@fastly/js-compute": "^3.0.0", + "@fastly/js-compute": "^3.33.2", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^20.0.0", + "@types/node": "^20.11.0", "rimraf": "^4.3.0", - "typescript": "^5.0.2" + "typescript": "^5.8.0" }, "engines": { - "node": ">=20" + "node": ">=20.11.0" }, "peerDependencies": { - "@fastly/js-compute": "^2.0.0 || ^3.0.0" + "@fastly/js-compute": "^3.33.2" } }, "node_modules/@bytecodealliance/componentize-js": { @@ -673,10 +674,160 @@ "node": ">=18" } }, + "node_modules/@fastly/cli": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli/-/cli-11.2.0.tgz", + "integrity": "sha512-1J2ksB5FtEqA9JMgJlD4ooMOUTS1zpsONJItSnu0jkA1v8g3n3yngpLMtbjFJ9ZWcuYlrHbjrlZnbQ44FNd6qA==", + "license": "Apache-2.0", + "bin": { + "fastly": "fastly.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@fastly/cli-darwin-arm64": "=11.2.0", + "@fastly/cli-darwin-x64": "=11.2.0", + "@fastly/cli-linux-arm64": "=11.2.0", + "@fastly/cli-linux-x32": "=11.2.0", + "@fastly/cli-linux-x64": "=11.2.0", + "@fastly/cli-win32-arm64": "=11.2.0", + "@fastly/cli-win32-x32": "=11.2.0", + "@fastly/cli-win32-x64": "=11.2.0" + } + }, + "node_modules/@fastly/cli-darwin-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-arm64/-/cli-darwin-arm64-11.2.0.tgz", + "integrity": "sha512-U/AFa0+B5aG1B0ye4F4aEi1xwBwV9iqC/hpXIKNzv9MjXwxhFpAxWEym58RDyQdivr8ID5bOA+v4T4KZscNGsA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "cli-darwin-arm64": "fastly" + } + }, + "node_modules/@fastly/cli-darwin-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-x64/-/cli-darwin-x64-11.2.0.tgz", + "integrity": "sha512-O0nnkO6jZjsfV72Gyr49jKTJ2jKL3gNJCzkq5ImYGX2fLIc+0Fd2BGFi2MUVSZbfktAnogHXPZcy8iCshE2Zgg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "cli-darwin-x64": "fastly" + } + }, + "node_modules/@fastly/cli-linux-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-arm64/-/cli-linux-arm64-11.2.0.tgz", + "integrity": "sha512-ofUn4uqdgbM0gMk6K3eV2PqplNA6jZ13MtZSFh0Bp4DYVunIPqhBbDxu5ndmxRan14KGTt+EH4vNa6trh3Za7w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-arm64": "fastly" + } + }, + "node_modules/@fastly/cli-linux-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x32/-/cli-linux-x32-11.2.0.tgz", + "integrity": "sha512-owq6DVR7snlM0fAOLWrp/FbS7f2/xbwjS3n8Vxy9Kjjjfe1DzEUGBKBzpIBZqqqdZ55CKz6Xf9gC/pjRR6f3DQ==", + "cpu": [ + "x32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-x32": "fastly" + } + }, + "node_modules/@fastly/cli-linux-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x64/-/cli-linux-x64-11.2.0.tgz", + "integrity": "sha512-cdQYWBDuk3fH+7J9LscB3bxWnBn+Iy82DYPdA1NFfDqNTLEjQ+xiePa3EoTv3x6v8sw8b3IjTDiuPmUMBS0w7A==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "cli-linux-x64": "fastly" + } + }, + "node_modules/@fastly/cli-win32-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-arm64/-/cli-win32-arm64-11.2.0.tgz", + "integrity": "sha512-8qOqKpGU6lymV7IXnL3rdItle5MeDq7ntO90JKIx66PxaZIpAznKvqxGQlFBYaj58xjnGBMcRe+Zcxm1j5tItw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-arm64": "fastly.exe" + } + }, + "node_modules/@fastly/cli-win32-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x32/-/cli-win32-x32-11.2.0.tgz", + "integrity": "sha512-OjJaoAkg3rDqTrQKabZ0e1a+l97ELAT4kVCMxDLdd7Y13qWGrKzSx6G3tpYo03Ud9DHV8rnPvfaZtVlTvyHEEw==", + "cpu": [ + "x32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-x32": "fastly.exe" + } + }, + "node_modules/@fastly/cli-win32-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x64/-/cli-win32-x64-11.2.0.tgz", + "integrity": "sha512-3Kh4D4Ii1avHulaJO7qi4Hpapt1GS5JJC+0hX+NsrypqVwuTsevChZlwKLcBYEFKfM5gFvnDsjOE1WMm9APnBA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "cli-win32-x64": "fastly.exe" + } + }, "node_modules/@fastly/js-compute": { - "version": "3.33.1", - "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.1.tgz", - "integrity": "sha512-G93hZpJO5USH3K+URGK/tF2C4OPQq19txPKtemy4OHODBpEQqoqgBspUgO8QI9SXvjrtR/WIJYfTrMaW8HrO+g==", + "version": "3.33.2", + "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.2.tgz", + "integrity": "sha512-CfU/uqyc365+m7O8gAn2p0hxQHTz5uJ6/ifEunrFrVmjgiRv4PssO1X+06JFye1ts8X3OJo8Nm26XhqL3p5XIA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2341,16 +2492,17 @@ "optional": true }, "node_modules/typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/typical": { @@ -2768,10 +2920,73 @@ "dev": true, "optional": true }, + "@fastly/cli": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli/-/cli-11.2.0.tgz", + "integrity": "sha512-1J2ksB5FtEqA9JMgJlD4ooMOUTS1zpsONJItSnu0jkA1v8g3n3yngpLMtbjFJ9ZWcuYlrHbjrlZnbQ44FNd6qA==", + "requires": { + "@fastly/cli-darwin-arm64": "=11.2.0", + "@fastly/cli-darwin-x64": "=11.2.0", + "@fastly/cli-linux-arm64": "=11.2.0", + "@fastly/cli-linux-x32": "=11.2.0", + "@fastly/cli-linux-x64": "=11.2.0", + "@fastly/cli-win32-arm64": "=11.2.0", + "@fastly/cli-win32-x32": "=11.2.0", + "@fastly/cli-win32-x64": "=11.2.0" + } + }, + "@fastly/cli-darwin-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-arm64/-/cli-darwin-arm64-11.2.0.tgz", + "integrity": "sha512-U/AFa0+B5aG1B0ye4F4aEi1xwBwV9iqC/hpXIKNzv9MjXwxhFpAxWEym58RDyQdivr8ID5bOA+v4T4KZscNGsA==", + "optional": true + }, + "@fastly/cli-darwin-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-darwin-x64/-/cli-darwin-x64-11.2.0.tgz", + "integrity": "sha512-O0nnkO6jZjsfV72Gyr49jKTJ2jKL3gNJCzkq5ImYGX2fLIc+0Fd2BGFi2MUVSZbfktAnogHXPZcy8iCshE2Zgg==", + "optional": true + }, + "@fastly/cli-linux-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-arm64/-/cli-linux-arm64-11.2.0.tgz", + "integrity": "sha512-ofUn4uqdgbM0gMk6K3eV2PqplNA6jZ13MtZSFh0Bp4DYVunIPqhBbDxu5ndmxRan14KGTt+EH4vNa6trh3Za7w==", + "optional": true + }, + "@fastly/cli-linux-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x32/-/cli-linux-x32-11.2.0.tgz", + "integrity": "sha512-owq6DVR7snlM0fAOLWrp/FbS7f2/xbwjS3n8Vxy9Kjjjfe1DzEUGBKBzpIBZqqqdZ55CKz6Xf9gC/pjRR6f3DQ==", + "optional": true + }, + "@fastly/cli-linux-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-linux-x64/-/cli-linux-x64-11.2.0.tgz", + "integrity": "sha512-cdQYWBDuk3fH+7J9LscB3bxWnBn+Iy82DYPdA1NFfDqNTLEjQ+xiePa3EoTv3x6v8sw8b3IjTDiuPmUMBS0w7A==", + "optional": true + }, + "@fastly/cli-win32-arm64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-arm64/-/cli-win32-arm64-11.2.0.tgz", + "integrity": "sha512-8qOqKpGU6lymV7IXnL3rdItle5MeDq7ntO90JKIx66PxaZIpAznKvqxGQlFBYaj58xjnGBMcRe+Zcxm1j5tItw==", + "optional": true + }, + "@fastly/cli-win32-x32": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x32/-/cli-win32-x32-11.2.0.tgz", + "integrity": "sha512-OjJaoAkg3rDqTrQKabZ0e1a+l97ELAT4kVCMxDLdd7Y13qWGrKzSx6G3tpYo03Ud9DHV8rnPvfaZtVlTvyHEEw==", + "optional": true + }, + "@fastly/cli-win32-x64": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastly/cli-win32-x64/-/cli-win32-x64-11.2.0.tgz", + "integrity": "sha512-3Kh4D4Ii1avHulaJO7qi4Hpapt1GS5JJC+0hX+NsrypqVwuTsevChZlwKLcBYEFKfM5gFvnDsjOE1WMm9APnBA==", + "optional": true + }, "@fastly/js-compute": { - "version": "3.33.1", - "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.1.tgz", - "integrity": "sha512-G93hZpJO5USH3K+URGK/tF2C4OPQq19txPKtemy4OHODBpEQqoqgBspUgO8QI9SXvjrtR/WIJYfTrMaW8HrO+g==", + "version": "3.33.2", + "resolved": "https://registry.npmjs.org/@fastly/js-compute/-/js-compute-3.33.2.tgz", + "integrity": "sha512-CfU/uqyc365+m7O8gAn2p0hxQHTz5uJ6/ifEunrFrVmjgiRv4PssO1X+06JFye1ts8X3OJo8Nm26XhqL3p5XIA==", "dev": true, "requires": { "@bytecodealliance/jco": "^1.7.0", @@ -3842,9 +4057,9 @@ "optional": true }, "typescript": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", - "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true }, "typical": { diff --git a/package.json b/package.json index 7be042c..ecaf569 100644 --- a/package.json +++ b/package.json @@ -3,13 +3,20 @@ "type": "module", "version": "6.3.0", "description": "Static Publisher for Fastly Compute JavaScript", - "main": "./build/index.js", - "types": "./build/index.d.ts", + "main": "build/index.js", + "exports": { + ".": { + "types": "./build/index.d.ts", + "default": "./build/index.js" + } + }, "scripts": { "clean": "rimraf build", "prepack": "npm run build", - "build": "tsc", - "test": "echo \"Error: no test specified\" && exit 1" + "build": "npm run clean && npm run compile", + "compile": "tsc", + "test": "echo \"Error: no test specified\" && exit 1", + "prepublishOnly": "cp README.short.md README.md" }, "bin": { "compute-js-static-publish": "./build/cli/index.js" @@ -31,22 +38,23 @@ }, "homepage": "https://github.com/fastly/compute-js-static-publish#readme", "dependencies": { + "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", "glob-to-regexp": "^0.4.1" }, "peerDependencies": { - "@fastly/js-compute": "^2.0.0 || ^3.0.0" + "@fastly/js-compute": "^3.33.2" }, "devDependencies": { - "@fastly/js-compute": "^3.0.0", + "@fastly/js-compute": "^3.33.2", "@types/command-line-args": "^5.2.0", "@types/glob-to-regexp": "^0.4.1", - "@types/node": "^20.0.0", + "@types/node": "^20.11.0", "rimraf": "^4.3.0", - "typescript": "^5.0.2" + "typescript": "^5.8.0" }, "engines": { - "node": ">=20" + "node": ">=20.11.0" }, "files": [ "build", diff --git a/resources/README.md b/resources/README.md deleted file mode 100644 index 4a81915..0000000 --- a/resources/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## Resources - -The files in this directory are copied into the output folder. diff --git a/resources/statics-metadata.d.ts b/resources/statics-metadata.d.ts deleted file mode 100644 index f95dd63..0000000 --- a/resources/statics-metadata.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Generated by @fastly/compute-js-static-publish. - */ - -import type { - ContentAssetMetadataMap, -} from "@fastly/compute-js-static-publish"; - -export declare const kvStoreName: string | null; -export declare const contentAssetMetadataMap: ContentAssetMetadataMap; diff --git a/resources/statics.d.ts b/resources/statics.d.ts deleted file mode 100644 index 91658ce..0000000 --- a/resources/statics.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Generated by @fastly/compute-js-static-publish. - */ - -import type { - ModuleAssetMap, - PublisherServerConfigNormalized, - ContentAssets, - ModuleAssets, - PublisherServer, -} from '@fastly/compute-js-static-publish'; - -export declare const moduleAssetMap: ModuleAssetMap; - -export declare const serverConfig: PublisherServerConfigNormalized; -export declare const contentAssets: ContentAssets; -export declare const moduleAssets: ModuleAssets; - -export declare function getServer(): PublisherServer; diff --git a/resources/webpack.config.js b/resources/webpack.config.js deleted file mode 100644 index a7f0737..0000000 --- a/resources/webpack.config.js +++ /dev/null @@ -1,41 +0,0 @@ -const path = require("path"); - -module.exports = { - entry: "./src/index.js", - target: false, - devtool: false, - optimization: { - minimize: true - }, - output: { - filename: "index.js", - path: path.resolve(__dirname, "bin"), - chunkFormat: 'commonjs', - library: { - type: 'commonjs', - }, - }, - module: { - // Loaders go here. - // e.g., ts-loader for TypeScript - // rules: [ - // ], - }, - resolve: { - conditionNames: [ - 'fastly', - '...', - ], - }, - plugins: [ - // Webpack Plugins and Polyfills go here - // e.g., cross-platform WHATWG or core Node.js modules needed for your application. - // new webpack.ProvidePlugin({ - // }), - ], - externals: [ - // Allow webpack to handle 'fastly:*' namespaced module imports by treating - // them as modules rather than trying to process them as URLs - /^fastly:.*$/, - ], -}; diff --git a/src/cli/commands/build-static.ts b/src/cli/commands/build-static.ts deleted file mode 100644 index 4edae95..0000000 --- a/src/cli/commands/build-static.ts +++ /dev/null @@ -1,741 +0,0 @@ -// This program builds static resources out of the files in the -// public folder to be served. -// e.g., with create-react-app, this would be the ./build directory. - -// Outputs: -// -// statics-metadata.js -// -// kvStoreName: string - Name of KV Store, or null if not used. -// contentAssetMetadataMap: Map - mapping of asset keys to their metadata -// - key: string - assetKey, the filename relative to root dir (e.g., /public/foo/index.html) -// - value: ContentAssetMetadataMapEntry - metadata for asset -// -// ContentAssetMetadataMapEntry structure -// - assetKey: string - filename relative to root dir -// - contentType: string - MIME type -// - text: boolean - whether this is a text file -// - lastModifiedTime: number - last modified time as unix time (seconds) -// - fileInfo: FileInfo - information about the file -// - compressedFileInfos: Map - information about any compressed versions -// - key: string - compression algorithm name (e.g., "gzip", "br", etc.) -// - value: FileInfo - information about the compressed version of the file -// - type: string - where the data is available. Usually 'wasm-inline' or 'kv-store'. -// FileInfo structure -// - hash: string - SHA-256 hash of file -// - size: number - file size in bytes -// - staticFilePath: string - path to file in local filesystem, relative to root dir -// - kvStoreKey: string - KV Store key, present only for items in KV Store -// -// statics.js -// imports statics-metadata.js and adds utilities for working with the content assets, as well as -// for loading module assets. Also gives access to StaticPublisher and its config. -// See README.md for details -// -// moduleAssetMap: Map -// - key: string - assetKey, the filename relative to root dir (e.g., /module/hello.js) -// - value: ModuleAssetMapEntry - information about the module -// -// ModuleAssetMapEntry structure -// - isStaticImport: boolean - if true, then uses a static import statement to load the module -// - if false, then module is loaded when getModule is called -// - module: any - the statically loaded module if isStaticImport is true, or null -// - loadModule: function - a function that dynamically loads the module and returns it, if isStaticImport is false, or null -// -// contentAssets: ContentAssets instance -// moduleAssets: ModuleAssets instance -// -// getServer(): function - instantiates PublisherServer singleton and returns it -// serverConfig: PublisherServerConfigNormalized - publisher server config settings -// -// PublisherServerConfigNormalized structure -// - publicDirPrefix: string -// - staticItems: string[] -// - compression: string[] -// - spaFile: string or null -// - notFoundPageFile: string or null -// - autoExt: string[] -// - autoIndex: string[] -// - -import * as fs from "fs"; -import * as path from "path"; -import * as url from "url"; -import commandLineArgs from "command-line-args"; - -import { loadConfigFile } from "../load-config.js"; -import { applyDefaults } from "../util/data.js"; -import { calculateFileSizeAndHash } from "../util/hash.js"; -import { getFiles } from "../util/files.js"; -import { generateOrLoadPublishId } from "../util/publish-id.js"; -import { FastlyApiContext, FetchError, loadApiKey } from "../util/fastly-api.js"; -import { kvStoreEntryExists, kvStoreSubmitFile } from "../util/kv-store.js"; -import { mergeContentTypes, testFileContentType } from "../../util/content-types.js"; -import { algs } from "../compression/index.js"; - -import type { - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, -} from "../../types/content-assets.js"; -import type { - ContentTypeDef, - ContentTypeTestResult, -} from "../../types/content-types.js"; -import type { - ContentAssetInclusionResultNormalized, - ModuleAssetInclusionResultNormalized, - PublisherServerConfigNormalized, -} from "../../types/config-normalized.js"; -import type { - ContentCompressionTypes, -} from "../../constants/compression.js"; -import type { - CompressedFileInfos, - ContentFileInfoForWasmInline, - ContentFileInfoForKVStore, -} from "../../types/content-assets.js"; -import { - attemptWithRetries, -} from "../util/retryable.js"; - -type AssetInfo = - ContentTypeTestResult & - ContentAssetInclusionResultNormalized & - ModuleAssetInclusionResultNormalized & - { - // Full path to file on local filesystem - file: string, - - // Asset key (relative to public dir) - assetKey: string, - - // Last modified time - lastModifiedTime: number, - - // Hash (to be used as etag and as part of file id) - hash: string, - - // Size of file - size: number, - }; - -type KVStoreItemDesc = { - kvStoreKey: string, - staticFilePath: string, - text: boolean, -}; - -async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { - - const maxConcurrent = 12; - let index = 0; // Shared among workers - - async function worker() { - while (index < kvStoreItems.length) { - const currentIndex = index; - index = index + 1; - const { kvStoreKey, staticFilePath, text } = kvStoreItems[currentIndex]; - - try { - await attemptWithRetries( - async() => { - if (await kvStoreEntryExists(fastlyApiContext, kvStoreName, kvStoreKey)) { - console.log(`✔️ Asset already exists in KV Store with key "${kvStoreKey}".`); - return; - } - const fileData = fs.readFileSync(staticFilePath); - await kvStoreSubmitFile(fastlyApiContext, kvStoreName, kvStoreKey, fileData); - console.log(`✔️ Submitted ${text ? 'text' : 'binary'} asset "${staticFilePath}" to KV Store with key "${kvStoreKey}".`) - }, - { - onAttempt(attempt) { - if (attempt > 0) { - console.log(`Attempt ${attempt + 1} for: ${kvStoreKey}`); - } - }, - onRetry(attempt, err, delay) { - let statusMessage = 'unknown'; - if (err instanceof FetchError) { - statusMessage = `HTTP ${err.status}`; - } else if (err instanceof TypeError) { - statusMessage = 'transport'; - } - console.log(`Attempt ${attempt + 1} for ${kvStoreKey} gave retryable error (${statusMessage}), delaying ${delay} ms`); - }, - } - ); - } catch (err) { - const e = err instanceof Error ? err : new Error(String(err)); - console.error(`❌ Failed: ${kvStoreKey} → ${e.message}`); - console.error(e.stack); - } - } - } - - const workers = Array.from({ length: maxConcurrent }, () => worker()); - await Promise.all(workers); -} - -function writeKVStoreEntriesToFastlyToml(kvStoreName: string, kvStoreItems: KVStoreItemDesc[]) { - - let fastlyToml = fs.readFileSync('./fastly.toml', 'utf-8'); - - let before: string = ''; - let after: string = ''; - - kvStoreName = kvStoreName.includes(".") ? `"${kvStoreName}"` : kvStoreName; - - const tableMarker = `[[local_server.kv_stores.${kvStoreName}]]`; - - const startPos = fastlyToml.indexOf(tableMarker); - if (startPos === -1) { - - // KV Store decl not in fastly.toml yet - - if (fastlyToml.indexOf(kvStoreName) !== -1) { - // don't do this! - console.error(`improperly configured entry for '${kvStoreName}' in fastly.toml`); - // TODO: handle thrown exception from callers - throw "No"! - } - - let newLines; - if (fastlyToml.endsWith('\n\n')) { - newLines = ''; - } else if (fastlyToml.endsWith('\n')) { - newLines = '\n' - } else { - newLines = '\n\n'; - } - - before = fastlyToml + newLines; - after = ''; - - } else { - - const lastObjStoreTablePos = fastlyToml.lastIndexOf(tableMarker); - const nextTablePos = fastlyToml.indexOf('[', lastObjStoreTablePos + tableMarker.length); - - before = fastlyToml.slice(0, startPos); - - if (nextTablePos === -1) { - - after = ''; - - } else { - - after = fastlyToml.slice(nextTablePos); - - } - - } - - let tablesToml = ''; - - for (const {kvStoreKey, staticFilePath} of kvStoreItems) { - // Probably, JSON.stringify is wrong, but it should do its job - tablesToml += tableMarker + '\n'; - tablesToml += 'key = ' + JSON.stringify(kvStoreKey) + '\n'; - tablesToml += 'path = ' + JSON.stringify(path.relative('./', staticFilePath)) + '\n'; - tablesToml += '\n'; - } - - fastlyToml = before + tablesToml + after; - - fs.writeFileSync('./fastly.toml', fastlyToml, 'utf-8'); -} - -export async function buildStaticLoader(commandLineValues: commandLineArgs.CommandLineOptions) { - - const { 'suppress-framework-warnings': suppressFrameworkWarnings } = commandLineValues; - const displayFrameworkWarnings = !suppressFrameworkWarnings; - - const { publishId, created } = generateOrLoadPublishId(); - - if (created) { - console.log("✅ Created publish ID"); - } - - console.log(`Publish ID: ${publishId}`); - - console.log("🚀 Building loader..."); - - const errors: string[] = []; - const { normalized: config, raw: configRaw } = await loadConfigFile(errors); - - if (config == null) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - for (const error of errors) { - console.error(error); - } - process.exitCode = 1; - return; - } - - const outputDir = path.resolve(); - const publicDirRoot = path.resolve(config.rootDir); - - console.log(`✔️ Public directory '${publicDirRoot}'.`); - - const kvStoreName = config.kvStoreName; - let fastlyApiContext: FastlyApiContext | null = null; - if (kvStoreName != null) { - // TODO: load api key from command line - const apiKeyResult = loadApiKey(); - if (apiKeyResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); - process.exitCode = 1; - return; - } - fastlyApiContext = { apiToken: apiKeyResult.apiToken }; - console.log(`✔️ Using KV Store mode, with KV Store: ${kvStoreName}`); - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiKeyResult.source}'`); - } else { - if (displayFrameworkWarnings) { - console.log(`✔️ Not using KV Store mode.`); - } - } - - const excludeDirs = config.excludeDirs; - if (excludeDirs.length > 0) { - console.log(`✔️ Using exclude directories: ${excludeDirs.join(', ')}`); - } else { - if (displayFrameworkWarnings) { - console.log(`✔️ No exclude directories defined.`); - } - } - - const excludeDotFiles = config.excludeDotFiles; - if (excludeDotFiles) { - console.log(`✔️ Files/Directories starting with . are excluded.`); - } - - const includeWellKnown = config.includeWellKnown; - if (includeWellKnown) { - console.log(`✔️ (.well-known is exempt from exclusion.)`); - } - - const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); - - // Load content types - const finalContentTypes: ContentTypeDef[] = mergeContentTypes(config.contentTypes); - - // getFiles() applies excludeDirs, excludeDotFiles, and includeWellKnown. - let files = getFiles(publicDirRoot, { ...config, publicDirRoot }); - - // If public dir is not inside the c@e dir, then - // exclude files that come from C@E app dir - if (!publicDirRoot.startsWith(outputDir)) { - files = files.filter(file => !file.startsWith(outputDir)); - } - - // And then we apply assetInclusionTest. - const assetInfos: AssetInfo[] = files.map(file => { - const assetKey = file.slice(publicDirRoot.length) - // in Windows, assetKey will otherwise end up as \path\file.html - .replace(/\\/g, '/'); - - let contentTypeTestResult = testFileContentType(finalContentTypes, assetKey); - - if (contentTypeTestResult == null) { - contentTypeTestResult = { - text: false, - contentType: 'application/octet-stream', - precompressAsset: false, - }; - if (displayFrameworkWarnings) { - console.log('⚠️ Notice: Unknown file type ' + assetKey + '. Treating as binary file.'); - } - } - - let contentAssetInclusionResultValue = config.contentAssetInclusionTest?.(assetKey, contentTypeTestResult.contentType); - if (typeof contentAssetInclusionResultValue === 'boolean') { - contentAssetInclusionResultValue = { - includeContent: contentAssetInclusionResultValue, - }; - } else if (contentAssetInclusionResultValue === 'inline') { - contentAssetInclusionResultValue = { - includeContent: true, - inline: true, - }; - } - const contentAssetInclusionResult = applyDefaults(contentAssetInclusionResultValue ?? null, { - includeContent: true, - inline: false, - }); - - let moduleAssetInclusionResultValue = config.moduleAssetInclusionTest?.(assetKey, contentTypeTestResult.contentType); - if (typeof moduleAssetInclusionResultValue === 'boolean') { - moduleAssetInclusionResultValue = { - includeModule: moduleAssetInclusionResultValue, - }; - } else if (moduleAssetInclusionResultValue === 'static-import') { - moduleAssetInclusionResultValue = { - includeModule: true, - useStaticImport: true, - }; - } - - const moduleAssetInclusionResult = applyDefaults(moduleAssetInclusionResultValue ?? null, { - includeModule: false, - useStaticImport: false, - }); - - const stats = fs.statSync(file); - const lastModifiedTime = Math.floor((stats.mtime).getTime() / 1000); - - const { size, hash } = calculateFileSizeAndHash(file); - - return { - file, - assetKey, - lastModifiedTime, - hash, - size, - ...contentAssetInclusionResult, - ...moduleAssetInclusionResult, - ...contentTypeTestResult, - }; - - }); - - console.log("🚀 Preparing content assets ..."); - - if (kvStoreName == null && configRaw != null && !("contentCompression" in configRaw)) { - console.log(`⚠️ Notice: By default, pre-compressed content assets are not generated when KV Store is not used.`); - console.log(" If you want to pre-compress assets, add a value for 'contentCompression' to your static-publish.rc.js."); - } - - // The Static Content Root Dir, which will hold the loaders and a copy of static files. - const staticContentRootDir = config.staticContentRootDir; - - // Create "static content" dir that will be used to hold a copy of static files. - // NOTE: This copy is created so that they can be loaded with includeBytes(), which requires - // all files to be under the Compute project dir. - const staticContentDir = `${staticContentRootDir}/static-content`; - fs.rmSync(staticContentDir, { recursive: true, force: true }); - fs.mkdirSync(staticContentDir, { recursive: true }); - - // Build content assets metadata - const contentAssetMetadataMap: ContentAssetMetadataMap = {}; - - // KV Store items to upload - const kvStoreItems: KVStoreItemDesc[] = []; - - let contentItems = 0; - const counts = { - inline: 0, - kvStore: 0, - excluded: 0, - } - for (const assetInfo of assetInfos) { - if (!assetInfo.includeContent) { - // Non-content asset - counts.excluded++; - continue; - } - contentItems++; - console.log(`✔️ [${contentItems}] ${assetInfo.assetKey}: ${JSON.stringify(assetInfo.contentType)}`); - - const { - file, - assetKey, - contentType, - text, - precompressAsset, - hash, - size, - lastModifiedTime, - } = assetInfo; - - const entryBase = { - assetKey, - contentType, - lastModifiedTime, - text, - fileInfo: { - hash, - size, - }, - }; - - let metadata: ContentAssetMetadataMapEntry; - - const isInline = kvStoreName == null || assetInfo.inline; - - const staticContentFilePath = `${staticContentDir}/file${contentItems}.${assetInfo.text ? 'txt' : 'bin'}`; - - type PrepareCompressionVersionFunc = (alg: ContentCompressionTypes, staticFilePath: string, hash: string, size: number) => void; - async function prepareCompressedVersions(contentCompressions: ContentCompressionTypes[], func: PrepareCompressionVersionFunc) { - for (const alg of contentCompressions) { - const compressTo = algs[alg]; - if (precompressAsset && compressTo != null) { - - const staticFilePath = `${staticContentFilePath}_${alg}`; - if (await compressTo(file, staticFilePath, text)) { - console.log(`✔️ Compressed ${text ? 'text' : 'binary'} asset "${file}" to "${staticFilePath}" [${alg}].`) - const { size, hash } = calculateFileSizeAndHash(staticFilePath); - func(alg, staticFilePath, hash, size); - } - - } - } - } - - const contentCompression = config.contentCompression; - - if (isInline) { - // We will inline this file using includeBytes() - // so we copy it to the static-content directory. - const staticFilePath = staticContentFilePath; - fs.cpSync(file, staticFilePath); - console.log(`✔️ Copied ${text ? 'text' : 'binary'} asset "${file}" to "${staticFilePath}".`); - - // 'prepareCompressedVersions' uses the static content directory for compressed copies, - // even for 'inline' files. - const compressedFileInfos: CompressedFileInfos = {}; - await prepareCompressedVersions(contentCompression, (alg, staticFilePath, hash, size) => { - compressedFileInfos[alg] = { staticFilePath, hash, size }; - }); - - metadata = { - ...entryBase, - type: 'wasm-inline', - fileInfo: { - ...entryBase.fileInfo, - staticFilePath, - }, - compressedFileInfos, - }; - - counts.inline++; - } else { - // For KV Store mode, we don't need to make a copy of the original file - const staticFilePath = file; - - // Use the hash as part of the key name. This avoids having to - // re-upload a file if it already exists. - const kvStoreKey = `${publishId}:${assetKey}_${hash}`; - - kvStoreItems.push({ - kvStoreKey, - staticFilePath, - text, - }); - - const compressedFileInfos: CompressedFileInfos = {}; - await prepareCompressedVersions(contentCompression, (alg, staticFilePath, hash, size) => { - const kvStoreKey = `${publishId}:${assetKey}_${alg}_${hash}`; - compressedFileInfos[alg] = { staticFilePath, kvStoreKey, hash, size }; - kvStoreItems.push({ - kvStoreKey, - staticFilePath, - text, - }); - }); - - metadata = { - ...entryBase, - type: 'kv-store', - fileInfo: { - ...entryBase.fileInfo, - staticFilePath, - kvStoreKey, - }, - compressedFileInfos, - }; - - counts.kvStore++; - } - - contentAssetMetadataMap[assetKey] = metadata; - } - - if (kvStoreName != null) { - await uploadFilesToKVStore(fastlyApiContext!, kvStoreName, kvStoreItems); - writeKVStoreEntriesToFastlyToml(kvStoreName, kvStoreItems); - } - - console.log("✅ Prepared " + (counts.inline + counts.kvStore) + " content asset(s):"); - if (counts.inline > 0) { - console.log(" " + counts.inline + " inline"); - } - if (counts.kvStore > 0) { - console.log(" " + counts.kvStore + " KV Store"); - } - - // Build statics-metadata.js - let metadataFileContents = `/* - * Generated by @fastly/compute-js-static-publish. - */ - -`; - metadataFileContents += `\nexport const kvStoreName = ${JSON.stringify(kvStoreName)};\n`; - metadataFileContents += `\nexport const contentAssetMetadataMap = {\n`; - for (const [key, value] of Object.entries(contentAssetMetadataMap)) { - metadataFileContents += ` ${JSON.stringify(key)}: ${JSON.stringify(value)},\n`; - } - metadataFileContents += '};\n'; - fs.writeFileSync(`${staticContentRootDir}/statics-metadata.js`, metadataFileContents); - - console.log(`✅ Wrote static file metadata for ${contentItems} file(s).`); - - // Copy Types file for static file loader - try { - const typesFile = path.resolve(__dirname, '../../../resources/statics-metadata.d.ts'); - fs.copyFileSync(typesFile, `${staticContentRootDir}/statics-metadata.d.ts`); - - console.log("✅ Wrote content assets metadata types file statics-metadata.d.ts."); - } catch { - console.log("⚠️ Notice: could not write content assets metadata types file statics-metadata.d.ts."); - } - - // Build statics.js - let fileContents = `/* - * Generated by @fastly/compute-js-static-publish. - */ - -`; - - fileContents += 'import { ContentAssets, ModuleAssets, PublisherServer } from "@fastly/compute-js-static-publish";\n\n'; - fileContents += 'import "@fastly/compute-js-static-publish/build/compute-js.js";\n\n'; - fileContents += 'import { kvStoreName, contentAssetMetadataMap } from "./statics-metadata.js";\n'; - - // Add import statements for assets that are modules and that need to be statically imported. - const staticImportModuleNumbers: Record = {}; - let staticImportModuleNumber = 0; - for (const assetInfo of assetInfos) { - if (assetInfo.includeModule && assetInfo.useStaticImport) { - staticImportModuleNumber++; - staticImportModuleNumbers[assetInfo.assetKey] = staticImportModuleNumber; - - // static-content lives in the staticContentRootDir dir, so the import must be declared as - // relative to that file. - const relativeFilePath = path.relative(staticContentRootDir, assetInfo.file); - fileContents += `import * as fileModule${staticImportModuleNumber} from "${relativeFilePath}";\n`; - } - } - - fileContents += `\nexport const moduleAssetsMap = {\n`; - - for (const assetInfo of assetInfos) { - if (!assetInfo.includeModule) { - continue; - } - - let module; - let loadModuleFunction; - - if (assetInfo.useStaticImport) { - const moduleNumber = staticImportModuleNumbers[assetInfo.assetKey]; - if (moduleNumber == null) { - throw new Error(`Unexpected! module asset number for "${assetInfo.assetKey}" was not found!`); - } - loadModuleFunction = `() => Promise.resolve(fileModule${moduleNumber})`; - module = `fileModule${moduleNumber}`; - } else { - // static-content lives in the staticContentRootDir dir, so the import must be declared as - // relative to that file. - const relativeFilePath = path.relative(staticContentRootDir, assetInfo.file); - loadModuleFunction = `() => import("${relativeFilePath}")`; - module = 'null'; - } - - fileContents += ` ${JSON.stringify(assetInfo.assetKey)}: { isStaticImport: ${JSON.stringify(assetInfo.useStaticImport)}, module: ${module}, loadModule: ${loadModuleFunction} },\n`; - } - - fileContents += '};\n'; - - const server = applyDefaults(config.server, { - publicDirPrefix: '', - staticItems: [], - compression: [ 'br', 'gzip' ], - spaFile: null, - notFoundPageFile: null, - autoExt: [], - autoIndex: [], - }); - - let publicDirPrefix = server.publicDirPrefix; - console.log(`✔️ Server public dir prefix '${publicDirPrefix}'.`); - - let staticItems = server.staticItems; - - let compression = server.compression; - - let spaFile = server.spaFile; - if(spaFile != null) { - console.log(`✔️ Application SPA file '${spaFile}'.`); - const spaAsset = assetInfos.find(assetInfo => assetInfo.assetKey === spaFile); - if(spaAsset == null || spaAsset.contentType !== 'text/html') { - if (displayFrameworkWarnings) { - console.log(`⚠️ Notice: '${spaFile}' does not exist or is not of type 'text/html'. Ignoring.`); - } - spaFile = null; - } - } else { - if (displayFrameworkWarnings) { - console.log(`✔️ Application is not a SPA.`); - } - } - - let notFoundPageFile = server.notFoundPageFile; - if(notFoundPageFile != null) { - console.log(`✔️ Application 'not found (404)' file '${notFoundPageFile}'.`); - const notFoundPageAsset = assetInfos.find(assetInfo => assetInfo.assetKey === notFoundPageFile); - - if(notFoundPageAsset == null || notFoundPageAsset.contentType !== 'text/html') { - if (displayFrameworkWarnings) { - console.log(`⚠️ Notice: '${notFoundPageFile}' does not exist or is not of type 'text/html'. Ignoring.`); - } - notFoundPageFile = null; - } - } else { - if (displayFrameworkWarnings) { - console.log(`✔️ Application specifies no 'not found (404)' page.`); - } - } - - let autoIndex: string[] = server.autoIndex; - let autoExt: string[] = server.autoExt; - - const serverConfig: PublisherServerConfigNormalized = { - publicDirPrefix, - staticItems, - compression, - spaFile, - notFoundPageFile, - autoExt, - autoIndex, - }; - - fileContents += `\nexport const serverConfig = ${JSON.stringify(serverConfig, null, 2)};\n`; - - fileContents += '\nexport const contentAssets = new ContentAssets(contentAssetMetadataMap, {kvStoreName});'; - fileContents += '\nexport const moduleAssets = new ModuleAssets(moduleAssetsMap);\n'; - - fileContents += '\nlet server = null;'; - fileContents += '\nexport function getServer() {' + - '\n if (server == null) {' + - '\n server = new PublisherServer(serverConfig, contentAssets);' + - '\n }' + - '\n return server;' + - '\n}\n'; - - fs.writeFileSync(`${staticContentRootDir}/statics.js`, fileContents); - - console.log("✅ Wrote static file loader for " + files.length + " file(s)."); - - // Copy Types file for static file loader - try { - const staticsTypeFile = path.resolve(__dirname, '../../../resources/statics.d.ts'); - fs.copyFileSync(staticsTypeFile, `${staticContentRootDir}/statics.d.ts`); - - console.log(`✅ Wrote static file loader types file ${staticContentRootDir}/statics.d.ts.`); - } catch { - console.log(`⚠️ Notice: could not write static file loader types file ${staticContentRootDir}/statics.d.ts.`); - } - -} diff --git a/src/cli/commands/clean-kv-store.ts b/src/cli/commands/clean-kv-store.ts deleted file mode 100644 index 79c56b3..0000000 --- a/src/cli/commands/clean-kv-store.ts +++ /dev/null @@ -1,82 +0,0 @@ -import path from 'path'; -import type { CommandLineOptions } from "command-line-args"; - -import { generateOrLoadPublishId } from "../util/publish-id.js"; -import { loadConfigFile } from "../load-config.js"; -import { getKVStoreKeys, kvStoreDeleteFile } from "../util/kv-store.js"; -import { FastlyApiContext, loadApiKey } from "../util/fastly-api.js"; -import { getKVStoreKeysFromMetadata } from "../../util/metadata.js"; -import { ContentAssetMetadataMap } from "../../types/index.js"; - -type StaticsMetadataModule = { - kvStoreName: string | null, - contentAssetMetadataMap: ContentAssetMetadataMap, -}; - -export async function cleanKVStore(commandLineValues: CommandLineOptions) { - - const { publishId } = generateOrLoadPublishId(); - - const errors: string[] = []; - const { normalized: config } = await loadConfigFile(errors); - - if (config == null) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - for (const error of errors) { - console.error(error); - } - process.exitCode = 1; - return; - } - - let fastlyApiContext: FastlyApiContext | null = null; - - const apiKeyResult = loadApiKey(); - if (apiKeyResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Specify one on the command line, or use the FASTLY_API_TOKEN environment variable."); - process.exitCode = 1; - return; - } - - fastlyApiContext = { apiToken: apiKeyResult.apiToken }; - - const staticContentRootDir = config.staticContentRootDir; - - const staticsMetadata: StaticsMetadataModule = await import(path.resolve(`${staticContentRootDir}/statics-metadata.js`)); - - const { kvStoreName, contentAssetMetadataMap } = staticsMetadata; - - if (kvStoreName == null) { - console.error("❌ KV Store not specified."); - console.error("This only has meaning in KV Store mode."); - process.exitCode = 1; - return; - } - - // TODO: Enable getting kvStoreName and publishId from command line - - // These are the items that are currently in the KV Store and that belong to this publish ID. - const items = ((await getKVStoreKeys(fastlyApiContext, kvStoreName)) ?? []) - .filter(x => x.startsWith(`${publishId}:`)); - - // These are the items that are currently are being used. - const keys = getKVStoreKeysFromMetadata(contentAssetMetadataMap); - - // So these are the items that we should be deleting. - const itemsToDelete = items.filter(x => !keys.has(x)); - - console.log("Publish ID: " + publishId); - console.log("KV Store contains " + items.length + " item(s) for this publish ID."); - console.log("Current site metadata contains " + keys.size + " item(s) (including compressed alternates)."); - - console.log("Number of items to delete: " + itemsToDelete.length); - - for (const [index, item] of itemsToDelete.entries()) { - console.log("Deleting item [" + (index+1) + "]: " + item); - await kvStoreDeleteFile(fastlyApiContext, kvStoreName, item); - } - - console.log("✅ Completed.") -} diff --git a/src/cli/commands/clean.ts b/src/cli/commands/clean.ts new file mode 100644 index 0000000..daf2a00 --- /dev/null +++ b/src/cli/commands/clean.ts @@ -0,0 +1,161 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import commandLineArgs, { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap } from '../../models/assets/kvstore-assets.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../util/config.js'; +import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteFile } from '../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../fastly-api/api-token.js'; + +export async function action(argv: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const { + verbose, + ['fastly-api-token']: fastlyApiToken, + } = commandLineValues; + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(`✔️ Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(`✔️ Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + + // ### KVStore Keys to delete + const kvKeysToDelete = new Set(); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + const indexKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + indexesPrefix, + ); + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); + console.log('Found collections'); + for (const collection of foundCollections) { + console.log(collection); + } + + // TODO ### Determine which ones are not expired, based on an expiration meta + const liveCollections = foundCollections; + + // ### List all settings ### + const settingsPrefix = publishId + '_index_'; + const settingsKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + settingsPrefix, + ); + if (settingsKeys == null) { + throw new Error(`Can't query settings in KV Store`); + } + + // If a settings object is found that doesn't match a live collection name then + // mark it for deletion + const foundSettingsCollections = settingsKeys.map(key => ({ key, name: key.slice(settingsPrefix.length) })); + for (const foundSettings of foundSettingsCollections) { + if (!foundCollections.includes(foundSettings.name)) { + kvKeysToDelete.add(foundSettings.key); + } + } + + // ### Go through the index files and make a list of all keys (hashes) that we are keeping + const assetsIdsInUse = new Set(); + for (const collection of liveCollections) { + + // TODO deal with when the index file is > 20MB + const kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext, + kvStoreName, + indexesPrefix + collection + ); + if (!kvAssetsIndexResponse) { + throw new Error(`Can't load KV Store entry ${indexesPrefix + collection}`); + } + const kvAssetsIndex = (await kvAssetsIndexResponse.response.json()) as KVAssetEntryMap; + for (const [_assetKey, assetEntry] of Object.entries(kvAssetsIndex)) { + if (assetEntry.key.startsWith('sha256:')) { + assetsIdsInUse.add(`sha256_${assetEntry.key.slice('sha256:'.length)}`); + } + } + } + + // ### Obtain the assets in the KV Store and find the ones that are not in use + const assetPrefix = publishId + '_files_'; + const assetKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + assetPrefix, + ); + if (assetKeys == null) { + throw new Error(`Can't query assets in KV Store`); + } + + for (const assetKey of assetKeys) { + let assetId = assetKey.slice(assetPrefix.length); + if (assetId.startsWith('sha256_')) { + assetId = assetId.slice(0, 'sha256_'.length + 64); + } else { + // If we don't know what the prefix is, we ignore it + continue; + } + + if (assetsIdsInUse.has(assetId)) { + console.log('Asset ID ' + assetId + ' in use'); + } else { + kvKeysToDelete.add(assetKey); + console.log('Asset ID ' + assetId + ' not in use'); + } + } + + // ### Delete items that have been flagged + for (const key of kvKeysToDelete) { + console.log("Deleting item: " + key); + await kvStoreDeleteFile(fastlyApiContext, kvStoreName, key); + } + + console.log("✅ Completed.") +} diff --git a/src/cli/commands/collections/delete.ts b/src/cli/commands/collections/delete.ts new file mode 100644 index 0000000..4dfb61f --- /dev/null +++ b/src/cli/commands/collections/delete.ts @@ -0,0 +1,111 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import commandLineArgs, { type OptionDefinition } from 'command-line-args'; + +import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; +import { getKVStoreKeys, kvStoreDeleteFile } from '../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; + +export async function action(argv: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + { name: 'collection-name', type: String, multiple: true } + ]; + + const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const { + verbose, + ['fastly-api-token']: fastlyApiToken, + ['collection-name']: collectionNameValue, + } = commandLineValues; + + const collectionNamesAsArray = (Array.isArray(collectionNameValue) ? collectionNameValue : [ collectionNameValue ]) + .filter(x => typeof x === 'string') + .map(x => x.split(',')) + .flat() + .map(x => x.trim()) + .filter(Boolean); + if (collectionNamesAsArray.length === 0) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(`✔️ Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(`✔️ Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + + console.log(`✔️ Collections to delete: ${collectionNamesAsArray.map(x => `'${x}'`).join(', ')}`) + const collectionNames = new Set(collectionNamesAsArray); + + const kvKeysToDelete = new Set(); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + const indexKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + indexesPrefix, + ); + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(key => ({ key, name: key.slice(indexesPrefix.length), })); + if (foundCollections.length === 0) { + console.log('No collections found.'); + } else { + console.log(`Found collections: ${foundCollections.map(x => `'${x.name}'`).join(', ')}`); + for (const collection of foundCollections) { + if (collectionNames.has(collection.name)) { + console.log(`Flagging collection '${collection.name}' for deletion: ${collection.key}`); + kvKeysToDelete.add(collection.key); + } + } + } + + // ### Delete items that have been flagged + for (const key of kvKeysToDelete) { + console.log("Deleting key from KV Store: " + key); + await kvStoreDeleteFile(fastlyApiContext, kvStoreName, key); + } + + console.log("✅ Completed.") +} diff --git a/src/cli/commands/collections/index.ts b/src/cli/commands/collections/index.ts new file mode 100644 index 0000000..ef4b81f --- /dev/null +++ b/src/cli/commands/collections/index.ts @@ -0,0 +1,66 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { getCommandAndArgs } from '../../util/args.js'; +import * as collectionsDelete from './delete.js'; +import * as collectionsList from './list.js'; + +export async function action(actionArgs: string[]) { + + const modes = { + 'delete': collectionsDelete, + 'list': collectionsList, + }; + + const commandAndArgs = getCommandAndArgs( + actionArgs, + Object.keys(modes) as (keyof typeof modes)[], + ); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.command != null) { + console.error(`Unknown subcommand: ${commandAndArgs.command}`); + console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); + +} + +function help() { + + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections [options] + +Description: + Manage named collections within a Compute application built with @fastly/compute-js-static-publish. + +Available Subcommands: + list List all published collections + delete Delete a specific collection index + update-expiration Modify expiration time for an existing collection + +Global Options: + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command or any subcommand + +Examples: + npx @fastly/compute-js-static-publish collections list + npx @fastly/compute-js-static-publish collections delete --collection-name=preview-42 +`); + +} diff --git a/src/cli/commands/collections/list.ts b/src/cli/commands/collections/list.ts new file mode 100644 index 0000000..7d32a63 --- /dev/null +++ b/src/cli/commands/collections/list.ts @@ -0,0 +1,80 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import commandLineArgs, { type OptionDefinition } from 'command-line-args'; + +import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; +import { getKVStoreKeys } from '../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; + +export async function action(argv: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const { + verbose, + ['fastly-api-token']: fastlyApiToken, + } = commandLineValues; + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(`✔️ Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(`✔️ Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + const indexKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + indexesPrefix, + ); + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(key => ({ key, name: key.slice(indexesPrefix.length), })); + if (foundCollections.length === 0) { + console.log('No collections found.'); + } else { + console.log(`Found collections: ${foundCollections.map(x => `'${x.name}'`).join(', ')}`); + } + + console.log("✅ Completed.") +} diff --git a/src/cli/commands/index.ts b/src/cli/commands/index.ts new file mode 100644 index 0000000..793e48f --- /dev/null +++ b/src/cli/commands/index.ts @@ -0,0 +1,85 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; + +import { getCommandAndArgs } from '../util/args.js'; +import * as initApp from './init-app.js'; +import * as publishContent from './publish-content.js'; +import * as collections from './collections/index.js'; + +export async function action(actionArgs: string[]) { + + if (!fs.existsSync('./static-publish.rc.js')) { + + await initApp.action(actionArgs); + return; + + } + + const modes = { + 'publish-content': publishContent, + 'collections': collections, + }; + + const commandAndArgs = getCommandAndArgs( + actionArgs, + Object.keys(modes) as (keyof typeof modes)[], + ); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.command != null) { + console.error(`Unknown command: ${commandAndArgs.command}`); + console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); + +} + +function help() { + + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish [options] + +Description: + Manage and publish static content to Fastly Compute using KV Store-backed collections. + If run outside a scaffolded project, this tool will automatically enter project initialization mode. + +Available Commands: + publish-content Publish static files to the KV Store under a named collection + clean Remove expired collections and unused KV Store content + collections list List all published collections + collections delete Delete a specific collection index + collections promote Copy a collection to another name + collections update-expiration Modify expiration time for an existing collection + +Global Options: + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command or any subcommand + +Automatic Project Initialization: + If run in a directory that does not contain a \`static-publish.rc.js\` file, this tool will scaffold a new + Compute application for you, including Fastly configuration, default routes, and publishing setup. + +Examples: + npx @fastly/compute-js-static-publish publish-content --collection-name live + npx @fastly/compute-js-static-publish collections list + npx @fastly/compute-js-static-publish clean --dry-run +`); + +} diff --git a/src/cli/commands/init-app.ts b/src/cli/commands/init-app.ts index 35a0025..a553b98 100644 --- a/src/cli/commands/init-app.ts +++ b/src/cli/commands/init-app.ts @@ -1,23 +1,45 @@ -import { CommandLineOptions } from "command-line-args"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ // This program creates a Fastly Compute JavaScript application // in a subfolder named compute-js. // This project can be served using fastly compute serve // or deployed to a Compute service using fastly compute publish. +import * as child_process from 'node:child_process'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +import commandLineArgs, { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; + +import { dotRelative, rootRelative } from '../util/files.js'; +import { findComputeJsStaticPublisherVersion, type PackageJson } from '../util/package.js'; + +export type InitAppOptions = { + outputDir: string | undefined, + rootDir: string | undefined, + publicDir: string | undefined, + staticDirs: string[], + staticPublisherWorkingDir: string | undefined, + spa: string | undefined, + notFoundPage: string | undefined, + autoIndex: string[], + autoExt: string[], + name: string | undefined, + author: string | undefined, + description: string | undefined, + serviceId: string | undefined, + publishId: string | undefined, + kvStoreName: string | undefined, +}; -import * as child_process from "child_process"; -import * as path from "path"; -import * as fs from "fs"; -import * as url from 'url'; - -import { AppOptions, IPresetBase } from '../presets/preset-base.js'; -import { presets } from '../presets/index.js'; - -const defaultOptions: AppOptions = { +const defaultOptions: InitAppOptions = { + outputDir: undefined, rootDir: undefined, publicDir: undefined, staticDirs: [], - staticContentRootDir: undefined, + staticPublisherWorkingDir: undefined, spa: undefined, notFoundPage: '[public-dir]/404.html', autoIndex: [ 'index.html', 'index.htm' ], @@ -26,79 +48,91 @@ const defaultOptions: AppOptions = { name: 'compute-js-static-site', description: 'Fastly Compute static site', serviceId: undefined, + publishId: undefined, kvStoreName: undefined, }; -// Current directory of this program that's running. -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); +function buildOptions( + packageJson: PackageJson | null, + commandLineValues: CommandLineOptions, +): InitAppOptions { -function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial { + // Applied in this order for proper overriding + // 1. defaults + // 2. package.json + // 3. command-line args - // All paths are relative to CWD. + const options = structuredClone(defaultOptions); + + if (packageJson?.name !== undefined) { + options.name = packageJson!.name; + } + if (packageJson?.author !== undefined) { + options.author = packageJson!.author; + } + if (packageJson?.description !== undefined) { + options.description = packageJson!.description; + } - let preset: string | undefined; { - const presetValue = commandLineValues['preset']; - if (presetValue == null || typeof presetValue === 'string') { - preset = presetValue; + let outputDir: string | undefined; + const outputDirValue = commandLineValues['output']; + if (outputDirValue == null || typeof outputDirValue === 'string') { + outputDir = outputDirValue; + } + if (outputDir !== undefined) { + options.outputDir = outputDir; } } - let rootDir: string | undefined; - let publicDir: string | undefined; { + let rootDir: string | undefined; const rootDirValue = commandLineValues['root-dir']; if (rootDirValue == null || typeof rootDirValue === 'string') { rootDir = rootDirValue; } - if (rootDir != null) { - rootDir = path.resolve(rootDir); + if (rootDir !== undefined) { + options.rootDir = rootDir; } + } + { + let publicDir: string | undefined; const publicDirValue = commandLineValues['public-dir']; if (publicDirValue == null || typeof publicDirValue === 'string') { publicDir = publicDirValue; } - if (publicDir != null) { - publicDir = path.resolve(publicDir); + if (publicDir !== undefined) { + options.publicDir = publicDir; } - - // If we don't have a preset, then for backwards compatibility - // we check if we have public-dir but no root-dir. If that is - // the case then we use the value public-dir as root-dir. - if (preset == null && rootDir == null && publicDir != null) { - rootDir = publicDir; - publicDir = undefined; - } - } - // Filepaths provided on the command line are always given relative to CWD, - // so we need to resolve them. - - let staticDirs: string[] | undefined; { + let staticDirs: string[] | undefined; const staticDirsValue = commandLineValues['static-dir']; const asArray = Array.isArray(staticDirsValue) ? staticDirsValue : [ staticDirsValue ]; if (asArray.every((x: any) => typeof x === 'string')) { - staticDirs = (asArray as string[]).map(x => path.resolve(x)); + staticDirs = asArray; + } + if (staticDirs !== undefined) { + options.staticDirs = staticDirs; } } - let staticContentRootDir: string | undefined; { - const staticContentRootDirValue = commandLineValues['static-content-root-dir']; - if (staticContentRootDirValue == null || typeof staticContentRootDirValue === 'string') { - staticContentRootDir = staticContentRootDirValue; + let staticPublisherWorkingDir: string | undefined; + const staticPublisherWorkingDirValue = commandLineValues['static-publisher-working-dir']; + if (staticPublisherWorkingDirValue == null || typeof staticPublisherWorkingDirValue === 'string') { + staticPublisherWorkingDir = staticPublisherWorkingDirValue; } - if (staticContentRootDir != null) { - staticContentRootDir = path.resolve(staticContentRootDir); + if (staticPublisherWorkingDir !== undefined) { + options.staticPublisherWorkingDir = staticPublisherWorkingDir; } } - let spa: string | undefined; { + let spa: string | undefined; const spaValue = commandLineValues['spa']; if (spaValue === null) { // If 'spa' is provided with a null value, then the flag was provided @@ -108,10 +142,13 @@ function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial< } else if (spaValue == null || typeof spaValue === 'string') { spa = spaValue; } + if (spa !== undefined) { + options.spa = spa; + } } - let notFoundPage: string | undefined; { + let notFoundPage: string | undefined; const notFoundPageValue = commandLineValues['not-found-page']; if (notFoundPageValue === null) { // If 'spa' is provided with a null value, then the flag was provided @@ -121,10 +158,13 @@ function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial< } else if (notFoundPageValue == null || typeof notFoundPageValue === 'string') { notFoundPage = notFoundPageValue; } + if (notFoundPage !== undefined) { + options.notFoundPage = notFoundPage; + } } - let autoIndex: string[] | undefined; { + let autoIndex: string[] | undefined; const autoIndexValue = commandLineValues['auto-index']; const asArray = Array.isArray(autoIndexValue) ? autoIndexValue : [ autoIndexValue ]; @@ -145,10 +185,13 @@ function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial< }, []); } + if (autoIndex !== undefined) { + options.autoIndex = autoIndex; + } } - let autoExt: string[] = []; { + let autoExt: string[] = []; const autoExtValue = commandLineValues['auto-ext']; const asArray = Array.isArray(autoExtValue) ? autoExtValue : [ autoExtValue ]; @@ -170,182 +213,187 @@ function processCommandLineArgs(commandLineValues: CommandLineOptions): Partial< }, []); } + if (autoExt !== undefined) { + options.autoExt = autoExt; + } } - let name: string | undefined; { + let name: string | undefined; const nameValue = commandLineValues['name']; if (nameValue == null || typeof nameValue === 'string') { name = nameValue; } + if (name !== undefined) { + options.name = name; + } } - let author: string | undefined; { + let author: string | undefined; const authorValue = commandLineValues['author']; if (authorValue == null || typeof authorValue === 'string') { author = authorValue; } + if (author !== undefined) { + options.author = author; + } } - let description: string | undefined; { + let description: string | undefined; const descriptionValue = commandLineValues['description']; if (descriptionValue == null || typeof descriptionValue === 'string') { description = descriptionValue; } + if (description !== undefined) { + options.description = description; + } } - let serviceId: string | undefined; { + let serviceId: string | undefined; const serviceIdValue = commandLineValues['service-id']; if (serviceIdValue == null || typeof serviceIdValue === 'string') { serviceId = serviceIdValue; } + if (serviceId !== undefined) { + options.serviceId = serviceId; + } } - let kvStoreName: string | undefined; { + let publishId: string | undefined; + const publishIdValue = commandLineValues['publish-id']; + if (publishIdValue == null || typeof publishIdValue === 'string') { + publishId = publishIdValue; + } + if (publishId !== undefined) { + options.publishId = publishId; + } + } + + { + let kvStoreName: string | undefined; const kvStoreNameValue = commandLineValues['kv-store-name']; if (kvStoreNameValue == null || typeof kvStoreNameValue === 'string') { kvStoreName = kvStoreNameValue; } - } - - return { - rootDir, - publicDir, - staticDirs, - staticContentRootDir, - spa, - notFoundPage, - autoIndex, - autoExt, - name, - author, - description, - serviceId, - kvStoreName, - }; - -} - -function pickKeys>(keys: (keyof TModel)[], object: TModel): Partial { - - const result: Partial = {}; - - for (const key of keys) { - if(object[key] !== undefined) { - result[key] = object[key]; + if (kvStoreName !== undefined) { + options.kvStoreName = kvStoreName; } } - return result; + return options; } const PUBLIC_DIR_TOKEN = '[public-dir]'; function processPublicDirToken(filepath: string, publicDir: string) { if (!filepath.startsWith(PUBLIC_DIR_TOKEN)) { - return filepath; + return path.resolve(filepath); } const processedPath = '.' + filepath.slice(PUBLIC_DIR_TOKEN.length); - const resolvedPath = path.resolve(publicDir, processedPath) - return path.relative(path.resolve(), resolvedPath); + return path.resolve(publicDir, processedPath) } -export function initApp(commandLineValues: CommandLineOptions) { +export async function action(argv: string[]) { - let options: AppOptions = defaultOptions; - let preset: IPresetBase | null = null; + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, - const presetName = (commandLineValues['preset'] as string | null) ?? 'none'; - if(presetName !== 'none') { - const presetClass = presets[presetName]; - if(presetClass == null) { - console.error('Unknown preset name.'); - console.error("--preset must be one of: none, " + (Object.keys(presets).join(', '))); - process.exitCode = 1; - return; - } - preset = new presetClass(); - } + // Required. The name of a Fastly KV Store to hold the content assets. + // It is also added to the fastly.toml that is generated. + { name: 'kv-store-name', type: String, }, - let packageJson; - try { - const packageJsonText = fs.readFileSync("./package.json", "utf-8"); - packageJson = JSON.parse(packageJsonText); - } catch { - console.log("Can't read/parse package.json in current directory, making no assumptions!"); - packageJson = null; - } + // Output directory. "Required" (if not specified, then defaultValue is used). + { name: 'output', alias: 'o', type: String, defaultValue: './compute-js', }, - // Get the current compute js static publisher version. - let computeJsStaticPublisherVersion: string | null = null; - if (packageJson != null) { - // First try current project's package.json - computeJsStaticPublisherVersion = - packageJson.dependencies?.["@fastly/compute-js-static-publish"] ?? - packageJson.devDependencies?.["@fastly/compute-js-static-publish"]; + // Name of the application, to be inserted into fastly.toml + { name: 'name', type: String, }, - // This may be a file url if during development - if (computeJsStaticPublisherVersion != null) { + // Description of the application, to be inserted into fastly.toml + { name: 'description', type: String, }, - if (computeJsStaticPublisherVersion.startsWith('file:')) { - // this is a relative path from the current directory. - // we replace it with an absolute path - const relPath = computeJsStaticPublisherVersion.slice('file:'.length); - const absPath = path.resolve(relPath); - computeJsStaticPublisherVersion = 'file:' + absPath; - } + // Name of the author, to be inserted into fastly.toml + { name: 'author', type: String, }, - } - } + // Fastly Service ID to be added to the fastly.toml that is generated. + { name: 'service-id', type: String }, - if (computeJsStaticPublisherVersion == null) { - // Also try package.json of the package that contains the currently running program - // This is used when the program doesn't actually install the package (running via npx). - const computeJsStaticPublishPackageJsonPath = path.resolve(__dirname, '../../../package.json'); - const computeJsStaticPublishPackageJsonText = fs.readFileSync(computeJsStaticPublishPackageJsonPath, 'utf-8'); - const computeJsStaticPublishPackageJson = JSON.parse(computeJsStaticPublishPackageJsonText); - computeJsStaticPublisherVersion = computeJsStaticPublishPackageJson?.version; - } + // Required. The 'root' directory for the publishing. + // All assets are expected to exist under this root. Required. + // For backwards compatibility, if this value is not provided, + // then the value of 'public-dir' is used. + { name: 'root-dir', type: String, }, - if (computeJsStaticPublisherVersion == null) { - // Unexpected, but if it's still null then we go to a literal - computeJsStaticPublisherVersion = '^4.0.0'; - } + // Publish ID to be used as a prefix for all KV Store entries. + // If not provided, the default value of 'default' is used. + { name: 'publish-id', type: String, }, - if (!computeJsStaticPublisherVersion.startsWith('^') && - !computeJsStaticPublisherVersion.startsWith('file:') - ) { - computeJsStaticPublisherVersion = '^' + computeJsStaticPublisherVersion; - } + // The 'static publisher working directory' is the directory under the Compute + // application where asset files are written in preparation for upload to the + // KV Store and for serving for local mode. + { name: 'static-publisher-working-dir', type: String, }, - const commandLineAppOptions = processCommandLineArgs(commandLineValues); + // The 'public' directory. The Publisher Server will + // resolve requests relative to this directory. If not specified, + // defaults to the same value as 'root-dir'. See README for + // details. + { name: 'public-dir', type: String, }, - type PackageJsonAppOptions = Pick; + // Directories to specify as containing 'static' files. The + // Publisher Server will serve files from these directories + // with a long TTL. + { name: 'static-dir', type: String, multiple: true, }, - options = { - ...options, - ...(preset != null ? preset.defaultOptions : {}), - ...pickKeys(['author', 'name', 'description'], (packageJson ?? {}) as PackageJsonAppOptions), - ...pickKeys(['rootDir', 'publicDir', 'staticDirs', 'staticContentRootDir', 'spa', 'notFoundPage', 'autoIndex', 'autoExt', 'author', 'name', 'description', 'serviceId', 'kvStoreName'], commandLineAppOptions), - }; + // Path to a file to be used to serve in a SPA application. + // The Publisher Server will serve this file with a 200 status code + // when the request doesn't match a known file, and the accept + // header includes text/html. You may use the '[public-dir]' token + // if you wish to specify this as a relative path from the 'public-dir'. + { name: 'spa', type: String, }, - if(preset != null) { - if(!preset.check(packageJson, options)) { - console.log("Failed preset check."); - process.exitCode = 1; - return; - } + // Path to a file to be used to serve as a 404 not found page. + // The Publisher Server will serve this file with a 404 status code + // when the request doesn't match a known file, and the accept + // header includes text/html. You may use the '[public-dir]' token + // if you wish to specify this as a relative path from the 'public-dir'. + { name: 'not-found-page', type: String, }, + + // List of files to automatically use as index, for example, index.html,index.htm + // If a request comes in but the route does not exist, we check the route + // plus a slash plus the items in this array. + { name: 'auto-index', type: String, multiple: true, }, + + // List of extensions to apply to a path name, for example, if + // http://example.com/about is requested, we can respond with http://example.com/about.html + { name: 'auto-ext', type: String, multiple: true, }, + ]; + + const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + + let packageJson; + try { + const packageJsonText = fs.readFileSync("./package.json", "utf-8"); + packageJson = JSON.parse(packageJsonText) as PackageJson; + } catch { + console.log("Can't read/parse package.json in current directory, making no assumptions!"); + packageJson = null; } - // Webpack now optional as of v4 - const useWebpack = commandLineValues['webpack'] as boolean; + const options = buildOptions( + packageJson, + commandLineValues, + ); - const COMPUTE_JS_DIR = commandLineValues.output as string; + const COMPUTE_JS_DIR = options.outputDir; + if (COMPUTE_JS_DIR == null) { + console.error("❌ required parameter --output not provided."); + process.exitCode = 1; + return; + } const computeJsDir = path.resolve(COMPUTE_JS_DIR); // Resolve the root dir, relative to current directory, and make sure it exists. @@ -358,7 +406,7 @@ export function initApp(commandLineValues: CommandLineOptions) { const rootDir = path.resolve(ROOT_DIR); if (!fs.existsSync(rootDir) || !fs.statSync(rootDir).isDirectory()) { console.error(`❌ Specified root directory '${ROOT_DIR}' does not exist.`); - console.error(` * ${rootDir} must exist and be a directory.`); + console.error(` * ${rootRelative(rootDir)} must exist and be a directory.`); process.exitCode = 1; return; } @@ -368,7 +416,7 @@ export function initApp(commandLineValues: CommandLineOptions) { const publicDir = path.resolve(PUBLIC_DIR); if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) { console.error(`❌ Specified public directory '${PUBLIC_DIR}' does not exist.`); - console.error(` * ${publicDir} must exist and be a directory.`); + console.error(` * ${rootRelative(publicDir)} must exist and be a directory.`); process.exitCode = 1; return; } @@ -376,7 +424,7 @@ export function initApp(commandLineValues: CommandLineOptions) { // Public dir must be inside the root dir. if (!publicDir.startsWith(rootDir)) { console.error(`❌ Specified public directory '${PUBLIC_DIR}' is not under the asset root directory.`); - console.error(` * ${publicDir} must be under ${rootDir}`); + console.error(` * ${rootRelative(publicDir)} must be under ${rootRelative(rootDir)}`); process.exitCode = 1; return; } @@ -386,38 +434,38 @@ export function initApp(commandLineValues: CommandLineOptions) { const staticDirs: string[] = []; for (const STATIC_DIR of STATIC_DIRS) { // For backwards compatibility, these values can start with [public-dir] - const staticDir = path.resolve(processPublicDirToken(STATIC_DIR, publicDir)); + const staticDir = processPublicDirToken(STATIC_DIR, publicDir); if (!staticDir.startsWith(publicDir)) { console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); - console.log(` * ${staticDir} is not under ${publicDir}`); + console.log(` * ${rootRelative(staticDir)} is not under ${rootRelative(publicDir)}`); continue; } if (!fs.existsSync(staticDir) || !fs.statSync(staticDir).isDirectory()) { console.log(`⚠️ Ignoring static directory '${STATIC_DIR}'`); - console.log(` * ${staticDir} does not exist or is not a directory.`); + console.log(` * ${rootRelative(staticDir)} does not exist or is not a directory.`); continue; } staticDirs.push(staticDir); } - // Static Content Root Dir must be under the current dir - let staticContentRootDir = options.staticContentRootDir; - if (staticContentRootDir == null) { - staticContentRootDir = path.resolve(computeJsDir, './static-publisher'); - } + // Static Publisher Working Root Dir must be under the current dir + // This comes in relative to cwd. + const STATIC_PUBLISHER_WORKING_DIR = options.staticPublisherWorkingDir ?? + path.resolve(computeJsDir, './static-publisher'); + const staticPublisherWorkingDir = path.resolve(STATIC_PUBLISHER_WORKING_DIR); if ( - !staticContentRootDir.startsWith(computeJsDir) || - staticContentRootDir === computeJsDir || - staticContentRootDir === path.resolve(computeJsDir, './bin') || - staticContentRootDir === path.resolve(computeJsDir, './pkg') || - staticContentRootDir === path.resolve(computeJsDir, './node_modules') + !staticPublisherWorkingDir.startsWith(computeJsDir) || + staticPublisherWorkingDir === computeJsDir || + staticPublisherWorkingDir === path.resolve(computeJsDir, './bin') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './pkg') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './src') || + staticPublisherWorkingDir === path.resolve(computeJsDir, './node_modules') ) { - console.error(`❌ Specified static content root directory '${staticContentRootDir}' must be under ${computeJsDir}.`); - console.error(` It also must not be bin, pkg, or node_modules.`); + console.error(`❌ Specified static publisher working directory '${rootRelative(staticPublisherWorkingDir)}' must be under ${rootRelative(computeJsDir)}.`); + console.error(` It also must not be bin, pkg, src, or node_modules.`); process.exitCode = 1; return; } - staticContentRootDir = './' + path.relative(computeJsDir, staticContentRootDir); // SPA and Not Found are relative to the asset root dir. @@ -425,16 +473,15 @@ export function initApp(commandLineValues: CommandLineOptions) { let spaFilename: string | undefined; if (SPA != null) { // If it starts with [public-dir], then resolve it relative to public directory. - spaFilename = path.resolve(processPublicDirToken(SPA, publicDir)); + spaFilename = processPublicDirToken(SPA, publicDir); // At any rate it must exist under the root directory if (!spaFilename.startsWith(rootDir)) { console.log(`⚠️ Ignoring specified SPA file '${SPA}' as is not under the asset root directory.`); - console.log(` * ${spaFilename} is not under ${rootDir}`); + console.log(` * ${rootRelative(spaFilename)} is not under ${rootRelative(rootDir)}`); spaFilename = undefined; } else if (!fs.existsSync(spaFilename)) { - console.log(`⚠️ Ignoring specified SPA file '${SPA}' as it does not exist.`); - console.log(` * ${spaFilename} does not exist.`); - spaFilename = undefined; + console.log(`⚠️ Warning: Ignoring specified SPA file '${SPA}' does not exist.`); + console.log(` * ${rootRelative(spaFilename)} does not exist.`); } } @@ -442,16 +489,15 @@ export function initApp(commandLineValues: CommandLineOptions) { let notFoundPageFilename: string | undefined; if (NOT_FOUND_PAGE != null) { // If it starts with [public-dir], then resolve it relative to public directory. - notFoundPageFilename = path.resolve(processPublicDirToken(NOT_FOUND_PAGE, publicDir)); + notFoundPageFilename = processPublicDirToken(NOT_FOUND_PAGE, publicDir); // At any rate it must exist under the root directory if (!notFoundPageFilename.startsWith(rootDir)) { console.log(`⚠️ Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as is not under the asset root directory.`); - console.log(` * ${notFoundPageFilename} is not under ${rootDir}`); + console.log(` * ${rootRelative(notFoundPageFilename)} is not under ${rootRelative(rootDir)}`); notFoundPageFilename = undefined; } else if (!fs.existsSync(notFoundPageFilename)) { - console.log(`⚠️ Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as it does not exist.`); - console.log(` * ${notFoundPageFilename} does not exist.`); - notFoundPageFilename = undefined; + console.log(`⚠️ Warning: Ignoring specified Not Found file '${NOT_FOUND_PAGE}' as it does not exist.`); + console.log(` * ${rootRelative(notFoundPageFilename)} does not exist.`); } } @@ -462,8 +508,9 @@ export function initApp(commandLineValues: CommandLineOptions) { if(exists) { console.error(`❌ '${COMPUTE_JS_DIR}' directory already exists!`); console.error(` You should not run this command if this directory exists.`); - console.error(` If you need to re-scaffold the static publisher, delete the following directory and then try again:`); - console.error(` ${computeJsDir}`); + console.error(` If you need to re-scaffold the static publisher Compute App,`); + console.error(` delete the following directory and then try again:`); + console.error(` ${rootRelative(computeJsDir)}`); process.exitCode = 1; return; } @@ -473,130 +520,131 @@ export function initApp(commandLineValues: CommandLineOptions) { const description = options.description; const fastlyServiceId = options.serviceId; const kvStoreName = options.kvStoreName; - - function rootRelative(itemPath: string | null | undefined) { - if (itemPath == null) { - return null; - } - const v = path.relative(path.resolve(), itemPath); - return v.startsWith('..') ? v : './' + v; + if (kvStoreName == null) { + console.error(`❌ required parameter --kv-store-name not provided.`); + process.exitCode = 1; + return; } + let publishId = options.publishId; + if (publishId == null) { + publishId = 'default'; + } + + const defaultCollectionName = 'live'; console.log(''); - console.log('Asset Root Dir :', rootRelative(rootDir)); - console.log('Public Dir :', rootRelative(publicDir)); - console.log('Static Dir :', staticDirs.length > 0 ? staticDirs.map(rootRelative) : '(None)'); - console.log('Static Content Root Dir :', staticContentRootDir); - console.log('SPA :', rootRelative(spaFilename) ?? '(None)'); - console.log('404 Page :', rootRelative(notFoundPageFilename) ?? '(None)'); - console.log('Auto-Index :', autoIndex.length > 0 ? autoIndex.map(rootRelative) : '(None)') - console.log('Auto-Ext :', autoExt.length > 0 ? autoExt.map(rootRelative) : '(None)') - console.log('name :', name); - console.log('author :', author); - console.log('description :', description); - console.log('Service ID :', fastlyServiceId ?? '(None)'); - console.log('KV Store Name :', kvStoreName ?? '(None)'); + console.log('Compute Application Settings'); + console.log('----------------------------'); + console.log('Compute Application Output Dir :', rootRelative(computeJsDir)); + console.log('name :', name); + console.log('author :', author); + console.log('description :', description); + console.log('Service ID :', fastlyServiceId ?? '(None)'); + console.log('KV Store Name :', kvStoreName); + console.log('Default Collection Name :', defaultCollectionName); + console.log('Publish ID :', publishId); + + console.log(''); + console.log('Publish Settings'); + console.log('----------------'); + console.log('Publish Root Dir :', rootRelative(rootDir)); + console.log('Publisher Working Dir :', rootRelative(staticPublisherWorkingDir)); + + console.log(''); + console.log('Publisher Server Settings'); + console.log('-------------------------'); + console.log('Server Public Dir :', rootRelative(publicDir)); + console.log('SPA :', spaFilename != null ? rootRelative(spaFilename) : '(None)'); + console.log('404 Page :', notFoundPageFilename != null ? rootRelative(notFoundPageFilename) : '(None)'); + console.log('Auto-Index :', autoIndex.length > 0 ? autoIndex : '(None)') + console.log('Auto-Ext :', autoExt.length > 0 ? autoExt : '(None)') + console.log('Static Files Dir :', staticDirs.length > 0 ? staticDirs.map(rootRelative) : '(None)'); + console.log(''); - if (useWebpack) { - console.log('Creating project with webpack.'); - console.log(''); - } console.log("Initializing Compute Application in " + computeJsDir + "..."); fs.mkdirSync(computeJsDir); fs.mkdirSync(path.resolve(computeJsDir, './src')); - const staticContentRootDirFromRoot = staticContentRootDir.slice(1); - - let staticFiles = staticContentRootDirFromRoot; - if (staticContentRootDirFromRoot === '/src') { - staticFiles = `\ -${staticContentRootDirFromRoot}/statics.js -${staticContentRootDirFromRoot}/statics.d.ts -${staticContentRootDirFromRoot}/statics-metadata.js -${staticContentRootDirFromRoot}/statics-metadata.d.ts -${staticContentRootDirFromRoot}/static-content`; - } + const resourceFiles: Record = {}; // .gitignore - const gitIgnoreContent = `\ + resourceFiles['.gitignore'] = `\ /node_modules /bin /pkg -${staticFiles} +/${path.relative(computeJsDir, staticPublisherWorkingDir)} `; - const gitIgnorePath = path.resolve(computeJsDir, '.gitignore'); - fs.writeFileSync(gitIgnorePath, gitIgnoreContent, "utf-8"); // package.json - const packageJsonContent: Record = { + const computeJsStaticPublisherVersion = findComputeJsStaticPublisherVersion(packageJson); + resourceFiles['package.json'] = JSON.stringify({ name, version: '0.1.0', description, author, type: 'module', devDependencies: { - "@fastly/cli": "^10.14.0", + "@fastly/cli": "^11.2.0", '@fastly/compute-js-static-publish': computeJsStaticPublisherVersion, }, dependencies: { - '@fastly/js-compute': '^3.0.0', + '@fastly/js-compute': '^3.26.0', }, engines: { - node: '>=20.0.0', + node: '>=20.11.0', }, - license: 'UNLICENSED', private: true, scripts: { + prestart: "npx @fastly/compute-js-static-publish publish-content --local-only", start: "fastly compute serve", deploy: "fastly compute publish", - prebuild: 'npx @fastly/compute-js-static-publish --build-static', + "publish-content": 'npx @fastly/compute-js-static-publish publish-content', build: 'js-compute-runtime ./src/index.js ./bin/main.wasm' }, - }; - - if (useWebpack) { - delete packageJsonContent.type; - packageJsonContent.devDependencies = { - ...packageJsonContent.devDependencies, - 'webpack': '^5.75.0', - 'webpack-cli': '^5.0.0', - }; - packageJsonContent.scripts = { - ...packageJsonContent.scripts, - prebuild: 'npx @fastly/compute-js-static-publish --build-static && webpack', - build: 'js-compute-runtime ./bin/index.js ./bin/main.wasm' - }; - } - - const packageJsonContentJson = JSON.stringify(packageJsonContent, undefined, 2); - const packageJsonPath = path.resolve(computeJsDir, 'package.json'); - fs.writeFileSync(packageJsonPath, packageJsonContentJson, "utf-8"); + }, undefined, 2); // fastly.toml - - // language=toml - const fastlyTomlContent = `\ + const localServerKvStorePath = dotRelative(computeJsDir, path.resolve(staticPublisherWorkingDir, 'kvstore.json')); + resourceFiles['fastly.toml'] = /* language=text */ `\ # This file describes a Fastly Compute package. To learn more visit: # https://developer.fastly.com/reference/fastly-toml/ authors = [ "${author}" ] description = "${description}" language = "javascript" -manifest_version = 2 +manifest_version = 3 name = "${name}" -${fastlyServiceId != null ? `service_id = "${fastlyServiceId}" -` : ''} +${fastlyServiceId != null ? `service_id = "${fastlyServiceId}"\n` : ''} [scripts] - build = "npm run build" - `; +build = "npm run build" + +[local_server.kv_stores] +${kvStoreName} = { file = "${localServerKvStorePath}", format = "json" } + +[setup.kv_stores.${kvStoreName}] +`; - const fastlyTomlPath = path.resolve(computeJsDir, 'fastly.toml'); - fs.writeFileSync(fastlyTomlPath, fastlyTomlContent, "utf-8"); + // kvstore.json + resourceFiles[localServerKvStorePath] = '{}'; // static-publish.rc.js - const rootDirRel = path.relative(computeJsDir, rootDir); + resourceFiles['static-publish.rc.js'] = `\ +/* + * Generated by @fastly/compute-js-static-publish. + */ +/** @type {import('@fastly/compute-js-static-publish').StaticPublishRc} */ +const rc = { + kvStoreName: ${JSON.stringify(kvStoreName)}, + publishId: ${JSON.stringify(publishId)}, + defaultCollectionName: ${JSON.stringify(defaultCollectionName)}, +}; + +export default rc; +`; + + // publish-content.config.js // publicDirPrefix -- if public dir is deeper than the root dir, then // public dir is used as a prefix to drill into asset names. // e.g., @@ -637,33 +685,32 @@ ${fastlyServiceId != null ? `service_id = "${fastlyServiceId}" notFoundPageFile = notFoundPageFilename.slice(rootDir.length); } - const staticPublishJsContent = `\ + resourceFiles['publish-content.config.js'] = `\ /* - * Copyright Fastly, Inc. - * Licensed under the MIT license. See LICENSE file for details. + * Generated by @fastly/compute-js-static-publish. */ // Commented items are defaults, feel free to modify and experiment! // See README for a detailed explanation of the configuration options. -/** @type {import('@fastly/compute-js-static-publish').StaticPublisherConfig} */ +/** @type {import('@fastly/compute-js-static-publish').PublishContentConfig} */ const config = { - rootDir: ${JSON.stringify(rootDirRel)}, - staticContentRootDir: ${JSON.stringify(staticContentRootDir)}, - ${(kvStoreName != null ? 'kvStoreName: ' + JSON.stringify(kvStoreName) : '// kvStoreName: false')}, + rootDir: ${JSON.stringify(dotRelative(computeJsDir, rootDir))}, + staticPublisherWorkingDir: ${JSON.stringify(dotRelative(computeJsDir, staticPublisherWorkingDir))}, // excludeDirs: [ './node_modules' ], // excludeDotFiles: true, // includeWellKnown: true, - // contentAssetInclusionTest: (filename) => true, - // contentCompression: [ 'br', 'gzip' ], // For this config value, default is [] if kvStoreName is null. - // moduleAssetInclusionTest: (filename) => false, + // kvStoreAssetInclusionTest: (assetKey) => true, + // contentCompression: [ 'br', 'gzip' ], // contentTypes: [ // { test: /.custom$/, contentType: 'application/x-custom', text: false }, // ], + + // Server settings are saved to the KV Store per collection server: { publicDirPrefix: ${JSON.stringify(publicDirPrefix)}, staticItems: ${JSON.stringify(staticItems)}, - // compression: [ 'br', 'gzip' ], + allowedEncodings: [ 'br', 'gzip' ], spaFile: ${JSON.stringify(spaFile)}, notFoundPageFile: ${JSON.stringify(notFoundPageFile)}, autoExt: ${JSON.stringify(autoExt)}, @@ -671,35 +718,21 @@ const config = { }, }; -${useWebpack ? 'module.exports =' : 'export default'} config; +export default config; `; - const staticPublishJsonPath = path.resolve(computeJsDir, 'static-publish.rc.js'); - fs.writeFileSync(staticPublishJsonPath, staticPublishJsContent, "utf-8"); - - // Copy resource files - - if (useWebpack) { - // webpack.config.js - const webpackConfigJsSrcPath = path.resolve(__dirname, '../../../resources/webpack.config.js'); - const webpackConfigJsPath = path.resolve(computeJsDir, 'webpack.config.js'); - fs.copyFileSync(webpackConfigJsSrcPath, webpackConfigJsPath); - } - // src/index.js - const staticsRelativePath = staticContentRootDir !== './src' ? - path.relative('./src', staticContentRootDir) : - '.'; - const indexJsContent = /* language=JavaScript */ `\ + resourceFiles['./src/index.js'] = /* language=text */ `\ /// -import { getServer } from '${staticsRelativePath}/statics.js'; -const staticContentServer = getServer(); +import { PublisherServer } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; +const publisherServer = PublisherServer.fromStaticPublishRc(rc); // eslint-disable-next-line no-restricted-globals addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); async function handleRequest(event) { - const response = await staticContentServer.serveRequest(event.request); + const response = await publisherServer.serveRequest(event.request); if (response != null) { return response; } @@ -710,8 +743,13 @@ async function handleRequest(event) { return new Response('Not found', { status: 404 }); } `; - const indexJsPath = path.resolve(computeJsDir, './src/index.js'); - fs.writeFileSync(indexJsPath, indexJsContent, 'utf-8'); + + // Write out the files + for (const [filename, content] of Object.entries(resourceFiles)) { + const filePath = path.resolve(computeJsDir, filename); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, content, 'utf-8'); + } console.log("🚀 Compute application created!"); @@ -731,5 +769,4 @@ async function handleRequest(event) { console.log(' cd ' + COMPUTE_JS_DIR); console.log(' npm run deploy'); console.log(''); - } diff --git a/src/cli/commands/publish-content.ts b/src/cli/commands/publish-content.ts new file mode 100644 index 0000000..78e9cf8 --- /dev/null +++ b/src/cli/commands/publish-content.ts @@ -0,0 +1,585 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import commandLineArgs, { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap, type KVAssetVariantMetadata, isKVAssetVariantMetadata } from '../../models/assets/kvstore-assets.js'; +import { type ContentCompressionTypes } from '../../models/compression/index.js'; +import { type PublisherServerConfigNormalized } from '../../models/config/publisher-server-config.js'; +import { type ContentTypeDef } from '../../models/config/publish-content-config.js'; +import { type FastlyApiContext, FetchError, loadApiToken } from '../fastly-api/api-token.js'; +import { getKvStoreEntryInfo } from '../fastly-api/kv-store.js'; +import { mergeContentTypes, testFileContentType } from '../util/content-types.js'; +import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../util/config.js'; +import { applyDefaults } from '../util/data.js'; +import { calculateFileSizeAndHash, enumerateFiles, getFileSize, rootRelative } from '../util/files.js'; +import { applyKVStoreEntriesChunks, type KVStoreItemDesc, uploadFilesToKVStore } from '../util/kv-store-items.js'; +import { writeKVStoreEntriesForLocal } from '../util/kv-store-local-server.js'; +import { attemptWithRetries } from '../util/retryable.js'; +import { ensureVariantFileExists, type Variants } from '../util/variants.js'; + +// KV Store key format: +// _index_.json +// _settings_.json +// _files_sha256__ + +// split large files into 20MiB chunks +const KV_STORE_CHUNK_SIZE = 1_024 * 1_024 * 20; + +export async function action(argv: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean, }, + + // Fastly API Token to use for this publishing. + { name: 'fastly-api-token', type: String, }, + + // Collection name to be used for this publishing. + { name: 'collection-name', type: String, }, + + // The 'root' directory for the publishing. + // All assets are expected to exist under this root. Required. + // For backwards compatibility, if this value is not provided, + // then the value of 'public-dir' is used. + { name: 'root-dir', type: String, }, + + { name: 'force-upload', type: Boolean }, + + { name: 'no-local', type: Boolean }, + + { name: 'local-only', type: Boolean }, + + { name: 'config', type: String }, + ]; + + const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + + const { + verbose, + ['fastly-api-token']: fastlyApiToken, + ['collection-name']: collectionNameValue, + ['root-dir']: rootDir, + ['force-upload']: forceUpload, + ['no-local']: noLocalMode, + ['local-only']: localOnlyMode, + ['config']: configFilePathValue, + } = commandLineValues; + + // no-local and local-only are mutually exclusive + if (noLocalMode && localOnlyMode) { + console.error("❌ '--no-local' and '--local-only' are mutually exclusive."); + process.exitCode = 1; + return; + } + + // Create local files unless 'no-local' is set + const createLocalFiles = !noLocalMode; + // Use the KV Store unless 'local-only' is set + const useKvStore = !localOnlyMode; + + const segments: string[] = []; + if (createLocalFiles) { + segments.push('for local simluated KV Store'); + } + if (useKvStore) { + segments.push('to the Fastly KV Store'); + } + + console.log(`🚀 Publishing content ${segments.join(' and ')}...`); + + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (useKvStore) { + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + } + + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const configFilePath = configFilePathValue ?? './publish-content.config.js'; + + let publishContentConfig; + try { + publishContentConfig = await loadPublishContentConfigFile(configFilePath); + } catch (err) { + console.error(`❌ Can't load ${configFilePath}`); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publicDirRoot = path.resolve(rootDir != null ? rootDir : publishContentConfig.rootDir); + if ((computeAppDir + '/').startsWith(publicDirRoot + '/')) { + if (verbose) { + console.log(`‼️ Public directory '${rootRelative(publicDirRoot)}' includes the Compute app directory.`); + console.log(`This may be caused by an incorrect configuration, as this could cause Compute application source`); + console.log(`files to be included in your published output.`); + } + } + + console.log(`✔️ Public directory '${rootRelative(publicDirRoot)}'.`); + + const publishId = staticPublisherRc.publishId; + console.log(`✔️ Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(`✔️ Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + + // The Static Content Root Dir, which will hold loaders and content generated by this publishing. + const staticPublisherWorkingDir = publishContentConfig.staticPublisherWorkingDir; + + // Load content types + const contentTypes: ContentTypeDef[] = mergeContentTypes(publishContentConfig.contentTypes); + + // #### Collect all file paths + console.log(`🔍 Scanning root directory ${rootRelative(publicDirRoot)}...`); + + const excludeDirs = publishContentConfig.excludeDirs; + if (excludeDirs.length > 0) { + console.log(`✔️ Using exclude directories: ${excludeDirs.join(', ')}`); + } else { + if (verbose) { + console.log(`✔️ No exclude directories defined.`); + } + } + + const excludeDotFiles = publishContentConfig.excludeDotFiles; + if (excludeDotFiles) { + console.log(`✔️ Files/Directories starting with . are excluded.`); + } + + const includeWellKnown = publishContentConfig.includeWellKnown; + if (includeWellKnown) { + console.log(`✔️ (.well-known is exempt from exclusion.)`); + } + + // files to be included in the build/publish + const files = enumerateFiles({ + publicDirRoot, + excludeDirs, + excludeDotFiles, + includeWellKnown, + }); + + // ### Collection name that we're currently publishing + // for example, live, staging + const collectionName = collectionNameValue ?? process.env.PUBLISHER_COLLECTION_NAME ?? defaultCollectionName; + + const stats = { + kvStore: 0, + }; + + // Create "KV Store content" sub dir if it doesn't exist already. + // This will be used to hold a copy of files to prepare for upload to the KV Store + // and for serving using the local development server. + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + // A list of items in the KV Store at the end of the publishing. + // Includes items that already exist as well. 'write' signifies + // that the item is to be written + const kvStoreItemDescriptions: KVStoreItemDesc[] = []; + + // Assets included in the publishing, keyed by asset key + const kvAssetsIndex: KVAssetEntryMap = {}; + + // All the metadata of the variants we know about during this publishing, keyed on the base version's hash. + type VariantMetadataEntry = KVAssetVariantMetadata & { + existsInKvStore: boolean, + }; + type VariantMetadataMap = Map; + const baseHashToVariantMetadatasMap = new Map(); + + // #### Iterate files + for (const file of files) { + // #### asset key + const assetKey = file.slice(publicDirRoot.length) + // in Windows, assetKey will otherwise end up as \path\file.html + .replace(/\\/g, '/'); + + // #### decide what the Content Type will be + let contentTypeTestResult = testFileContentType(contentTypes, assetKey); + if (contentTypeTestResult == null) { + contentTypeTestResult = { + text: false, + contentType: 'application/octet-stream', + precompressAsset: false, + }; + if (verbose) { + console.log('⚠️ Notice: Unknown file type ' + assetKey + '. Treating as binary file.'); + } + } + + const contentType = contentTypeTestResult.contentType; + const contentCompression = + contentTypeTestResult.precompressAsset ? publishContentConfig.contentCompression : []; + + // #### Are we going to include this file? + let includeAsset; + if (publishContentConfig.kvStoreAssetInclusionTest != null) { + + includeAsset = publishContentConfig.kvStoreAssetInclusionTest(assetKey, contentType); + + } else { + // If no test is set, then default to inclusion + includeAsset = true; + } + + if (!includeAsset) { + continue; + } + + // #### Base file size, hash, last modified time + const { size: baseSize, hash: baseHash } = await calculateFileSizeAndHash(file); + console.log(`📄 File '${rootRelative(file)}' - ${baseSize} bytes, sha256: ${baseHash}`); + const stats = fs.statSync(file); + const lastModifiedTime = Math.floor((stats.mtime).getTime() / 1000); + + // #### Metadata per variant + let variantMetadatas = baseHashToVariantMetadatasMap.get(baseHash); + if (variantMetadatas == null) { + variantMetadatas = new Map(); + baseHashToVariantMetadatasMap.set(baseHash, variantMetadatas); + } + + const variantsToKeep: ContentCompressionTypes[] = []; + + const variants = [ + 'original', + ...contentCompression, + ] as const; + for (const variant of variants) { + let variantKey = `${publishId}_files_sha256_${baseHash}`; + let variantFilename = `${baseHash}`; + if (variant !== 'original') { + variantKey = `${variantKey}_${variant}`; + variantFilename = `${variantFilename}_${variant}`; + } + + const variantFilePath = path.resolve(staticPublisherKvStoreContent, variantFilename); + + let variantMetadata = variantMetadatas.get(variant); + if (variantMetadata != null) { + + console.log(` 🏃‍♂️ Asset "${variantKey}" is identical to an item we already know about, reusing existing copy.`); + + } else { + + let kvStoreItemMetadata: KVAssetVariantMetadata | null = null; + + if (useKvStore && !forceUpload) { + await attemptWithRetries( + async () => { + // fastlyApiContext is non-null if useKvStore is true + const kvStoreEntryInfo = await getKvStoreEntryInfo(fastlyApiContext!, kvStoreName, variantKey); + if (!kvStoreEntryInfo) { + return; + } + let itemMetadata; + if (kvStoreEntryInfo.metadata != null) { + try { + itemMetadata = JSON.parse(kvStoreEntryInfo.metadata); + } catch { + // if the metadata does not parse successfully as JSON, + // treat it as though it didn't exist. + } + } + if (isKVAssetVariantMetadata(itemMetadata)) { + let exists = false; + if (itemMetadata.size <= KV_STORE_CHUNK_SIZE) { + // For an item equal to or smaller than the chunk size, if it exists + // and its metadata asserts no chunk count, then we assume it exists. + if (itemMetadata.numChunks === undefined) { + exists = true; + } + } else { + // For chunked objects, if the first chunk exists, and its metadata asserts + // the same number of chunks based on size, then we assume it exists (for now). + // In the future we might actually check for the existence and sizes of + // every chunk in the KV Store. + const expectedNumChunks = Math.ceil(itemMetadata.size / KV_STORE_CHUNK_SIZE); + if (itemMetadata.numChunks === expectedNumChunks) { + exists = true; + } + } + if (exists) { + kvStoreItemMetadata = { + contentEncoding: itemMetadata.contentEncoding, + size: itemMetadata.size, + hash: itemMetadata.hash, + numChunks: itemMetadata.numChunks, + }; + } + } + }, + { + onAttempt(attempt) { + if (attempt > 0) { + console.log(`Attempt ${attempt + 1} for: ${variantKey}`); + } + }, + onRetry(attempt, err, delay) { + let statusMessage = 'unknown'; + if (err instanceof FetchError) { + statusMessage = `HTTP ${err.status}`; + } else if (err instanceof TypeError) { + statusMessage = 'transport'; + } + console.log(`Attempt ${attempt + 1} for ${variantKey} gave retryable error (${statusMessage}), delaying ${delay} ms`); + }, + }, + ); + } + + if ((kvStoreItemMetadata as KVAssetVariantMetadata | null) != null) { + + console.log(` ・ Asset found in KV Store with key "${variantKey}".`); + // And we already know its hash and size. + + variantMetadata = { + contentEncoding: kvStoreItemMetadata!.contentEncoding, + size: kvStoreItemMetadata!.size, + hash: kvStoreItemMetadata!.hash, + numChunks: kvStoreItemMetadata!.numChunks, + existsInKvStore: true, + }; + + } else { + + await ensureVariantFileExists( + variantFilePath, + variant, + file, + ); + if (useKvStore) { + console.log(` ・ Flagging asset for upload to KV Store with key "${variantKey}".`); + } + + let contentEncoding, hash, size; + if (variant === 'original') { + contentEncoding = undefined; + hash = baseHash; + size = baseSize; + } else { + contentEncoding = variant; + ({hash, size} = await calculateFileSizeAndHash(variantFilePath)); + } + + const numChunks = Math.ceil(size / KV_STORE_CHUNK_SIZE); + + variantMetadata = { + contentEncoding, + size, + hash, + numChunks: numChunks > 1 ? numChunks : undefined, + existsInKvStore: false, + }; + } + + variantMetadatas.set(variant, variantMetadata); + + kvStoreItemDescriptions.push({ + write: !variantMetadata.existsInKvStore, + size: variantMetadata.size, + key: variantKey, + filePath: variantFilePath, + metadataJson: { + contentEncoding: variantMetadata.contentEncoding, + size: variantMetadata.size, + hash: variantMetadata.hash, + numChunks: variantMetadata.numChunks, + }, + }); + + if (createLocalFiles) { + // Although we already know the size and hash of the file, the local server + // needs a copy of the file so we create it if it doesn't exist. + // This may happen for example if files were uploaded to the KV Store in a previous + // publishing, but local static content files have been removed since. + await ensureVariantFileExists( + variantFilePath, + variant, + file, + ); + console.log(` ・ Prepping asset for local KV Store with key "${variantKey}".`); + } + } + + // Only keep variants whose file size actually ends up smaller than + // what we started with. + if (variant !== 'original' && variantMetadata.size < baseSize) { + variantsToKeep.push(variant); + } + } + + kvAssetsIndex[assetKey] = { + key: `sha256:${baseHash}`, + size: baseSize, + contentType: contentTypeTestResult.contentType, + lastModifiedTime, + variants: variantsToKeep, + }; + + } + console.log(`✅ Scan complete.`) + + // #### INDEX FILE + console.log(`🗂️ Creating Index...`); + const indexFileName = `index_${collectionName}.json`; + const indexFileKey = `${publishId}_index_${collectionName}`; + + // Metadata can have build time, expiration date, build name + // const indexMetadata = {}; + + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + fs.writeFileSync(indexFilePath, JSON.stringify(kvAssetsIndex)); + + const indexFileSize = getFileSize(indexFilePath); + + kvStoreItemDescriptions.push({ + write: true, + size: indexFileSize, + key: indexFileKey, + filePath: indexFilePath, + // metadata: indexMetadata, + }); + console.log(`✅ Index has been saved.`) + + // #### SERVER SETTINGS + // These are saved to KV Store + console.log(`⚙️ Saving server settings...`); + + const server = applyDefaults(publishContentConfig.server, { + publicDirPrefix: '', + staticItems: [], + allowedEncodings: [ 'br', 'gzip' ], + spaFile: null, + notFoundPageFile: null, + autoExt: [], + autoIndex: [], + }); + + let publicDirPrefix = server.publicDirPrefix; + console.log(` ✔️ Server public dir prefix '${publicDirPrefix}'.`); + + let staticItems = server.staticItems; + + let allowedEncodings = server.allowedEncodings; + + let spaFile = server.spaFile; + + if(spaFile != null) { + console.log(` ✔️ Application SPA file '${spaFile}'.`); + const spaAsset = kvAssetsIndex[spaFile]; + if(spaAsset == null || spaAsset.contentType !== 'text/html') { + if (verbose) { + console.log(` ⚠️ Notice: '${spaFile}' does not exist or is not of type 'text/html'. Ignoring.`); + } + spaFile = null; + } + } else { + if (verbose) { + console.log(` ✔️ Application is not a SPA.`); + } + } + + let notFoundPageFile = server.notFoundPageFile; + if(notFoundPageFile != null) { + console.log(` ✔️ Application 'not found (404)' file '${notFoundPageFile}'.`); + const notFoundPageAsset = kvAssetsIndex[notFoundPageFile]; + if(notFoundPageAsset == null || notFoundPageAsset.contentType !== 'text/html') { + if (verbose) { + console.log(` ⚠️ Notice: '${notFoundPageFile}' does not exist or is not of type 'text/html'. Ignoring.`); + } + notFoundPageFile = null; + } + } else { + if (verbose) { + console.log(` ✔️ Application specifies no 'not found (404)' page.`); + } + } + + let autoIndex: string[] = server.autoIndex; + let autoExt: string[] = server.autoExt; + + const serverSettings: PublisherServerConfigNormalized = { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + }; + + const settingsFileName = `settings_${collectionName}.json`; + const settingsFileKey = `${publishId}_settings_${collectionName}`; + + const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); + fs.writeFileSync(settingsFilePath, JSON.stringify(serverSettings)); + const settingsFileSize = getFileSize(settingsFilePath); + + kvStoreItemDescriptions.push({ + write: true, + size: settingsFileSize, + key: settingsFileKey, + filePath: settingsFilePath, + }); + console.log(`✅ Settings have been saved.`) + + console.log(`🍪 Chunking large files...`) + await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); + console.log(`✅ Large files have been chunked.`) + + if (useKvStore) { + console.log(`📤 Uploading entries to KV Store.`) + // fastlyApiContext is non-null if useKvStore is true + await uploadFilesToKVStore(fastlyApiContext!, kvStoreName, kvStoreItemDescriptions); + console.log(`✅ Uploaded entries to KV Store.`) + } + if (createLocalFiles) { + console.log(`📝 Writing local server KV Store entries.`) + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); + console.log(`✅ Wrote KV Store entries for local server.`) + } + + console.log(`🎉 Completed.`) + +} diff --git a/src/cli/compression/brotli.ts b/src/cli/compression/brotli.ts index 0cf9e37..34d7487 100644 --- a/src/cli/compression/brotli.ts +++ b/src/cli/compression/brotli.ts @@ -1,19 +1,17 @@ -import fs from 'fs'; -import zlib from 'zlib'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import zlib from 'node:zlib'; export const key = 'br'; -export async function compressTo(src: string, dest: string, isText: boolean): Promise { +export async function compressTo(src: string, dest: string): Promise { const buffer = fs.readFileSync(src); const resultBuffer = zlib.brotliCompressSync(buffer); - - // Don't actually create the file if it would be bigger - if (resultBuffer.length < buffer.length) { - fs.writeFileSync(dest, resultBuffer); - return true; - } else { - return false; - } + fs.writeFileSync(dest, resultBuffer); } diff --git a/src/cli/compression/gzip.ts b/src/cli/compression/gzip.ts index ab18232..16e6b26 100644 --- a/src/cli/compression/gzip.ts +++ b/src/cli/compression/gzip.ts @@ -1,19 +1,17 @@ -import fs from 'fs'; -import zlib from 'zlib'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import zlib from 'node:zlib'; export const key = 'gzip'; -export async function compressTo(src: string, dest: string, isText: boolean): Promise { +export async function compressTo(src: string, dest: string): Promise { const buffer = fs.readFileSync(src); const resultBuffer = zlib.gzipSync(buffer); - - // Don't actually create the file if it would be bigger - if (resultBuffer.length < buffer.length) { - fs.writeFileSync(dest, resultBuffer); - return true; - } else { - return false; - } + fs.writeFileSync(dest, resultBuffer); } diff --git a/src/cli/compression/index.ts b/src/cli/compression/index.ts index 936c857..e80b6f7 100644 --- a/src/cli/compression/index.ts +++ b/src/cli/compression/index.ts @@ -1,23 +1,18 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + import * as brotli from './brotli.js'; import * as gzip from './gzip.js'; -import { compressionTypes, ContentCompressionTypes } from "../../constants/compression.js"; +import { type ContentCompressionTypes } from '../../models/compression/index.js'; -export type CompressAlg = (src: string, dest: string, text: boolean) => Promise; +export type CompressAlg = (src: string, dest: string) => Promise; -export type AlgLib = { - key: string, - compressTo: CompressAlg, +const algs: Record = { + br: brotli.compressTo, + gzip: gzip.compressTo, }; -const libs: AlgLib[] = [ brotli, gzip ]; - -const algs = compressionTypes.reduce>>((obj, alg) => { - const lib = libs.find((lib: AlgLib) => lib.key === alg); - if (lib != null) { - obj[alg] = lib.compressTo; - } - return obj; -}, {}); - export { algs }; diff --git a/src/cli/util/fastly-api.ts b/src/cli/fastly-api/api-token.ts similarity index 64% rename from src/cli/util/fastly-api.ts rename to src/cli/fastly-api/api-token.ts index d52719c..b5a6c11 100644 --- a/src/cli/util/fastly-api.ts +++ b/src/cli/fastly-api/api-token.ts @@ -1,32 +1,52 @@ -import { execSync } from 'child_process'; -import { makeRetryable } from './retryable.js'; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { spawnSync } from 'node:child_process'; +import cli from '@fastly/cli'; + +import { makeRetryable } from '../util/retryable.js'; export interface FastlyApiContext { apiToken: string, }; -export type LoadApiKeyResult = { +export type LoadApiTokenResult = { apiToken: string, source: string, }; -export function loadApiKey(): LoadApiKeyResult | null { +export type LoadApiParams = { + commandLine: any, +}; + +export function loadApiToken(params: LoadApiParams): LoadApiTokenResult | null { let apiToken: string | null = null; let source: string = ''; - // Try to get API key from FASTLY_API_TOKEN - apiToken = process.env.FASTLY_API_TOKEN || null; - if (apiToken != null) { - source = 'env'; + // Try command line arg + if (typeof params.commandLine === 'string') { + apiToken = params.commandLine.trim(); + source = 'commandline'; + } + + // Try env (FASTLY_API_TOKEN) + if (apiToken == null) { + apiToken = process.env.FASTLY_API_TOKEN || null; + if (apiToken != null) { + source = 'env'; + } } + // Try fastly cli if (apiToken == null) { - // Try to get API key from fastly cli try { - apiToken = execSync('fastly profile token --quiet', { + const { stdout, error } = spawnSync(cli, ['profile', 'token', '--quiet'], { encoding: 'utf-8', - })?.trim() || null; + }); + apiToken = error ? null : stdout.trim(); } catch { apiToken = null; } @@ -71,16 +91,13 @@ export async function callFastlyApi( requestInit?: RequestInit, ): Promise { - let finalEndpoint = endpoint; + const url = new URL(endpoint, 'https://api.fastly.com/'); if (queryParams != null) { - const queryString = String(queryParams); - if (queryString.length > 0) { - finalEndpoint += '?' + queryString; + for (const [key, value] of queryParams.entries()) { + url.searchParams.append(key, value); } } - const url = new URL(finalEndpoint, 'https://api.fastly.com/'); - const headers = new Headers(requestInit?.headers); headers.set('Fastly-Key', fastlyApiContext.apiToken); diff --git a/src/cli/util/kv-store.ts b/src/cli/fastly-api/kv-store.ts similarity index 73% rename from src/cli/util/kv-store.ts rename to src/cli/fastly-api/kv-store.ts index 3eb2733..ca78c87 100644 --- a/src/cli/util/kv-store.ts +++ b/src/cli/fastly-api/kv-store.ts @@ -1,4 +1,9 @@ -import { callFastlyApi, FastlyApiContext, FetchError } from "./fastly-api.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { callFastlyApi, type FastlyApiContext, FetchError } from './api-token.js'; type KVStoreInfo = { id: string, @@ -100,19 +105,47 @@ export async function getKVStoreInfos(fastlyApiContext: FastlyApiContext) { return kvStoreInfos; } -export const _getKVStoreKeys = createArrayGetter()((kvStoreId: string) => `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys`); +export const _getKVStoreKeys = createArrayGetter()( + (kvStoreId: string, prefix?: string) => { + let endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys`; + if (prefix != null) { + endpoint += '?prefix=' + encodeURIComponent(prefix); + } + return endpoint; + } +); -export async function getKVStoreKeys(fastlyApiContext: FastlyApiContext, kvStoreName: string) { +export async function getKVStoreKeys( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + prefix?: string, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { return null; } - return await _getKVStoreKeys(fastlyApiContext, `Listing Keys for KV Store [${kvStoreId}] ${kvStoreName}`, kvStoreId); + return await _getKVStoreKeys( + fastlyApiContext, + `Listing Keys for KV Store [${kvStoreId}] ${kvStoreName}${prefix != null ? ` (prefix '${prefix}')` : ''}`, + kvStoreId, + prefix, + ); +} + +export async function getKvStoreEntryInfo(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { + + return getKvStoreEntry(fastlyApiContext, kvStoreName, key, true); + } -export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { +export async function getKvStoreEntry( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + key: string, + metadataOnly?: boolean, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -121,9 +154,10 @@ export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvS const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; + let response: Response; try { - await callFastlyApi(fastlyApiContext, endpoint, `Checking existence of [${key}]`, null, { method: 'HEAD' }); + response = await callFastlyApi(fastlyApiContext, endpoint, `Checking existence of [${key}]`, null, { method: metadataOnly ? 'HEAD' : 'GET' }); } catch(err) { if (err instanceof FetchError && err.status === 404) { @@ -132,12 +166,18 @@ export async function kvStoreEntryExists(fastlyApiContext: FastlyApiContext, kvS throw err; } - return true; + const metadata = response.headers.get('metadata'); + const generation = response.headers.get('generation'); + return { + metadata, + generation, + response, + }; } const encoder = new TextEncoder(); -export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string) { +export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string, metadata: string | undefined) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -151,6 +191,7 @@ export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvSt method: 'PUT', headers: { 'content-type': 'application/octet-stream', + ...(metadata != null ? { metadata } : null) }, body, }); diff --git a/src/cli/index.ts b/src/cli/index.ts index 053ab73..5e090f0 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,115 +1,11 @@ #!/usr/bin/env node +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ -import commandLineArgs, { OptionDefinition } from "command-line-args"; +import { action } from './commands/index.js'; -import { initApp } from "./commands/init-app.js"; -import { buildStaticLoader } from "./commands/build-static.js"; -import { cleanKVStore } from "./commands/clean-kv-store.js"; +console.log("🧑‍💻 Fastly Compute JavaScript Static Publisher"); -const optionDefinitions: OptionDefinition[] = [ - // (optional) Should be one of: - // - cra (or create-react-app) - // - vite - // - sveltekit - // - vue - // - next - // - astro - // - gatsby - // - docusaurus - { name: 'preset', type: String, }, - - { name: 'build-static', type: Boolean }, - { name: 'suppress-framework-warnings', type: Boolean }, - { name: 'output', alias: 'o', type: String, defaultValue: './compute-js', }, - - // Whether the scaffolded project should use webpack to bundle assets. - { name: 'webpack', type: Boolean, defaultValue: false }, - - // The 'root' directory for the publishing. - // All assets are expected to exist under this root. Required. - // For backwards compatibility, if this value is not provided, - // then the value of 'public-dir' is used. - { name: 'root-dir', type: String, }, - - // The 'public' directory. The Publisher Server will - // resolve requests relative to this directory. If not specified, - // defaults to the same value as 'root-dir'. See README for - // details. - { name: 'public-dir', type: String, }, - - // Directories to specify as containing 'static' files. The - // Publisher Server will serve files from these directories - // with a long TTL. - { name: 'static-dir', type: String, multiple: true, }, - - // The 'static content root directory' where the Static Publisher - // outputs its metadata files and loaders. - { name: 'static-content-root-dir', type: String, }, - - // Path to a file to be used to serve in a SPA application. - // The Publisher Server will serve this file with a 200 status code - // when the request doesn't match a known file, and the accept - // header includes text/html. You may use the '[public-dir]' token - // if you wish to specify this as a relative path from the 'public-dir'. - { name: 'spa', type: String, }, - - // Path to a file to be used to serve as a 404 not found page. - // The Publisher Server will serve this file with a 404 status code - // when the request doesn't match a known file, and the accept - // header includes text/html. You may use the '[public-dir]' token - // if you wish to specify this as a relative path from the 'public-dir'. - { name: 'not-found-page', type: String, }, - - // List of files to automatically use as index, for example, index.html,index.htm - // If a request comes in but the route does not exist, we check the route - // plus a slash plus the items in this array. - { name: 'auto-index', type: String, multiple: true, }, - - // List of extensions to apply to a path name, for example, if - // http://example.com/about is requested, we can respond with http://example.com/about.html - { name: 'auto-ext', type: String, multiple: true, }, - - // Components from fastly.toml - - // Name of the application, to be inserted into fastly.toml - { name: 'name', type: String, }, - - // Name of the author, to be inserted into fastly.toml - { name: 'author', type: String, }, - - // Description of the application, to be inserted into fastly.toml - { name: 'description', type: String, }, - - // Fastly Service ID to be added to the fastly.toml that is generated. - { name: 'service-id', type: String }, - - // The name of a Fastly KV Store to hold the content assets. - // It must be linked to the service specified by `--service-id`. - { name: 'kv-store-name', type: String }, - - // Clean KV Store mode - { name: 'clean-kv-store', type: Boolean, }, -]; - -const commandLineValues = commandLineArgs(optionDefinitions); - -console.log("Fastly Compute JavaScript Static Publisher"); - -let mode: 'init-app' | 'build-static' | 'clean-kv-store' = 'init-app'; -if(commandLineValues['build-static']) { - mode = 'build-static'; -} else if(commandLineValues['clean-kv-store']) { - mode = 'clean-kv-store'; -} - -switch(mode) { -case 'build-static': - await buildStaticLoader(commandLineValues); - break; -case 'init-app': - initApp(commandLineValues); - break; -case 'clean-kv-store': - await cleanKVStore(commandLineValues); - break; -} +await action(process.argv); diff --git a/src/cli/presets/implementations/astro.ts b/src/cli/presets/implementations/astro.ts deleted file mode 100644 index adf4153..0000000 --- a/src/cli/presets/implementations/astro.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class AstroPreset implements IPresetBase { - name = 'Astro'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-astro-app', - description: 'Fastly Compute static site from Astro', - }; - check(packageJson: any) { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a Astro project directory."); - return false; - } - if(packageJson?.dependencies?.['astro'] == null) { - console.error("❌ Can't find astro in dependencies"); - console.error("Run this from a Astro project directory."); - return false; - } - return true; - } -} diff --git a/src/cli/presets/implementations/create-react-app.ts b/src/cli/presets/implementations/create-react-app.ts deleted file mode 100644 index e463e20..0000000 --- a/src/cli/presets/implementations/create-react-app.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class CreateReactAppPreset implements IPresetBase { - name = 'Create React App'; - defaultOptions: Partial = { - rootDir: './build', - staticDirs: [ '[public-dir]/static' ], - name: 'my-create-react-app', - description: 'Fastly Compute static site from create-react-app', - }; - - check(packageJson: any): boolean { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a create-react-app project directory."); - return false; - } - if(packageJson?.dependencies?.['react-scripts'] == null) { - console.error("❌ Can't find react-scripts in dependencies"); - console.error("Run this from a create-react-app project directory."); - console.log("If this is a project created with create-react-app and has since been ejected, specify preset cra-eject to skip this check."); - return false; - } - return true; - } - -} - -export class CreateReactAppEjectedPreset extends CreateReactAppPreset { - name = 'Create React App (Ejected)'; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/docusaurus.ts b/src/cli/presets/implementations/docusaurus.ts deleted file mode 100644 index e383892..0000000 --- a/src/cli/presets/implementations/docusaurus.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class DocusaurusPreset implements IPresetBase { - name = 'Docusaurus'; - defaultOptions: Partial = { - rootDir: './build', - name: 'my-docusaurus-app', - description: 'Fastly Compute static site from docusaurus', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/gatsby.ts b/src/cli/presets/implementations/gatsby.ts deleted file mode 100644 index 46aea0c..0000000 --- a/src/cli/presets/implementations/gatsby.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class GatsbyPreset implements IPresetBase { - name = 'Gatsby'; - defaultOptions: Partial = { - rootDir: './public', - name: 'my-gatsby-app', - description: 'Fastly Compute static site from Gatsby', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/next.ts b/src/cli/presets/implementations/next.ts deleted file mode 100644 index c39dfd7..0000000 --- a/src/cli/presets/implementations/next.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class NextJsPreset implements IPresetBase { - name = 'Next.js'; - defaultOptions: Partial = { - rootDir: './out', - name: 'my-next-app', - description: 'Fastly Compute static site from Next.js', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/sveltekit.ts b/src/cli/presets/implementations/sveltekit.ts deleted file mode 100644 index 44f5fea..0000000 --- a/src/cli/presets/implementations/sveltekit.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class SvelteKitPreset implements IPresetBase { - name = 'SvelteKit'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-sveltekit-app', - description: 'Fastly Compute static site from SvelteKit', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/vite.ts b/src/cli/presets/implementations/vite.ts deleted file mode 100644 index 9740564..0000000 --- a/src/cli/presets/implementations/vite.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class VitePreset implements IPresetBase { - name = 'Vite'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-vite-app', - description: 'Fastly Compute static site from Vite', - }; - check() { - return true; - } -} diff --git a/src/cli/presets/implementations/vue.ts b/src/cli/presets/implementations/vue.ts deleted file mode 100644 index 5b960ed..0000000 --- a/src/cli/presets/implementations/vue.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { AppOptions, IPresetBase } from '../preset-base.js'; - -export class VuePreset implements IPresetBase { - name = 'Vue'; - defaultOptions: Partial = { - rootDir: './dist', - name: 'my-vue-app', - description: 'Fastly Compute static site from Vue (create-vue)', - }; - check(packageJson: any) { - if(packageJson == null) { - console.error("❌ Can't read/parse package.json"); - console.error("Run this from a Vue project directory created by create-vue."); - return false; - } - if(packageJson?.devDependencies?.['vite'] == null) { - console.error("❌ Can't find vite in dependencies"); - console.error("Run this from a Vue project directory created by create-vue."); - console.log("If this is a project created with the Vue CLI, migrate it to use Vite first. Refer to the create-vue documentation at https://www.npmjs.com/package/create-vue (not authored or maintained by Fastly) for details on this process."); - return false; - } - return true; - } -} diff --git a/src/cli/presets/index.ts b/src/cli/presets/index.ts deleted file mode 100644 index 5fbf809..0000000 --- a/src/cli/presets/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { IPresetBase } from './preset-base.js'; -import { CreateReactAppPreset, CreateReactAppEjectedPreset } from './implementations/create-react-app.js'; -import { VitePreset } from './implementations/vite.js'; -import { SvelteKitPreset } from './implementations/sveltekit.js'; -import { VuePreset } from './implementations/vue.js'; -import { NextJsPreset } from './implementations/next.js'; -import { AstroPreset } from "./implementations/astro.js"; -import { GatsbyPreset } from './implementations/gatsby.js'; -import { DocusaurusPreset } from './implementations/docusaurus.js'; - -export const presets: Record IPresetBase> = { - 'cra': CreateReactAppPreset, - 'create-react-app': CreateReactAppPreset, - 'cra-eject': CreateReactAppEjectedPreset, - 'vite': VitePreset, - 'sveltekit': SvelteKitPreset, - 'vue': VuePreset, - 'next': NextJsPreset, - 'astro': AstroPreset, - 'gatsby': GatsbyPreset, - 'docusaurus': DocusaurusPreset, -}; diff --git a/src/cli/presets/preset-base.ts b/src/cli/presets/preset-base.ts deleted file mode 100644 index 098258d..0000000 --- a/src/cli/presets/preset-base.ts +++ /dev/null @@ -1,21 +0,0 @@ -export type AppOptions = { - rootDir: string | undefined, - publicDir: string | undefined, - staticDirs: string[], - staticContentRootDir: string | undefined, - spa: string | undefined, - notFoundPage: string | undefined, - autoIndex: string[], - autoExt: string[], - name: string | undefined, - author: string | undefined, - description: string | undefined, - serviceId: string | undefined, - kvStoreName: string | undefined, -}; - -export interface IPresetBase { - name: string; - defaultOptions: Partial; - check(packageJson: any | null, options: AppOptions): boolean; -} diff --git a/src/cli/util/args.ts b/src/cli/util/args.ts new file mode 100644 index 0000000..a096c4c --- /dev/null +++ b/src/cli/util/args.ts @@ -0,0 +1,86 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import commandLineArgs from 'command-line-args'; + +export type ModeAction = (argv: string[]) => void | Promise; +export type ActionModule = { action: ModeAction }; + +export function isHelpArgs(argv: string[]) { + + const helpDefinitions = [ + { name: 'help', type: Boolean, }, + ]; + const helpOptions = commandLineArgs(helpDefinitions, { argv, stopAtFirstUnknown: true }); + return !!helpOptions['help']; + +} + +export function findMainCommandNameAndArgs(argv: string[]): [string | null, string[]] { + + const mainDefinitions = [ + { name: 'command', type: String, defaultOption: true }, + ]; + const mainOptions = commandLineArgs(mainDefinitions, { argv, stopAtFirstUnknown: true }); + const commandArgs = mainOptions._unknown || []; + + const command = mainOptions['command']; + if (typeof command !== 'string') { + return [ null, commandArgs ]; + } + + return [ command, commandArgs ]; + +} + +export type CommandAndArgs = +| { + needHelp: false, + command: T, + argv: string[], +} +| { + needHelp: true, + command: string | null, +} +; + +export function getCommandAndArgs( + argv: string[], + modes: T[], +): CommandAndArgs { + if (isHelpArgs(argv)) { + return { + needHelp: true, + command: null, + }; + } + + const result = findMainCommandNameAndArgs(argv); + + const [ command, actionArgv ] = result; + if (command != null) { + for (const modeName of modes) { + if (command === modeName) { + return { + needHelp: false, + command: command as T, + argv: actionArgv, + }; + } + } + } + + try { + commandLineArgs([], { argv: actionArgv }); + } catch(err) { + console.log(String(err)); + } + + return { + needHelp: true, + command, + }; +} diff --git a/src/cli/load-config.ts b/src/cli/util/config.ts similarity index 63% rename from src/cli/load-config.ts rename to src/cli/util/config.ts index 4dc137d..7e7029c 100644 --- a/src/cli/load-config.ts +++ b/src/cli/util/config.ts @@ -1,169 +1,159 @@ -import url from "url"; -import path from "path"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import path from 'node:path'; + import globToRegExp from 'glob-to-regexp'; +import { + type PublishContentConfigNormalized, + type ContentTypeDef, +} from '../../models/config/publish-content-config.js'; import { buildNormalizeFunctionForArray, buildNormalizeFunctionForObject, isSpecified, isStringAndNotEmpty, -} from "./util/data.js"; +} from './data.js'; +import { type PublisherServerConfigNormalized } from '../../models/config/publisher-server-config.js'; +import { type StaticPublishRc } from '../../models/config/static-publish-rc.js'; -import type { - ContentTypeDef, -} from "../types/content-types.js"; -import type { - StaticPublisherConfigNormalized, - PublisherServerConfigNormalized, -} from "../types/config-normalized.js"; +export class LoadConfigError extends Error { + errors: string[]; -const normalizeContentTypeDef = buildNormalizeFunctionForObject((config, errors): ContentTypeDef | null => { + constructor(configFilePath: string, errors: string[]) { + super(`Error loading config file ${configFilePath}`); + this.errors = errors; + } +} - let { test, contentType, text } = config; +export async function loadStaticPublisherRcFile(): Promise { - if (typeof test === 'function' || test instanceof RegExp) { - // ok - } else { - errors.push('test cannot be null.'); - } + let configRaw; - if (isStringAndNotEmpty(contentType)) { - // ok - } else { - errors.push('contentType must be a non-empty string.'); + const configFile = './static-publish.rc.js'; + + try { + const filePath = path.resolve(configFile); + configRaw = (await import(filePath)).default; + } catch { + // } - if (!isSpecified(config, 'text')) { - text = true; - } else { - if (typeof text === 'boolean') { - // ok - } else { - errors.push('text, if specified, must be a boolean value.'); - } + if (configRaw == null) { + throw new LoadConfigError(configFile, [ + 'Unable to load ' + configFile, + ]); } - if (errors.length > 0) { - return null; + const errors: string[] = []; + const config = normalizeStaticPublisherRc(configRaw, errors); + if (config == null) { + throw new LoadConfigError(configFile, errors); } - return { - test, - contentType, - text, - }; + return config; -}); +} -const normalizeContentTypeDefs = buildNormalizeFunctionForArray((config, errors) => { - return normalizeContentTypeDef(config, errors); -}); -const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { +export const normalizeStaticPublisherRc = buildNormalizeFunctionForObject((config, errors) => { - let { publicDirPrefix, staticItems, compression, spaFile, notFoundPageFile, autoExt, autoIndex } = config; + let { + kvStoreName, + publishId, + defaultCollectionName, + } = config; - if (!isSpecified(config, 'publicDirPrefix')) { - publicDirPrefix = ''; + if (!isSpecified(config, 'kvStoreName')) { + errors.push('kvStoreName must be specified.'); } else { - if (typeof publicDirPrefix === 'string') { + if (isStringAndNotEmpty(kvStoreName)) { // ok } else { - errors.push('publicDirPrefix, if specified, must be a string value.'); + errors.push('kvStoreName must be a non-empty string.'); } } - if (!isSpecified(config, 'staticItems')) { - staticItems = []; + if (!isSpecified(config, 'publishId')) { + errors.push('publishId must be specified.'); } else { - if (staticItems === null || staticItems === false) { - staticItems = []; - } - if (!Array.isArray(staticItems)) { - staticItems = [ staticItems ]; - } - if (staticItems.every((x: any) => typeof x === 'string')) { - staticItems = (staticItems as string[]).map((x, index) => { - if (x.includes('*')) { - let re; - try { - const regexp = globToRegExp(x, {globstar: true}); - re = 're:' + String(regexp); - } catch { - errors.push(`staticItems item at index ${index}, '${x}', cannot be parsed as glob pattern.`); - re = null; - } - return re; - } - return x; - }); + if (isStringAndNotEmpty(publishId)) { + // ok } else { - errors.push('staticItems, if specified, must be a string value, an array of string values, false, or null'); + errors.push('publishId must be a non-empty string.'); } } - if (!isSpecified(config, 'compression')) { - compression = [ 'br', 'gzip' ]; - } else if (compression === null) { - compression = [] + if (!isSpecified(config, 'defaultCollectionName')) { + errors.push('defaultCollectionName must be specified.'); } else { - if (!Array.isArray(compression)) { - compression = [ compression ]; - } - if (compression.some((x: any) => x !== 'br' && x !== 'gzip')) { - errors.push(`compression, if specified, must be null or an array and can only contain 'br' and 'gzip'.`); - } - } - - if (!isSpecified(config, 'spaFile')) { - spaFile = null; - } else { - if (typeof spaFile === 'string' || spaFile === null) { + if (isStringAndNotEmpty(defaultCollectionName)) { // ok - } else if (spaFile === false) { - spaFile = null; } else { - errors.push('spaFile, if specified, must be a string value, false, or null'); + errors.push('defaultCollectionName must be a non-empty string.'); } } - if (!isSpecified(config, 'notFoundPageFile')) { - notFoundPageFile = null; + return { + kvStoreName, + publishId, + defaultCollectionName, + }; +}); + +export async function loadPublishContentConfigFile(configFile: string): Promise { + + let configRaw; + + try { + const filePath = path.resolve(configFile); + configRaw = (await import(filePath)).default; + } catch { + // + } + + if (configRaw == null) { + throw new LoadConfigError(configFile, [ + 'Unable to load ' + configFile, + ]); + } + + const errors: string[] = []; + const config = normalizePublishContentConfig(configRaw, errors); + if (config == null) { + throw new LoadConfigError(configFile, errors); + } + + return config; + +} + +const normalizeContentTypeDef = buildNormalizeFunctionForObject((config, errors): ContentTypeDef | null => { + + let { test, contentType, text } = config; + + if (typeof test === 'function' || test instanceof RegExp) { + // ok } else { - if (typeof notFoundPageFile === 'string' || notFoundPageFile === null) { - // ok - } else if (notFoundPageFile === false) { - notFoundPageFile = null; - } else { - errors.push('notFoundPageFile, if specified, must be a string value, false, or null'); - } + errors.push('test cannot be null.'); } - if (!isSpecified(config, 'autoExt')) { - autoExt = []; + if (isStringAndNotEmpty(contentType)) { + // ok } else { - if (Array.isArray(autoExt) && autoExt.every(e => typeof e === 'string')) { - // ok - } else if (autoExt === false || autoExt === null) { - autoExt = []; - } else if (typeof autoExt === 'string') { - autoExt = [ autoExt ]; - } else { - errors.push('autoExt, if specified, must be an array of string values, a string value, false, or null'); - } + errors.push('contentType must be a non-empty string.'); } - if (!isSpecified(config, 'autoIndex')) { - autoIndex = []; + if (!isSpecified(config, 'text')) { + text = true; } else { - if (Array.isArray(autoIndex) && autoIndex.every(e => typeof e === 'string')) { + if (typeof text === 'boolean') { // ok - } else if (autoIndex === false || autoIndex === null) { - autoIndex = []; - } else if (typeof autoIndex === 'string') { - autoIndex = [ autoIndex ]; } else { - errors.push('autoIndex, if specified, must be an array of string values, a string value, false, or null'); + errors.push('text, if specified, must be a boolean value.'); } } @@ -172,29 +162,27 @@ const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { +const normalizeContentTypeDefs = buildNormalizeFunctionForArray((config, errors) => { + return normalizeContentTypeDef(config, errors); +}); + +export const normalizePublishContentConfig = buildNormalizeFunctionForObject((config, errors) => { let { rootDir, - staticContentRootDir, - kvStoreName, + staticPublisherWorkingDir, excludeDirs, excludeDotFiles, includeWellKnown, - contentAssetInclusionTest, + kvStoreAssetInclusionTest, contentCompression, - moduleAssetInclusionTest, contentTypes, server, } = config; @@ -209,33 +197,21 @@ const normalizeConfig = buildNormalizeFunctionForObject { +export const normalizePublisherServerConfig = buildNormalizeFunctionForObject((config, errors) => { - let raw: any = undefined; - const staticPublishRcPath = path.resolve('./static-publish.rc.js'); - const staticPublishRcFileURL = String(url.pathToFileURL(staticPublishRcPath)); - try { - raw = (await import(staticPublishRcFileURL)).default; - } catch { - errors.push('Unable to load ' + staticPublishRcFileURL); + let { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + } = config; + + if (!isSpecified(config, 'publicDirPrefix')) { + publicDirPrefix = ''; + } else { + if (typeof publicDirPrefix === 'string') { + // ok + } else { + errors.push('publicDirPrefix, if specified, must be a string value.'); + } } - let normalized: any = undefined; - if (raw != null) { - normalized = normalizeConfig(raw, errors); + if (!isSpecified(config, 'staticItems')) { + staticItems = []; + } else { + if (staticItems === null || staticItems === false) { + staticItems = []; + } + if (!Array.isArray(staticItems)) { + staticItems = [ staticItems ]; + } + if (staticItems.every((x: any) => typeof x === 'string')) { + staticItems = (staticItems as string[]).map((x, index) => { + if (x.includes('*')) { + let re; + try { + const regexp = globToRegExp(x, {globstar: true}); + re = 're:' + String(regexp); + } catch { + errors.push(`staticItems item at index ${index}, '${x}', cannot be parsed as glob pattern.`); + re = null; + } + return re; + } + return x; + }); + } else { + errors.push('staticItems, if specified, must be a string value, an array of string values, false, or null'); + } } - return { normalized: normalized ?? null, raw }; + if (!isSpecified(config, 'allowedEncodings')) { + allowedEncodings = [ 'br', 'gzip' ]; + } else if (allowedEncodings === null) { + allowedEncodings = [] + } else { + if (!Array.isArray(allowedEncodings)) { + allowedEncodings = [ allowedEncodings ]; + } + if (allowedEncodings.some((x: any) => x !== 'br' && x !== 'gzip')) { + errors.push(`allowedEncodings, if specified, must be null or an array and can only contain 'br' and 'gzip'.`); + } + } -} + if (!isSpecified(config, 'spaFile')) { + spaFile = null; + } else { + if (typeof spaFile === 'string' || spaFile === null) { + // ok + } else if (spaFile === false) { + spaFile = null; + } else { + errors.push('spaFile, if specified, must be a string value, false, or null'); + } + } + + if (!isSpecified(config, 'notFoundPageFile')) { + notFoundPageFile = null; + } else { + if (typeof notFoundPageFile === 'string' || notFoundPageFile === null) { + // ok + } else if (notFoundPageFile === false) { + notFoundPageFile = null; + } else { + errors.push('notFoundPageFile, if specified, must be a string value, false, or null'); + } + } + + if (!isSpecified(config, 'autoExt')) { + autoExt = []; + } else { + if (Array.isArray(autoExt) && autoExt.every(e => typeof e === 'string')) { + // ok + } else if (autoExt === false || autoExt === null) { + autoExt = []; + } else if (typeof autoExt === 'string') { + autoExt = [ autoExt ]; + } else { + errors.push('autoExt, if specified, must be an array of string values, a string value, false, or null'); + } + } + + if (!isSpecified(config, 'autoIndex')) { + autoIndex = []; + } else { + if (Array.isArray(autoIndex) && autoIndex.every(e => typeof e === 'string')) { + // ok + } else if (autoIndex === false || autoIndex === null) { + autoIndex = []; + } else if (typeof autoIndex === 'string') { + autoIndex = [ autoIndex ]; + } else { + errors.push('autoIndex, if specified, must be an array of string values, a string value, false, or null'); + } + } + + if (errors.length > 0) { + return null; + } + + return { + publicDirPrefix, + staticItems, + allowedEncodings, + spaFile, + notFoundPageFile, + autoExt, + autoIndex, + }; + +}); diff --git a/src/util/content-types.ts b/src/cli/util/content-types.ts similarity index 95% rename from src/util/content-types.ts rename to src/cli/util/content-types.ts index 9777c0b..73b8fda 100644 --- a/src/util/content-types.ts +++ b/src/cli/util/content-types.ts @@ -1,7 +1,12 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + import { - ContentTypeDef, - ContentTypeTestResult, -} from "../types/content-types.js"; + type ContentTypeDef, + type ContentTypeTestResult, +} from '../../models/config/publish-content-config.js'; const defaultContentTypes: ContentTypeDef[] = [ // Text formats @@ -38,10 +43,6 @@ const defaultContentTypes: ContentTypeDef[] = [ { test: /\.woff2$/, contentType: 'font/woff2', text: false }, ]; -export function getDefaultContentTypes() { - return defaultContentTypes; -} - export function mergeContentTypes(contentTypes: ContentTypeDef[]) { const finalContentTypes: ContentTypeDef[] = []; diff --git a/src/cli/util/data.ts b/src/cli/util/data.ts index a30b0b5..0433bdf 100644 --- a/src/cli/util/data.ts +++ b/src/cli/util/data.ts @@ -1,11 +1,16 @@ -export type NormalizeAction = (value: any, errors: string[]) => T | null; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type NormalizeAction = (value: Record, errors: string[]) => T | null; export function buildNormalizeFunctionForObject(action: NormalizeAction) { - return (obj: any, errCtx: string[] = []): TObject | null => { + return (obj: unknown, errCtx: string[] = []): TObject | null => { - if (obj == null) { - errCtx.push('obj cannot be null.'); + if (obj == null || typeof obj !== 'object') { + errCtx.push('obj must be an object.'); return null; } @@ -25,7 +30,7 @@ export function buildNormalizeFunctionForObject(action: NormalizeAction export function buildNormalizeFunctionForArray(action: NormalizeAction) { - return (obj: any, errCtx: string[] = []): TEntry[] | null => { + return (obj: unknown, errCtx: string[] = []): TEntry[] | null => { if (!Array.isArray(obj)) { errCtx.push('obj must be array.'); diff --git a/src/cli/util/files.ts b/src/cli/util/files.ts index 62760f2..24ee2fd 100644 --- a/src/cli/util/files.ts +++ b/src/cli/util/files.ts @@ -1,24 +1,30 @@ -import fs from "fs"; -import path from "path"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; export interface FilenameTest { test(name: string): boolean; } -export type GetFilesOpts = { +export type EnumerateFilesOpts = { publicDirRoot: string, excludeDirs: FilenameTest[], excludeDotFiles: boolean, includeWellKnown: boolean, }; -export function getFiles(dir: string, opts: GetFilesOpts) { +export function enumerateFiles(opts: EnumerateFilesOpts) { const results: string[] = []; - getFilesWorker(results, dir, opts); + enumerateFilesWorker(results, opts.publicDirRoot, opts); return results; } -function getFilesWorker(results: string[], dir: string, opts: GetFilesOpts) { +function enumerateFilesWorker(results: string[], dir: string, opts: EnumerateFilesOpts) { const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { const { name } = entry; @@ -45,9 +51,62 @@ function getFilesWorker(results: string[], dir: string, opts: GetFilesOpts) { } if (entry.isDirectory()) { - getFilesWorker(results, fullpath, opts); + enumerateFilesWorker(results, fullpath, opts); } else { results.push(fullpath); } } } + +export function getFileSize(filePath: string) { + const stats = fs.statSync(filePath); + return stats.size; +} + +export function directoryExists(dirPath: string) { + try { + const stats = fs.statSync(dirPath); + return stats.isDirectory(); + } catch (err) { + if ((err as any).code === 'ENOENT') { + return false; // Path doesn't exist + } + throw err; // Other errors, rethrow + } +} + +export async function calculateFileSizeAndHash(filePath: string) { + + const stream = fs.createReadStream(filePath); + + // Use createHash, rather than subtle, since it supports streaming. + const hash = crypto.createHash('sha256'); + let size = 0; + + return new Promise<{ size: number, hash: string }>((resolve, reject) => { + stream.on('data', chunk => { + hash.update(chunk); + size += chunk.length; + }); + + stream.on('end', () => { + resolve({ + size, + hash: hash.digest('hex'), + }); + }); + + stream.on('error', reject); + }); + +} + +export function rootRelative(itemPath: string) { + return dotRelative(null, itemPath); +} + +export function dotRelative(from: string | null, to: string) { + const relPath = path.relative(from ?? path.resolve(), to); + return relPath.startsWith('..') ? relPath : './' + relPath; +} + diff --git a/src/cli/util/hash.ts b/src/cli/util/hash.ts deleted file mode 100644 index e12e782..0000000 --- a/src/cli/util/hash.ts +++ /dev/null @@ -1,14 +0,0 @@ -import crypto from 'crypto'; -import fs from 'fs'; - -export function calculateFileSizeAndHash(filename: string) { - - const fileBuffer = fs.readFileSync(filename); - const size = fileBuffer.length; - const hash = crypto.createHash('sha256'); - - hash.update(fileBuffer); - - return { size, hash: hash.digest('hex') }; - -} diff --git a/src/cli/util/id.ts b/src/cli/util/id.ts deleted file mode 100644 index d9e64b3..0000000 --- a/src/cli/util/id.ts +++ /dev/null @@ -1,22 +0,0 @@ -import crypto from "crypto"; - -const ID_SYMBOLS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; -const ID_SYMBOLS_BASE = BigInt(ID_SYMBOLS.length); - -export function createStringId() { - return uuidToString(crypto.randomUUID()); -} - -export function uuidToString(uuid: string) { - const uuidWithoutHyphens = uuid.replace(/-/g, ''); - - let number = BigInt('0x' + uuidWithoutHyphens); - let result = ''; - do { - const [ quotient, remainder ] = [ number / ID_SYMBOLS_BASE, Number(number % ID_SYMBOLS_BASE) ]; - result = ID_SYMBOLS.charAt( remainder ) + result; - number = quotient; - } while(number > 0n); - - return result; -} diff --git a/src/cli/util/kv-store-items.ts b/src/cli/util/kv-store-items.ts new file mode 100644 index 0000000..f219b46 --- /dev/null +++ b/src/cli/util/kv-store-items.ts @@ -0,0 +1,206 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import { directoryExists, getFileSize, rootRelative } from './files.js'; +import { attemptWithRetries } from './retryable.js'; +import { type FastlyApiContext, FetchError } from '../fastly-api/api-token.js'; +import { kvStoreSubmitFile } from '../fastly-api/kv-store.js'; + +export type KVStoreItemDesc = { + write: boolean, + size: number, + key: string, + filePath: string, + metadataJson?: Record, +}; + +export async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItemDescriptions: KVStoreItemDesc[]) { + + const maxConcurrent = 12; + let index = 0; // Shared among workers + + async function worker() { + while (index < kvStoreItemDescriptions.length) { + const currentIndex = index; + index = index + 1; + const { write, key, filePath, metadataJson } = kvStoreItemDescriptions[currentIndex]; + if (!write) { + continue; + } + + try { + await attemptWithRetries( + async() => { + const fileBytes = fs.readFileSync(filePath); + await kvStoreSubmitFile(fastlyApiContext, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); + console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) + }, + { + onAttempt(attempt) { + if (attempt > 0) { + console.log(` Attempt ${attempt + 1} for: ${key}`); + } + }, + onRetry(attempt, err, delay) { + let statusMessage = 'unknown'; + if (err instanceof FetchError) { + statusMessage = `HTTP ${err.status}`; + } else if (err instanceof TypeError) { + statusMessage = 'transport'; + } + console.log(` ‼️ Attempt ${attempt + 1} for ${key} gave retryable error (${statusMessage}), delaying ${delay} ms`); + }, + } + ); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + console.error(` ❌ Failed: ${key} → ${e.message}`); + console.error(e.stack); + } + } + } + + const workers = Array.from({ length: maxConcurrent }, () => worker()); + await Promise.all(workers); +} + +export function shouldRecreateChunks(chunksDir: string, numChunks: number, item: KVStoreItemDesc, chunkSize: number) { + console.log(` 📄 '${item.key}' - ${item.size} bytes → ${numChunks} chunks`) + if (!directoryExists(chunksDir)) { + console.log(` ・ Creating chunks for '${item.key}' - found no existing chunks`); + return true; + } + + const existingChunks = fs.readdirSync(chunksDir).length; + if (existingChunks !== numChunks) { + console.log(` ・ Recreating chunks for '${item.key}' - found ${existingChunks} existing chunk(s), expected ${numChunks}`); + return true; + } + + const finalChunkSize = item.size % chunkSize; + for (let chunk = 0; chunk < numChunks; chunk++) { + const chunkFileName = `${chunksDir}/${chunk}`; + // If the chunk does not exist + if (!fs.existsSync(chunkFileName)) { + console.log(` ・ Recreating chunks for '${item.key}' - chunk ${chunk} does not exist`); + return true; + } + + // or if the chunk isn't the expected size + const expectedFileSize = chunk === numChunks - 1 ? finalChunkSize : chunkSize; + + const existingFileSize = getFileSize(chunkFileName); + if (existingFileSize !== expectedFileSize) { + console.log(` ・ Recreating chunks for '${item.key}' - chunk ${chunk} existing file size ${existingFileSize} does not match expected size ${expectedFileSize}`); + return true; + } + } + + console.log(` ・ Existing chunks for '${item.key}' look good.`); + return false; +} + +export async function applyKVStoreEntriesChunks(kvStoreItemDescriptions: KVStoreItemDesc[], chunkSize: number) { + + const items = kvStoreItemDescriptions.splice(0); + // kvStoreItemDescriptions is now empty. + + for (const item of items) { + if (item.size <= chunkSize) { + // If file is not over the chunk size, use it as it is. + kvStoreItemDescriptions.push(item); + continue; + } + + // Check if the chunks exist on disk, and if they don't, recreate them. + + const chunksDir = `${item.filePath}_chunks`; + const numChunks = Math.ceil(item.size / chunkSize); + + if (shouldRecreateChunks(chunksDir, numChunks, item, chunkSize)) { + // Recreate chunks + fs.rmSync(chunksDir, { recursive: true, force: true }); + fs.mkdirSync(chunksDir, { recursive: true }); + + const numWrittenChunks = await new Promise((resolve, reject) => { + const chunkPaths: string[] = []; + let chunkIndex = 0; + let bytesWritten = 0; + let currentStream: fs.WriteStream | null = null; + + const inputStream = fs.createReadStream(item.filePath); + inputStream.on('error', reject); + + inputStream.on('data', (chunk) => { + let offset = 0; + + while (offset < chunk.length) { + if (!currentStream || bytesWritten >= chunkSize) { + if (currentStream != null) { + currentStream.end(); + } + const chunkFileName = `${chunksDir}/${chunkIndex}`; + currentStream = fs.createWriteStream(chunkFileName); + chunkPaths.push(chunkFileName); + chunkIndex++; + bytesWritten = 0; + } + + const bytesLeftInChunk = chunkSize - bytesWritten; + const bytesLeftInBuffer = chunk.length - offset; + const bytesToWrite = Math.min(bytesLeftInChunk, bytesLeftInBuffer); + const slice = chunk.slice(offset, offset + bytesToWrite); + + currentStream.write(slice); + bytesWritten += bytesToWrite; + offset += bytesToWrite; + } + }); + + inputStream.on('end', () => { + if (currentStream != null) { + currentStream.end(); + } + resolve(chunkPaths.length); + }); + }); + + if (numWrittenChunks !== numChunks) { + throw new Error(`numWrittenChunks (${numWrittenChunks}) does not equal numChunks (${numChunks})`); + } + } + + for (let chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) { + + const kvItem = structuredClone(item); + + // Source the chunk from the chunk directory + kvItem.filePath = `${chunksDir}/${chunkIndex}`; + + // Add chunk ID to metadata + if (kvItem.metadataJson == null) { + kvItem.metadataJson = {}; + } + kvItem.metadataJson.chunkIndex = chunkIndex; + + if (chunkIndex !== 0) { + + // For additional chunks (all but the first): + // add suffix to key + kvItem.key = item.key + '_' + chunkIndex; + // remove other metadata + delete kvItem.metadataJson.hash; + delete kvItem.metadataJson.size; + delete kvItem.metadataJson.numChunks; + + } + + kvStoreItemDescriptions.push(kvItem); + + } + } + +} diff --git a/src/cli/util/kv-store-local-server.ts b/src/cli/util/kv-store-local-server.ts new file mode 100644 index 0000000..5750ba8 --- /dev/null +++ b/src/cli/util/kv-store-local-server.ts @@ -0,0 +1,46 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { type KVStoreItemDesc } from './kv-store-items.js'; + +export function writeKVStoreEntriesForLocal(storeFile: string, computeAppDir: string, kvStoreItemDescriptions: KVStoreItemDesc[]) { + + type KVStoreLocalServerEntry = ({ data: string, } | { file: string }) & { metadata?:string }; + type KVStoreLocalServerData = Record; + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + for (const kvStoreItemDescription of kvStoreItemDescriptions) { + store[kvStoreItemDescription.key] = { + file: path.relative(computeAppDir, kvStoreItemDescription.filePath), + metadata: kvStoreItemDescription.metadataJson != null ? JSON.stringify(kvStoreItemDescription.metadataJson) : undefined, + }; + } + + // Delete any keys that point to items that do not exist in the file system + const keysToDelete = new Set(); + for (const [key, value] of Object.entries(store)) { + if (!("file" in value) || fs.existsSync(path.resolve(computeAppDir, value.file))) { + continue; + } + keysToDelete.add(key); + } + for (const key of keysToDelete) { + delete store[key]; + } + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} diff --git a/src/cli/util/package.ts b/src/cli/util/package.ts new file mode 100644 index 0000000..6b86dfb --- /dev/null +++ b/src/cli/util/package.ts @@ -0,0 +1,63 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +export type PackageJson = { + author?: string, + name?: string, + description?: string, + dependencies?: Record, + devDependencies?: Record, +}; + +export function findComputeJsStaticPublisherVersion(packageJson: PackageJson | null) { + + // Get the current compute js static publisher version. + let computeJsStaticPublisherVersion: string | null = null; + if (packageJson != null) { + // First try current project's package.json + computeJsStaticPublisherVersion = + packageJson.dependencies?.["@fastly/compute-js-static-publish"] ?? + packageJson.devDependencies?.["@fastly/compute-js-static-publish"] ?? + null; + + // This may be a file url if during development + if (computeJsStaticPublisherVersion != null) { + + if (computeJsStaticPublisherVersion.startsWith('file:')) { + // this is a relative path from the current directory. + // we replace it with an absolute path + const relPath = computeJsStaticPublisherVersion.slice('file:'.length); + const absPath = path.resolve(relPath); + computeJsStaticPublisherVersion = 'file:' + absPath; + } + + } + } + + if (computeJsStaticPublisherVersion == null) { + // Also try package.json of the package that contains the currently running program + // This is used when the program doesn't actually install the package (running via npx). + const computeJsStaticPublishPackageJsonPath = path.resolve(import.meta.dirname, '../../../package.json'); + const computeJsStaticPublishPackageJsonText = fs.readFileSync(computeJsStaticPublishPackageJsonPath, 'utf-8'); + const computeJsStaticPublishPackageJson = JSON.parse(computeJsStaticPublishPackageJsonText); + computeJsStaticPublisherVersion = computeJsStaticPublishPackageJson?.version; + } + + if (computeJsStaticPublisherVersion == null) { + // Unexpected, but if it's still null then we go to a literal + computeJsStaticPublisherVersion = '7.0.0'; + } + + if (!computeJsStaticPublisherVersion.startsWith('^') && + !computeJsStaticPublisherVersion.startsWith('file:') + ) { + computeJsStaticPublisherVersion = '^' + computeJsStaticPublisherVersion; + } + + return computeJsStaticPublisherVersion; +} diff --git a/src/cli/util/publish-id.ts b/src/cli/util/publish-id.ts deleted file mode 100644 index b83233e..0000000 --- a/src/cli/util/publish-id.ts +++ /dev/null @@ -1,34 +0,0 @@ -import fs from "fs"; -import { createStringId } from "./id.js"; - -export function generateOrLoadPublishId() { - - let created = false; - - const filename = './.publish-id'; - let contents: string | null; - try { - contents = fs.readFileSync(filename, 'utf-8'); - } catch { - contents = null; - } - - let publishId: string | null = null; - if (contents != null) { - publishId = contents - .split('\n') - .find(line => line.length > 0 && !line.startsWith('#')) ?? null; - } - - if (publishId == null) { - - publishId = createStringId(); - created = true; - const fileContents = `# Generated by @fastly/compute-js-static-publish.\n${publishId}\n`; - fs.writeFileSync(filename, fileContents, 'utf-8'); - - } - - return { publishId, created }; - -} diff --git a/src/cli/util/retryable.ts b/src/cli/util/retryable.ts index fe4e543..90a2cbc 100644 --- a/src/cli/util/retryable.ts +++ b/src/cli/util/retryable.ts @@ -1,3 +1,8 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + let _globalBackoffUntil = 0; const retryableSymbol = Symbol(); diff --git a/src/cli/util/variants.ts b/src/cli/util/variants.ts new file mode 100644 index 0000000..b114102 --- /dev/null +++ b/src/cli/util/variants.ts @@ -0,0 +1,41 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'node:fs'; +import { type ContentCompressionTypes } from '../../models/compression/index.js'; +import { algs } from '../compression/index.js'; +import { rootRelative } from './files.js'; + +export type Variants = 'original' | ContentCompressionTypes; + +export async function ensureVariantFileExists( + variantFilePath: string, + variant: Variants, + file: string, +) { + + // Compress/prepare the asset if it doesn't already exist + // We disregard chunked copies of the file and only check for the existence + // of the main copy of the file at this point. + if (!fs.existsSync(variantFilePath)) { + if (variant === 'original') { + + fs.cpSync(file, variantFilePath); + console.log(` 📄→📄 Copied file '${rootRelative(file)}' to '${rootRelative(variantFilePath)}'.`); + + } else { + + const compressTo = algs[variant]; + await compressTo(file, variantFilePath); + console.log(` 📄→🗄️ Compressed file '${rootRelative(file)}' to '${rootRelative(variantFilePath)}' [${variant}].`); + + } + + // However, if we did just create the file, + // we delete any chunked copies that may exist as they can be out of date. + // (They will be recreated in a later step) + fs.rmSync(`${variantFilePath}_chunks`, { recursive: true, force: true }); + } +} diff --git a/src/compute-js.ts b/src/compute-js.ts deleted file mode 100644 index b07fa74..0000000 --- a/src/compute-js.ts +++ /dev/null @@ -1,2 +0,0 @@ -import './server/assets/content-asset-wasm-inline.js'; -import './server/assets/content-asset-kv-store.js'; diff --git a/src/constants/compression.ts b/src/constants/compression.ts deleted file mode 100644 index d959bc4..0000000 --- a/src/constants/compression.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const compressionTypes = [ - 'br', - 'gzip', -] as const; - -export type ContentCompressionTypes = typeof compressionTypes[number]; diff --git a/src/index.ts b/src/index.ts index 9b7352c..ad8dfb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,13 @@ -/// +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ -export * from './types/index.js'; -export * from './util/index.js'; - -export { ContentAssets } from './server/assets/content-assets.js'; -export { ModuleAssets } from './server/assets/module-assets.js'; -export { PublisherServer } from './server/publisher-server.js'; +export { type StaticPublishRc } from './models/config/static-publish-rc.js'; +export { type PublishContentConfig } from './models/config/publish-content-config.js'; +export * as collectionSelector from './server/collection-selector/index.js'; +export { + CookieCollectionSelector, + type CookieCollectionSelectorOpts, +} from './server/collection-selector/from-cookie.js'; +export { PublisherServer } from './server/publisher-server/index.js'; diff --git a/src/models/assets/kvstore-assets.ts b/src/models/assets/kvstore-assets.ts new file mode 100644 index 0000000..194f612 --- /dev/null +++ b/src/models/assets/kvstore-assets.ts @@ -0,0 +1,50 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type ContentCompressionTypes, isCompressionType } from '../compression/index.js'; + +export type KVAssetEntry = { + key: string, + size: number, + contentType: string, + lastModifiedTime: number, + variants: ContentCompressionTypes[], +}; + +export type KVAssetEntryMap = Record; + +export type KVAssetVariantMetadata = { + contentEncoding?: ContentCompressionTypes, + size: number, + hash: string, + numChunks?: number, +}; + +export function isKVAssetVariantMetadata(obj: unknown): obj is KVAssetVariantMetadata { + if (obj == null || typeof obj !== 'object') { + return false; + } + if ('contentEncoding' in obj) { + if ( + typeof obj.contentEncoding !== 'string' || + !isCompressionType(obj.contentEncoding) + ) { + return false; + } + } + if (!('size' in obj) || typeof obj.size !== 'number') { + return false; + } + if (!('hash' in obj) || typeof obj.hash !== 'string') { + return false; + } + if ('numChunks' in obj) { + if (typeof obj.numChunks !== 'number') { + return false; + } + } + + return true; +} diff --git a/src/models/compression/index.ts b/src/models/compression/index.ts new file mode 100644 index 0000000..0eb8d67 --- /dev/null +++ b/src/models/compression/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export const compressionTypes = [ + 'br', + 'gzip', +] as const; + +export type ContentCompressionTypes = typeof compressionTypes[number]; + +export function isCompressionType(value: string): value is ContentCompressionTypes { + return (compressionTypes as Readonly).includes(value); +} diff --git a/src/models/config/publish-content-config.ts b/src/models/config/publish-content-config.ts new file mode 100644 index 0000000..b7fb365 --- /dev/null +++ b/src/models/config/publish-content-config.ts @@ -0,0 +1,91 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { + type PublisherServerConfig, + type PublisherServerConfigNormalized, +} from './publisher-server-config.js'; +import { + type ContentCompressionTypes, +} from '../compression/index.js'; + +export type ContentTypeTest = (name: string) => boolean; + +// Content Type definition +export type ContentTypeDef = { + // A test on the asset key to perform on this content type. + test: RegExp | ContentTypeTest, + + // The Content-Type header value to provide for this content type. + contentType: string, + + // Whether this content type represents a text value encoded in utf-8. + // If so, conveniences can be provided. + text?: boolean, + + // Binary formats are usually not candidates for compression, but certain + // types can benefit from it. + precompressAsset?: boolean, +}; + +export type ContentTypeTestResult = { + contentType: string, + text: boolean, + precompressAsset: boolean, +}; + +export interface ExcludeDirTest { + test(name: string): boolean; +} + +export type KVStoreAssetInclusionTest = (assetKey: string, contentType?: string) => boolean; + +export type PublishContentConfig = { + // Set to a directory that acts as the root of all files that will be included in this publish. + rootDir: string, + + // Set to a directory that will hold the working files built by compute-js-static-publish + // These files should not be committed to source control. + staticPublisherWorkingDir: string, + + // An array of values used to exclude files and directories (as well as files within those directories) from being + // included in this publish. Each entry in the array can be a string or a RegExp and will be tested against the relative + // path from 'rootDir' of each file or directory. + // Defaults to [ './node_modules' ]. Set to an empty array or specifically to null to include all files. + excludeDirs?: (string | ExcludeDirTest)[] | string | ExcludeDirTest | null, + + // If true, then files whose names begin with a dot, as well as files in directories whose names begin with a .dot, + // are excluded from this publish. Defaults to true. + excludeDotFiles?: boolean, + + // If true, include .well-known even if excludeDotFiles is true. + // Defaults to true. + includeWellKnown?: boolean, + + // A test to run on each asset key to determine whether and how to include the file. + kvStoreAssetInclusionTest?: KVStoreAssetInclusionTest | null, + + // Pre-generate content in these formats as well and serve them in tandem with the + // allowedEncodings setting in the server settings. Default value is [ 'br' | 'gzip' ]. + contentCompression?: ('br' | 'gzip')[], + + // Additional / override content types. + contentTypes?: ContentTypeDef[], + + // Server settings + server?: PublisherServerConfig | null, +}; + +export type PublishContentConfigNormalized = { + rootDir: string, + staticPublisherWorkingDir: string, + excludeDirs: ExcludeDirTest[], + excludeDotFiles: boolean, + includeWellKnown: boolean, + kvStoreAssetInclusionTest: KVStoreAssetInclusionTest | null, + contentCompression: ContentCompressionTypes[], + contentTypes: ContentTypeDef[], + server: PublisherServerConfigNormalized | null, +}; diff --git a/src/models/config/publisher-server-config.ts b/src/models/config/publisher-server-config.ts new file mode 100644 index 0000000..8be9d4a --- /dev/null +++ b/src/models/config/publisher-server-config.ts @@ -0,0 +1,52 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type ContentCompressionTypes } from '../compression/index.js'; + +// Publisher Server configuration +export type PublisherServerConfig = { + // Prefix to apply to web requests. Effectively, a directory within rootDir that is used + // by the web server to determine the asset to respond with. Defaults to the empty string. + publicDirPrefix?: string, + + // A test to apply to item names to decide whether to serve them as "static" files, in other + // words, with a long TTL. These are used for files that are not expected to change. + // They can be provided as a string or array of strings. + // Items that contain asterisks, are interpreted as glob patterns. + // Items that end with a trailing slash are interpreted as directory names, + // Items that don't contain asterisks and that do not end in slash are checked for exact match. + staticItems?: string[] | string | false | null, + + // Allowed encodings. If the request contains an Accept-Encoding header, they are checked in the order listed + // in the header for the values listed here. The compression algorithm represented by the first match is applied. + // Default value is [ 'br', 'gzip' ]. + allowedEncodings?: ContentCompressionTypes[], + + // Set to the asset key of a content item to serve this when a GET request comes in for an unknown asset, and + // the Accept header includes text/html. + spaFile?: string | false | null, + + // Set to the asset key of a content item to serve this when a request comes in for an unknown asset, and + // the Accept header includes text/html. + notFoundPageFile?: string | false | null, + + // When a file is not found, and it doesn't end in a slash, then try auto-ext: try to serve a file with the same name + // postfixed with the specified strings, tested in the order listed. + autoExt?: string[] | string | false | null, + + // When a file is not found, then try auto-index: treat it as a directory, then try to serve a file that has the + // specified strings, tested in the order listed. + autoIndex?: string[] | string | false | null, +}; + +export type PublisherServerConfigNormalized = { + publicDirPrefix: string, + staticItems: string[], + allowedEncodings: ContentCompressionTypes[], + spaFile: string | null, + notFoundPageFile: string | null, + autoExt: string[], + autoIndex: string[], +}; diff --git a/src/models/config/static-publish-rc.ts b/src/models/config/static-publish-rc.ts new file mode 100644 index 0000000..1e4b122 --- /dev/null +++ b/src/models/config/static-publish-rc.ts @@ -0,0 +1,16 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type StaticPublishRc = { + // Name of KV Store to use. + kvStoreName: string, + + // Publish ID. This is a unique string "prefix" that identifies your assets in the + // KV Store. This is generated by init-app and is usually not to be changed. + publishId: string; + + // Default collection name. + defaultCollectionName: string, +} diff --git a/src/server/assets/asset-manager.ts b/src/server/assets/asset-manager.ts deleted file mode 100644 index 35b676c..0000000 --- a/src/server/assets/asset-manager.ts +++ /dev/null @@ -1,21 +0,0 @@ -export abstract class AssetManager { - - protected readonly assets: Record; - - protected constructor() { - this.assets = {}; - } - - protected initAsset(assetKey: string, asset: TAsset) { - this.assets[assetKey] = asset; - } - - getAsset(assetKey: string): TAsset | null { - return this.assets[assetKey] ?? null; - } - - getAssetKeys(): string[] { - return Object.keys(this.assets); - } - -} diff --git a/src/server/assets/content-asset-kv-store.ts b/src/server/assets/content-asset-kv-store.ts deleted file mode 100644 index 6ebb5b7..0000000 --- a/src/server/assets/content-asset-kv-store.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { KVStore } from "fastly:kv-store"; -import { - ContentAsset, - ContentAssetMetadataMapEntry, - StoreEntry, -} from "../../types/index.js"; -import { - CompressedFileInfos, - ContentAssetMetadataMapEntryKVStore, -} from "../../types/content-assets.js"; -import { ContentCompressionTypes } from "../../constants/compression.js"; -import { ContentAssets, findMatchingSourceAndInfo, SourceAndInfo } from "./content-assets.js"; - -export class ContentKVStoreAsset implements ContentAsset { - readonly type = 'kv-store'; - - private readonly metadata: ContentAssetMetadataMapEntryKVStore; - - private readonly kvStoreName: string; - - private readonly sourceAndInfo: SourceAndInfo; - private readonly compressedSourcesAndInfo: CompressedFileInfos>; - - constructor(metadata: ContentAssetMetadataMapEntryKVStore, kvStoreName: string) { - this.metadata = metadata; - this.kvStoreName = kvStoreName; - - this.sourceAndInfo = { - source: metadata.fileInfo.kvStoreKey, - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }; - - this.compressedSourcesAndInfo = Object.entries(metadata.compressedFileInfos) - .reduce>>((obj, [key, value]) => { - obj[key as ContentCompressionTypes] = { - source: value.kvStoreKey, - hash: value.hash, - size: value.size, - }; - return obj; - }, {}); - } - - get isLocal() { - return false; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { - - const { sourceAndInfo, contentEncoding } = findMatchingSourceAndInfo(acceptEncodingsGroups, this.sourceAndInfo, encoding => this.compressedSourcesAndInfo[encoding]); - - const kvStore = new KVStore(this.kvStoreName); - let retries = 3; - while (retries > 0) { - const storeEntry = await kvStore.get(sourceAndInfo.source); - if (storeEntry != null) { - const { hash, size } = sourceAndInfo; - return Object.assign(storeEntry, { contentEncoding, hash, size }); - } - - // Note the null does NOT mean 404. The fact that we are here means - // metadata exists for this path, and we're just trying to get the data from - // the KV Store. - - // So if we're here then the data is either not available yet (in which case - // we can wait just a bit and try again), or the data was deleted. - retries--; - - // We're going to wait 250ms/500ms/750ms and try again. - const delay = (3-retries) * 250; - await new Promise(resolve => setTimeout(resolve, delay)); - } - throw new Error("Asset could not be retrieved from the KV Store."); - } - - getBytes(): Uint8Array { - throw new Error("Can't getBytes() for KV Store asset"); - } - - getText(): string { - throw new Error("Can't getText() for KV Store asset"); - } - - getJson(): T { - throw new Error("Can't getJson() for KV Store asset"); - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } - -} - -ContentAssets.registerAssetBuilder('kv-store', (metadata, context) => { - const { kvStoreName } = context; - - if (kvStoreName == null) { - throw new Error("Unexpected! KV Store name should be specified!!"); - } - - return new ContentKVStoreAsset(metadata, kvStoreName); -}); diff --git a/src/server/assets/content-asset-wasm-inline.ts b/src/server/assets/content-asset-wasm-inline.ts deleted file mode 100644 index 1a05fb3..0000000 --- a/src/server/assets/content-asset-wasm-inline.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { includeBytes } from "fastly:experimental"; -import { - ContentAsset, - ContentAssetMetadataMapEntry, - StoreEntry, -} from "../../types/index.js"; -import { - CompressedFileInfos, - ContentAssetMetadataMapEntryWasmInline, -} from "../../types/content-assets.js"; -import { ContentCompressionTypes } from "../../constants/compression.js"; -import { InlineStoreEntry } from "../kv-store/inline-store-entry.js"; -import { ContentAssets, findMatchingSourceAndInfo, SourceAndInfo } from "./content-assets.js"; - -const decoder = new TextDecoder(); - -export class ContentWasmInlineAsset implements ContentAsset { - readonly type = 'wasm-inline'; - - private readonly metadata: ContentAssetMetadataMapEntryWasmInline; - - private readonly sourceAndInfo: SourceAndInfo; - private readonly compressedSourcesAndInfo: CompressedFileInfos>; - - constructor(metadata: ContentAssetMetadataMapEntryWasmInline) { - this.metadata = metadata; - - this.sourceAndInfo = { - source: includeBytes(metadata.fileInfo.staticFilePath), - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }; - - this.compressedSourcesAndInfo = Object.entries(metadata.compressedFileInfos) - .reduce>>((obj, [key, value]) => { - obj[key as ContentCompressionTypes] = { - source: includeBytes(value.staticFilePath), - hash: value.hash, - size: value.size, - }; - return obj; - }, {}); - } - - get isLocal() { - return true; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { - const { sourceAndInfo, contentEncoding } = findMatchingSourceAndInfo(acceptEncodingsGroups, this.sourceAndInfo, encoding => this.compressedSourcesAndInfo[encoding]); - const { source, hash, size } = sourceAndInfo; - return new InlineStoreEntry(source, contentEncoding, hash, size); - } - - getBytes(): Uint8Array { - return this.sourceAndInfo.source; - } - - getText(): string { - if (!this.metadata.text) { - throw new Error("Can't getText() for non-text content"); - } - return decoder.decode(this.sourceAndInfo.source); - } - - getJson(): T { - const text = this.getText(); - return JSON.parse(text) as T; - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } -} - -ContentAssets.registerAssetBuilder('wasm-inline', metadata => new ContentWasmInlineAsset(metadata)); diff --git a/src/server/assets/content-assets.ts b/src/server/assets/content-assets.ts deleted file mode 100644 index 18b4548..0000000 --- a/src/server/assets/content-assets.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { AssetManager } from "./asset-manager.js"; -import { InlineStoreEntry } from "../kv-store/inline-store-entry.js"; - -import type { - ContentAsset, - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, - ContentAssetMetadataMapEntryBytes, - ContentAssetMetadataMapEntryString, -} from "../../types/content-assets.js"; -import type { StoreEntry } from "../../types/compute.js"; -import type { ContentCompressionTypes } from "../../constants/compression.js"; - -const decoder = new TextDecoder(); - -export type SourceAndInfo = { - source: TSource, - hash: string, - size: number, -}; - -type SourceAndInfoForEncodingFn = (contentEncoding: ContentCompressionTypes) => SourceAndInfo | undefined; -export function findMatchingSourceAndInfo(acceptEncodingsGroups: ContentCompressionTypes[][], defaultSourceAndInfo: SourceAndInfo, sourceAndInfoForEncodingFn: SourceAndInfoForEncodingFn) { - - let sourceAndInfo: SourceAndInfo | undefined; - let contentEncoding: ContentCompressionTypes | null = null; - if (acceptEncodingsGroups != null) { - for(const encodingGroup of acceptEncodingsGroups) { - const sourceAndInfosForEncodingsGroup = encodingGroup - .map(encoding => ({ encoding, sourceAndInfo: sourceAndInfoForEncodingFn(encoding) })) - .filter(x => x.sourceAndInfo != null) as {encoding: ContentCompressionTypes, sourceAndInfo: SourceAndInfo}[]; - - if (sourceAndInfosForEncodingsGroup.length === 0) { - // If no encoding in this group is available then move to next group - continue; - } - - // Sort the items, putting the smallest size first - sourceAndInfosForEncodingsGroup - .sort((a, b) => a.sourceAndInfo.size - b.sourceAndInfo.size); - - // The first item is the one we want. - sourceAndInfo = sourceAndInfosForEncodingsGroup[0].sourceAndInfo; - contentEncoding = sourceAndInfosForEncodingsGroup[0].encoding; - break; - } - } - - sourceAndInfo ??= defaultSourceAndInfo; - - return { sourceAndInfo, contentEncoding }; -} - -abstract class ContentRuntimeAsset { - protected readonly metadata: TMetadataMapEntry; - protected sourceAndInfo: SourceAndInfo; - protected constructor(metadata: TMetadataMapEntry, sourceAndInfo: SourceAndInfo) { - this.metadata = metadata; - this.sourceAndInfo = sourceAndInfo; - } - - get isLocal() { - return true; - } - - get assetKey() { - return this.metadata.assetKey; - } - - async getStoreEntry(): Promise { - const { source, hash, size } = this.sourceAndInfo; - return new InlineStoreEntry(source, null, hash, size); - } - - getBytes(): Uint8Array { - return this.sourceAndInfo.source; - } - - getText(): string { - if (!this.metadata.text) { - throw new Error("Can't getText() for non-text content"); - } - return decoder.decode(this.sourceAndInfo.source); - } - - getJson(): T { - const text = this.getText(); - return JSON.parse(text) as T; - } - - getMetadata(): ContentAssetMetadataMapEntry { - return this.metadata; - } -} - -export class ContentBytesAsset - extends ContentRuntimeAsset - implements ContentAsset { - readonly type = 'bytes'; - - constructor(metadata: ContentAssetMetadataMapEntryBytes) { - super(metadata, { - source: metadata.fileInfo.bytes, - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }); - } -} - -export class ContentStringAsset - extends ContentRuntimeAsset - implements ContentAsset { - readonly type = 'string'; - - static encoder = new TextEncoder(); - constructor(metadata: ContentAssetMetadataMapEntryString) { - super(metadata, { - source: ContentStringAsset.encoder.encode(metadata.fileInfo.content), - hash: metadata.fileInfo.hash, - size: metadata.fileInfo.size, - }); - } -} - -export type AssetBuilder = (metadata: ContentAssetMetadataMapEntry & { type: TType }, assetBuilderContext: AssetBuilderContext) => ContentAsset; -type AssetBuilderContext = any; - -export class ContentAssets extends AssetManager { - - static builders: Record> = {}; - - constructor(contentAssetMetadataMap: ContentAssetMetadataMap, assetBuilderContext: AssetBuilderContext = {}) { - super(); - - for (const [assetKey, metadata] of Object.entries(contentAssetMetadataMap)) { - - if (!(metadata.type in ContentAssets.builders)) { - throw new Error(`Unknown content asset type '${metadata.type}'`); - } - - const asset = ContentAssets.builders[metadata.type](metadata, assetBuilderContext); - this.initAsset(assetKey, asset); - } - } - - static registerAssetBuilder(type: TType, builder: AssetBuilder) { - this.builders[type] = builder; - } -} - -ContentAssets.registerAssetBuilder('bytes', metadata => new ContentBytesAsset(metadata)); -ContentAssets.registerAssetBuilder('string', metadata => new ContentStringAsset(metadata)); diff --git a/src/server/assets/module-assets.ts b/src/server/assets/module-assets.ts deleted file mode 100644 index 1baf699..0000000 --- a/src/server/assets/module-assets.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { AssetManager } from "./asset-manager.js"; - -import type { ModuleAsset, ModuleAssetMap } from "../../types/module-assets.js"; - -export class ModuleAssetDynamic implements ModuleAsset { - readonly assetKey: string; - readonly isStaticImport: boolean = false; - - private readonly loadModule: () => Promise; - - constructor(assetKey: string, loadModule: () => Promise) { - this.assetKey = assetKey; - this.loadModule = loadModule; - } - - private _modulePromise: Promise | undefined; - getModule(): Promise { - if (this._modulePromise === undefined) { - this._modulePromise = this.loadModule(); - } - return this._modulePromise; - } - - getStaticModule(): any { - return null; - } - -} - -export class ModuleAssetStatic implements ModuleAsset { - readonly assetKey: string; - readonly isStaticImport: boolean = true; - - private readonly module: any; - constructor(assetKey: string, module: any) { - this.assetKey = assetKey; - this.module = module; - } - - getModule(): Promise { - return Promise.resolve(this.module); - } - - getStaticModule(): any { - return this.module; - } - -} - -export class ModuleAssets extends AssetManager { - - constructor(moduleAssetMap: ModuleAssetMap) { - super(); - - for (const [assetKey, moduleEntry] of Object.entries(moduleAssetMap)) { - - let asset: ModuleAsset; - if (moduleEntry.isStaticImport) { - - asset = new ModuleAssetStatic(assetKey, moduleEntry.module); - - } else { - - asset = new ModuleAssetDynamic(assetKey, moduleEntry.loadModule); - - } - - this.initAsset(assetKey, asset); - } - } -} diff --git a/src/server/collection-selector/from-config-store.ts b/src/server/collection-selector/from-config-store.ts new file mode 100644 index 0000000..a87b11a --- /dev/null +++ b/src/server/collection-selector/from-config-store.ts @@ -0,0 +1,18 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { ConfigStore } from 'fastly:config-store'; + +export function fromConfigStore(configStoreName: string, key: string) { + + try { + const configStore = new ConfigStore(configStoreName); + return configStore.get(key); + } catch { + // If there is no config store + return null; + } + +} diff --git a/src/server/collection-selector/from-cookie.ts b/src/server/collection-selector/from-cookie.ts new file mode 100644 index 0000000..34873b0 --- /dev/null +++ b/src/server/collection-selector/from-cookie.ts @@ -0,0 +1,136 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { getCookieValue } from '../util/cookies.js'; +import { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; + +export type CookieCollectionSelectorOpts = { + cookieName: string, + activatePath: string, + resetPath: string, + cookieHttpOnly: boolean, + cookieMaxAge: number | undefined, + cookiePath: string, +}; + +export class CookieCollectionSelector implements RequestCollectionNameSelector { + public constructor(); + public constructor(cookieName: string); + public constructor(opts: Partial); + public constructor(optsOrCookieName?: string | Partial) { + let opts = optsOrCookieName; + if (opts === undefined) { + opts = {}; + } else if (typeof opts === 'string') { + opts = { + cookieName: opts, + }; + } + + this.opts = { + cookieName: opts.cookieName ?? 'publisher-collection', + activatePath: opts.activatePath ?? '/activate', + resetPath: opts.resetPath ?? '/reset', + cookieHttpOnly: opts.cookieHttpOnly ?? true, + cookieMaxAge: opts.cookieMaxAge, + cookiePath: opts.cookiePath ?? '/', + }; + } + + public opts: CookieCollectionSelectorOpts; + + public getCollectionName(request: Request): CollectionNameSelectorResult { + return getCookieValue(request, this.opts.cookieName); + } + + public applySelector(request: Request) { + + const url = new URL(request.url); + + if (url.pathname === this.opts.activatePath) { + const collectionName = url.searchParams.get('collection'); + if (collectionName == null) { + return new Response(`Missing required 'collection' query parameter.`, { status: 400 }); + } + const redirectTo = url.searchParams.get('redirectTo') ?? '/'; + + const cookieSegments = [ + `${this.opts.cookieName}=${collectionName}`, + ]; + cookieSegments.push(`Path=${this.opts.cookiePath}`); + if (this.opts.cookieMaxAge != null) { + cookieSegments.push(`Max-Age=${this.opts.cookieMaxAge}`); + } + if (this.opts.cookieHttpOnly) { + cookieSegments.push('HttpOnly'); + } + if (url.protocol === 'https:') { + cookieSegments.push('Secure'); + } + cookieSegments.push(`SameSite=Lax`); + + const headers = new Headers(); + headers.append("Set-Cookie", cookieSegments.join(';')); + headers.set("Location", redirectTo); + + return new Response(null, { + status: 302, + headers, + }); + } + + if (url.pathname === this.opts.resetPath) { + const redirectTo = url.searchParams.get('redirectTo') ?? '/'; + + const cookieSegments = [ + `${this.opts.cookieName}=`, + ]; + cookieSegments.push(`Path=${this.opts.cookiePath}`); + cookieSegments.push(`Expires=Thu, 01 Jan 1970 00:00:00 GMT`); + if (this.opts.cookieHttpOnly) { + cookieSegments.push('HttpOnly'); + } + if (url.protocol === 'https:') { + cookieSegments.push('Secure'); + } + + const headers = new Headers(); + headers.append("Set-Cookie", cookieSegments.join(';')); + headers.set("Location", redirectTo); + + return new Response(null, { + status: 302, + headers, + }); + } + + return null; + } +} + +export interface FromCookieResult { + collectionName: CollectionNameSelectorResult, + redirectResponse: Response | null, +} + +export function fromCookie(request: Request, optsOrCookieName?: string | CookieCollectionSelectorOpts): FromCookieResult { + + let selector; + if (optsOrCookieName === undefined) { + selector = new CookieCollectionSelector(); + } else if (typeof optsOrCookieName === 'string') { + selector = new CookieCollectionSelector(optsOrCookieName); + } else { + selector = new CookieCollectionSelector(optsOrCookieName); + } + + return { + collectionName: selector.getCollectionName(request), + redirectResponse: selector.applySelector(request), + }; +} diff --git a/src/server/collection-selector/from-request.ts b/src/server/collection-selector/from-request.ts new file mode 100644 index 0000000..5b13454 --- /dev/null +++ b/src/server/collection-selector/from-request.ts @@ -0,0 +1,47 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; + +export type RequestToCollectionNameFunc = (request: Request) => CollectionNameSelectorResult; +type UrlToCollectionNameFunc = (url: URL) => string | null; + + +export class RequestCollectionNameFuncSelector implements RequestCollectionNameSelector { + public constructor( + func: RequestToCollectionNameFunc + ) { + this.func = func; + } + + public func: RequestToCollectionNameFunc; + + public getCollectionName(request: Request) { + return this.func(request); + } +} + +export function fromRequest(request: Request, func: RequestToCollectionNameFunc): CollectionNameSelectorResult { + + const selector = new RequestCollectionNameFuncSelector(func); + return selector.getCollectionName(request); + +} + +export function fromRequestUrl(request: Request, func: UrlToCollectionNameFunc): CollectionNameSelectorResult { + + return fromRequest(request, (req) => func(new URL(req.url))); + +} + +export function fromHostDomain(request: Request, hostRegex: RegExp): CollectionNameSelectorResult { + return fromRequestUrl( + request, + (url) => url.host.match(hostRegex)?.[1] ?? null, + ); +} diff --git a/src/server/collection-selector/index.ts b/src/server/collection-selector/index.ts new file mode 100644 index 0000000..444eba4 --- /dev/null +++ b/src/server/collection-selector/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export { + type CollectionNameSelectorResult, + type RequestCollectionNameSelector, +} from './request-collection-selector.js'; +export { + RequestCollectionNameFuncSelector, + fromRequest, + fromRequestUrl, + fromHostDomain, +} from './from-request.js'; +export { + CookieCollectionSelector, + fromCookie, + type CookieCollectionSelectorOpts, + type FromCookieResult, +} from './from-cookie.js'; +export { + fromConfigStore, +} from './from-config-store.js'; diff --git a/src/server/collection-selector/request-collection-selector.ts b/src/server/collection-selector/request-collection-selector.ts new file mode 100644 index 0000000..b19ab70 --- /dev/null +++ b/src/server/collection-selector/request-collection-selector.ts @@ -0,0 +1,10 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export type CollectionNameSelectorResult = string | null; + +export interface RequestCollectionNameSelector { + getCollectionName(request: Request): CollectionNameSelectorResult; +} diff --git a/src/server/kv-store/inline-store-entry.ts b/src/server/kv-store/inline-store-entry.ts deleted file mode 100644 index f406f58..0000000 --- a/src/server/kv-store/inline-store-entry.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { StoreEntry } from "../../types/compute.js"; - -// A class designed to give a Body-style interface to Uint8Array. -// The intended use case is arrays brought in from includeBytes(). - -export class InlineStoreEntry implements StoreEntry { - _consumed: boolean; - - _body: ReadableStreamForBytes; - - _contentEncoding: string | null; - - _hash: string; - - _size: number; - - constructor(array: Uint8Array, contentEncoding: string | null, hash: string, size: number) { - this._body = new ReadableStreamForBytes(array); - this._consumed = false; - this._contentEncoding = contentEncoding; - this._hash = hash; - this._size = size; - } - - get body(): ReadableStream { - return this._body; - } - - get bodyUsed(): boolean { - return this._consumed; - } - - async arrayBuffer(): Promise { - if (this._consumed) { - throw new Error('Body has already been consumed'); - } - if (this._body.locked) { - throw new Error('The ReadableStream body is already locked and can\'t be consumed'); - } - if (this._body._disturbed) { - throw new Error('Body object should not be disturbed or locked'); - } - this._consumed = true; - - let result = new Uint8Array(0); - const reader = this._body.getReader(); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - const newResult = new Uint8Array(result.length + value.length); - newResult.set(result); - newResult.set(value, result.length); - result = newResult; - } - - return result; - } finally { - reader.releaseLock(); - } - } - - static decoder = new TextDecoder(); - async text(): Promise { - const data = await this.arrayBuffer(); - return InlineStoreEntry.decoder.decode(data); - } - - async json(): Promise { - const text = await this.text(); - return JSON.parse(text); - } - - get contentEncoding() { - return this._contentEncoding; - } - - get hash() { - return this._hash; - } - - get size() { - return this._size; - } -} - -class ReadableStreamForBytes extends ReadableStream { - - // Closest we can get to the "disturbed" flag - _disturbed = false; - - constructor(array: Uint8Array) { - super({ - async start(controller) { - controller.enqueue(array); - controller.close(); - }, - }) - } - - override getReader(): ReadableStreamDefaultReader { - - const reader = super.getReader(); - - const stream = this; - - // Monkey-patch read - const _read = reader.read; - reader.read = async () => { - const result = await _read.call(reader); - if (result.done) { - // NOTE: C@E Request body does not seem to get marked as "disturbed" until - // end of stream is reached either... - stream._disturbed = true; - } - return result; - }; - - // Monkey-patch cancel - const _cancel = reader.cancel; - reader.cancel = async (reason?: any) => { - await _cancel.call(reader, reason); - stream._disturbed = true; - }; - - return reader; - - } - -} diff --git a/src/server/publisher-server.ts b/src/server/publisher-server.ts deleted file mode 100644 index 1c19444..0000000 --- a/src/server/publisher-server.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { ContentAssets } from "./assets/content-assets.js"; -import { checkIfModifiedSince, getIfModifiedSinceHeader } from "./serve-preconditions/if-modified-since.js"; -import { checkIfNoneMatch, getIfNoneMatchHeader } from "./serve-preconditions/if-none-match.js"; -import { buildHeadersSubset } from "./util/http.js"; - -import type { PublisherServerConfigNormalized } from "../types/config-normalized.js"; -import type { ContentAsset } from "../types/content-assets.js"; -import type { ContentCompressionTypes } from "../constants/compression.js"; - -// https://httpwg.org/specs/rfc9110.html#rfc.section.15.4.5 -// The server generating a 304 response MUST generate any of the following header fields that would have been sent in -// a 200 (OK) response to the same request: -// * Content-Location, Date, ETag, and Vary -// * Cache-Control and Expires -const headersToPreserveForUnmodified = ['Content-Location', 'ETag', 'Vary', 'Cache-Control', 'Expires'] as const; - -function requestAcceptsTextHtml(req: Request) { - const accept = (req.headers.get('Accept') ?? '') - .split(',') - .map(x => x.split(';')[0]); - if(!accept.includes('text/html') && !accept.includes('*/*') && accept.includes('*')) { - return false; - } - return true; -} - -type AssetInit = { - status?: number, - headers?: Record, - cache?: 'extended' | 'never' | null, -}; - -export class PublisherServer { - private readonly serverConfig: PublisherServerConfigNormalized; - private readonly staticItems: (string|RegExp)[]; - private readonly contentAssets: ContentAssets; - - public constructor( - serverConfig: PublisherServerConfigNormalized, - contentAssets: ContentAssets, - ) { - this.serverConfig = serverConfig; - this.contentAssets = contentAssets; - - this.staticItems = serverConfig.staticItems - .map((x, i) => { - if (x.startsWith('re:')) { - const fragments = x.slice(3).match(/\/(.*?)\/([a-z]*)?$/i); - if (fragments == null) { - console.warn(`Cannot parse staticItems item index ${i}: '${x}', skipping...`); - return ''; - } - return new RegExp(fragments[1], fragments[2] || ''); - } - return x; - }) - .filter(x => Boolean(x)); - } - - getMatchingAsset(path: string): ContentAsset | null { - const assetKey = this.serverConfig.publicDirPrefix + path; - - if(!assetKey.endsWith('/')) { - // A path that does not end in a slash can match an asset directly - const asset = this.contentAssets.getAsset(assetKey); - if (asset != null) { - return asset; - } - - // ... or, we can try auto-ext: - // looks for an asset that has the specified suffix (usually extension, such as .html) - for (const extEntry of this.serverConfig.autoExt) { - let assetKeyWithExt = assetKey + extEntry; - const asset = this.contentAssets.getAsset(assetKeyWithExt); - if (asset != null) { - return asset; - } - } - } - - if(this.serverConfig.autoIndex.length > 0) { - // try auto-index: - // treats the path as a directory, and looks for an asset with the specified - // suffix (usually an index file, such as index.html) - let assetNameAsDir = assetKey; - // remove all slashes from end, and add one trailing slash - while(assetNameAsDir.endsWith('/')) { - assetNameAsDir = assetNameAsDir.slice(0, -1); - } - assetNameAsDir = assetNameAsDir + '/'; - for (const indexEntry of this.serverConfig.autoIndex) { - let assetKeyIndex = assetNameAsDir + indexEntry; - const asset = this.contentAssets.getAsset(assetKeyIndex); - if (asset != null) { - return asset; - } - } - } - - return null; - } - - findAcceptEncodings(request: Request): ContentCompressionTypes[][] { - if (this.serverConfig.compression.length === 0) { - return []; - } - const found = (request.headers.get('accept-encoding') ?? '') - .split(',') - .map(x => { - let [encodingValue, qValueStr] = x.trim().split(';'); - let qValue; // q value multiplied by 1000 - if (qValueStr == null || !qValueStr.startsWith('q=')) { - // use default of 1.0 - qValue = 1000; - } else { - qValueStr = qValueStr.slice(2); // remove the q= - qValue = parseFloat(qValueStr); - if (Number.isNaN(qValue) || qValue > 1) { - qValue = 1; - } - if (qValue < 0) { - qValue = 0; - } - // q values can have up to 3 decimal digits - qValue = Math.floor(qValue * 1000); - } - return [encodingValue.trim(), qValue] as const; - }) - .filter(([encoding]) => - this.serverConfig.compression.includes(encoding as ContentCompressionTypes) - ); - - const priorityMap = new Map; - for (const [encoding, qValue] of found) { - let typesForQValue = priorityMap.get(qValue); - if (typesForQValue == null) { - typesForQValue = []; - priorityMap.set(qValue, typesForQValue); - } - typesForQValue.push(encoding as ContentCompressionTypes); - } - - // Sort keys, larger numbers to come first - const keysSorted = [...priorityMap.keys()] - .sort((qValueA, qValueB) => qValueB - qValueA); - - return keysSorted - .map(qValue => priorityMap.get(qValue)!); - } - - testExtendedCache(pathname: string) { - return this.staticItems - .some(x => { - if (x instanceof RegExp) { - return x.test(pathname); - } - if (x.endsWith('/')) { - return pathname.startsWith(x); - } - return x === pathname; - }); - } - - handlePreconditions(request: Request, asset: ContentAsset, responseHeaders: Record): Response | null { - // Handle preconditions according to https://httpwg.org/specs/rfc9110.html#rfc.section.13.2.2 - - // A recipient cache or origin server MUST evaluate the request preconditions defined by this specification in the following order: - // 1. When recipient is the origin server and If-Match is present, evaluate the If-Match precondition: - // - if true, continue to step 3 - // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.1) - - // 2. When recipient is the origin server, If-Match is not present, and If-Unmodified-Since is present, evaluate the If-Unmodified-Since precondition: - // - if true, continue to step 3 - // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.4) - - // 3. When If-None-Match is present, evaluate the If-None-Match precondition: - // - if true, continue to step 5 - // - if false for GET/HEAD, respond 304 (Not Modified) - // - if false for other methods, respond 412 (Precondition Failed) - - let skipIfNoneMatch = false; - { - const headerValue = getIfNoneMatchHeader(request); - if (headerValue.length > 0) { - const result = checkIfNoneMatch(responseHeaders['ETag'], headerValue); - if (result) { - skipIfNoneMatch = true; - } else { - return new Response(null, { - status: 304, - headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), - }); - } - } - } - - // 4. When the method is GET or HEAD, If-None-Match is not present, and If-Modified-Since is present, evaluate the - // If-Modified-Since precondition: - // - if true, continue to step 5 - // - if false, respond 304 (Not Modified) - - if (!skipIfNoneMatch) { - // For us, method is always GET or HEAD here. - const headerValue = getIfModifiedSinceHeader(request); - if (headerValue != null) { - const result = checkIfModifiedSince(asset, headerValue); - if (!result) { - return new Response(null, { - status: 304, - headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), - }); - } - } - } - - // 5. When the method is GET and both Range and If-Range are present, evaluate the If-Range precondition: - // - if true and the Range is applicable to the selected representation, respond 206 (Partial Content) - // - otherwise, ignore the Range header field and respond 200 (OK) - - // 6. Otherwise, - // - perform the requested method and respond according to its success or failure. - return null; - } - - async serveAsset(request: Request, asset: ContentAsset, init?: AssetInit): Promise { - const metadata = asset.getMetadata(); - const headers: Record = { - 'Content-Type': metadata.contentType, - }; - Object.assign(headers, init?.headers); - - if (init?.cache != null) { - let cacheControlValue; - switch(init.cache) { - case 'extended': - cacheControlValue = 'max-age=31536000'; - break; - case 'never': - cacheControlValue = 'no-store'; - break; - } - headers['Cache-Control'] = cacheControlValue; - } - - const acceptEncodings = this.findAcceptEncodings(request); - const storeEntry = await asset.getStoreEntry(acceptEncodings); - if (storeEntry.contentEncoding != null) { - headers['Content-Encoding'] = storeEntry.contentEncoding; - } - - headers['ETag'] = `"${storeEntry.hash}"`; - if (metadata.lastModifiedTime !== 0) { - headers['Last-Modified'] = (new Date( metadata.lastModifiedTime * 1000 )).toUTCString(); - } - - const preconditionResponse = this.handlePreconditions(request, asset, headers); - if (preconditionResponse != null) { - return preconditionResponse; - } - - return new Response(storeEntry.body, { - status: init?.status ?? 200, - headers, - }); - } - - async serveRequest(request: Request): Promise { - - // Only handle GET and HEAD - if (request.method !== 'GET' && request.method !== 'HEAD') { - return null; - } - - const url = new URL(request.url); - const pathname = decodeURI(url.pathname); - - const asset = this.getMatchingAsset(pathname); - if (asset != null) { - return this.serveAsset(request, asset, { - cache: this.testExtendedCache(pathname) ? 'extended' : null, - }); - } - - // fallback HTML responses, like SPA and "not found" pages - if (requestAcceptsTextHtml(request)) { - - // These are raw asset paths, not relative to public path - const { spaFile } = this.serverConfig; - - if (spaFile != null) { - const asset = this.contentAssets.getAsset(spaFile); - if (asset != null) { - return this.serveAsset(request, asset, { - cache: 'never', - }); - } - } - - const { notFoundPageFile } = this.serverConfig; - if (notFoundPageFile != null) { - const asset = this.contentAssets.getAsset(notFoundPageFile); - if (asset != null) { - return this.serveAsset(request, asset, { - status: 404, - cache: 'never', - }); - } - } - } - - return null; - } -} diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts new file mode 100644 index 0000000..1f7b956 --- /dev/null +++ b/src/server/publisher-server/index.ts @@ -0,0 +1,522 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +/// +import { KVStore, type KVStoreEntry } from 'fastly:kv-store'; + +import { + type StaticPublishRc, +} from '../../models/config/static-publish-rc.js'; +import { + type PublisherServerConfigNormalized, +} from '../../models/config/publisher-server-config.js'; +import { + type ContentCompressionTypes, +} from '../../models/compression/index.js'; +import { + isKVAssetVariantMetadata, + type KVAssetEntry, + type KVAssetEntryMap, + type KVAssetVariantMetadata, +} from '../../models/assets/kvstore-assets.js'; +import { getKVStoreEntry } from '../util/kv-store.js'; +import { checkIfModifiedSince, getIfModifiedSinceHeader } from './serve-preconditions/if-modified-since.js'; +import { checkIfNoneMatch, getIfNoneMatchHeader } from './serve-preconditions/if-none-match.js'; + +type KVAssetVariant = { + kvStoreEntry: KVStoreEntry, +} & KVAssetVariantMetadata; + +export function buildHeadersSubset(responseHeaders: Headers, keys: Readonly) { + const resultHeaders = new Headers(); + for (const value of keys) { + if (value in responseHeaders) { + const responseHeaderValue = responseHeaders.get(value); + if (responseHeaderValue != null) { + resultHeaders.set(value, responseHeaderValue); + } + } + } + return resultHeaders; +} + +// https://httpwg.org/specs/rfc9110.html#rfc.section.15.4.5 +// The server generating a 304 response MUST generate any of the following header fields that would have been sent in +// a 200 (OK) response to the same request: +// * Content-Location, Date, ETag, and Vary +// * Cache-Control and Expires +const headersToPreserveForUnmodified = ['Content-Location', 'ETag', 'Vary', 'Cache-Control', 'Expires'] as const; + +function requestAcceptsTextHtml(req: Request) { + const accept = (req.headers.get('Accept') ?? '') + .split(',') + .map(x => x.split(';')[0]); + if(!accept.includes('text/html') && !accept.includes('*/*') && accept.includes('*')) { + return false; + } + return true; +} + +type AssetInit = { + status?: number, + headers?: Record, + cache?: 'extended' | 'never' | null, +}; + +export class PublisherServer { + public constructor( + publishId: string, + kvStoreName: string, + defaultCollectionName: string, + ) { + this.publishId = publishId; + this.kvStoreName = kvStoreName; + this.defaultCollectionName = defaultCollectionName; + this.activeCollectionName = this.defaultCollectionName; + this.collectionNameHeader = 'X-Publisher-Server-Collection'; + } + + static fromStaticPublishRc(config: StaticPublishRc) { + return new PublisherServer( + config.publishId, + config.kvStoreName, + config.defaultCollectionName, + ); + } + + publishId: string; + kvStoreName: string; + defaultCollectionName: string; + activeCollectionName: string; + collectionNameHeader: string | null; + + setActiveCollectionName(collectionName: string) { + this.activeCollectionName = collectionName; + } + + setCollectionNameHeader(collectionHeader: string | null) { + this.collectionNameHeader = collectionHeader; + } + + // Server config is obtained from the KV Store, and cached for the duration of this object. + // TODO get from simple cache + async getServerConfig() { + const settingsFileKey = `${this.publishId}_settings_${this.activeCollectionName}`; + const kvStore = new KVStore(this.kvStoreName); + const settingsFile = await getKVStoreEntry(kvStore, settingsFileKey); + if (settingsFile == null) { + console.error(`Settings File not found at ${settingsFileKey}.`); + console.error(`You may need to publish your application.`); + return null; + } + return (await settingsFile.json()) as PublisherServerConfigNormalized; + } + + async getStaticItems() { + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return []; + } + return serverConfig.staticItems + .map((x, i) => { + if (x.startsWith('re:')) { + const fragments = x.slice(3).match(/\/(.*?)\/([a-z]*)?$/i); + if (fragments == null) { + console.warn(`Cannot parse staticItems item index ${i}: '${x}', skipping...`); + return ''; + } + return new RegExp(fragments[1], fragments[2] || ''); + } + return x; + }) + .filter(x => Boolean(x)); + } + + async getKvAssetsIndex() { + const indexFileKey = `${this.publishId}_index_${this.activeCollectionName}`; + const kvStore = new KVStore(this.kvStoreName); + const indexFile = await getKVStoreEntry(kvStore, indexFileKey); + if (indexFile == null) { + console.error(`Index File not found at ${indexFileKey}.`); + console.error(`You may need to publish your application.`); + return null; + } + return (await indexFile.json()) as KVAssetEntryMap; + } + + async getMatchingAsset(assetKey: string, applyAuto: boolean = false): Promise { + + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return null; + } + const kvAssetsIndex = await this.getKvAssetsIndex(); + if (kvAssetsIndex == null) { + return null; + } + + if(!assetKey.endsWith('/')) { + // A path that does not end in a slash can match an asset directly + const asset = kvAssetsIndex[assetKey]; + if (asset != null) { + return asset; + } + + if (applyAuto) { + // ... or, we can try auto-ext: + // looks for an asset that has the specified suffix (usually extension, such as .html) + for (const extEntry of serverConfig.autoExt) { + let assetKeyWithExt = assetKey + extEntry; + const asset = kvAssetsIndex[assetKeyWithExt]; + if (asset != null) { + return asset; + } + } + } + } + + if (applyAuto) { + if (serverConfig.autoIndex.length > 0) { + // try auto-index: + // treats the path as a directory, and looks for an asset with the specified + // suffix (usually an index file, such as index.html) + let assetNameAsDir = assetKey; + // remove all slashes from end, and add one trailing slash + while(assetNameAsDir.endsWith('/')) { + assetNameAsDir = assetNameAsDir.slice(0, -1); + } + assetNameAsDir = assetNameAsDir + '/'; + for (const indexEntry of serverConfig.autoIndex) { + let assetKeyIndex = assetNameAsDir + indexEntry; + const asset = kvAssetsIndex[assetKeyIndex]; + if (asset != null) { + return asset; + } + } + } + } + + return null; + } + + // A pedantic function that returns all content types that are requested for in the accept-encoding header that are + // accepted by the server config, grouped by descending order of q values. + // For example, if accept-encoding had br;q=1,gzip;q=0.5, and the server accepts both br and gzip, + // the result would be [['br'], ['gzip']] + async findAcceptEncodingsGroups(request: Request): Promise { + const serverConfig = await this.getServerConfig(); + if (serverConfig == null || serverConfig.allowedEncodings.length === 0) { + return []; + } + + const acceptEncodingHeader = request.headers.get('accept-encoding')?.trim() ?? ''; + if (acceptEncodingHeader == '') { + return []; + } + + const priorityMap = new Map; + + for (const headerValue of acceptEncodingHeader.trim().split(',')) { + let [encodingValue, qValueStr] = headerValue.trim().split(';'); + encodingValue = encodingValue.trim(); + if (!serverConfig.allowedEncodings.includes(encodingValue as ContentCompressionTypes)) { + continue; + } + let qValue; // q value multiplied by 1000 + if (qValueStr == null || !qValueStr.startsWith('q=')) { + // use default of 1.0 + qValue = 1000; + } else { + qValueStr = qValueStr.slice(2); // remove the q= + qValue = parseFloat(qValueStr); + if (Number.isNaN(qValue) || qValue > 1) { + qValue = 1; + } + if (qValue < 0) { + qValue = 0; + } + // q values can have up to 3 decimal digits + qValue = Math.floor(qValue * 1000); + } + + let typesForQValue = priorityMap.get(qValue); + if (typesForQValue == null) { + typesForQValue = []; + priorityMap.set(qValue, typesForQValue); + } + typesForQValue.push(encodingValue as ContentCompressionTypes); + } + + // Sort keys, larger numbers to come first + const keysSorted = [...priorityMap.keys()] + .sort((qValueA, qValueB) => qValueB - qValueA); + + return keysSorted + .map(qValue => priorityMap.get(qValue)!); + } + + async testExtendedCache(pathname: string) { + const staticItems = await this.getStaticItems(); + return staticItems + .some(x => { + if (x instanceof RegExp) { + return x.test(pathname); + } + if (x.endsWith('/')) { + return pathname.startsWith(x); + } + return x === pathname; + }); + } + + handlePreconditions(request: Request, asset: KVAssetEntry, responseHeaders: Headers): Response | null { + // Handle preconditions according to https://httpwg.org/specs/rfc9110.html#rfc.section.13.2.2 + + // A recipient cache or origin server MUST evaluate the request preconditions defined by this specification in the following order: + // 1. When recipient is the origin server and If-Match is present, evaluate the If-Match precondition: + // - if true, continue to step 3 + // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.1) + + // 2. When recipient is the origin server, If-Match is not present, and If-Unmodified-Since is present, evaluate the If-Unmodified-Since precondition: + // - if true, continue to step 3 + // - if false, respond 412 (Precondition Failed) unless it can be determined that the state-changing request has already succeeded (see Section 13.1.4) + + // 3. When If-None-Match is present, evaluate the If-None-Match precondition: + // - if true, continue to step 5 + // - if false for GET/HEAD, respond 304 (Not Modified) + // - if false for other methods, respond 412 (Precondition Failed) + + let skipIfNoneMatch = false; + { + const headerValue = getIfNoneMatchHeader(request); + if (headerValue.length > 0) { + const result = checkIfNoneMatch(responseHeaders.get('ETag')!, headerValue); + if (result) { + skipIfNoneMatch = true; + } else { + return new Response(null, { + status: 304, + headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), + }); + } + } + } + + // 4. When the method is GET or HEAD, If-None-Match is not present, and If-Modified-Since is present, evaluate the + // If-Modified-Since precondition: + // - if true, continue to step 5 + // - if false, respond 304 (Not Modified) + + if (!skipIfNoneMatch) { + // For us, method is always GET or HEAD here. + const headerValue = getIfModifiedSinceHeader(request); + if (headerValue != null) { + const result = checkIfModifiedSince(asset.lastModifiedTime, headerValue); + if (!result) { + return new Response(null, { + status: 304, + headers: buildHeadersSubset(responseHeaders, headersToPreserveForUnmodified), + }); + } + } + } + + // 5. When the method is GET and both Range and If-Range are present, evaluate the If-Range precondition: + // - if true and the Range is applicable to the selected representation, respond 206 (Partial Content) + // - otherwise, ignore the Range header field and respond 200 (OK) + + // 6. Otherwise, + // - perform the requested method and respond according to its success or failure. + return null; + } + + public async loadKvAssetVariant(entry: KVAssetEntry, variant: ContentCompressionTypes | null): Promise { + + const kvStore = new KVStore(this.kvStoreName); + + const baseHash = entry.key.slice(7); + const baseKey = `${this.publishId}_files_sha256_${baseHash}`; + const variantKey = variant != null ? `${baseKey}_${variant}` : baseKey; + + const kvStoreEntry = await getKVStoreEntry(kvStore, variantKey); + if (kvStoreEntry == null) { + return null; + } + const metadataText = kvStoreEntry.metadataText() ?? ''; + if (metadataText === '') { + return null; + } + let metadata; + try { + metadata = JSON.parse(metadataText); + } catch { + return null; + } + if (!isKVAssetVariantMetadata(metadata)) { + return null; + } + if (metadata.size == null) { + return null; + } + return { + kvStoreEntry, + ...metadata, + }; + } + + private async findKVAssetVariantForAcceptEncodingsGroups(entry: KVAssetEntry, acceptEncodingsGroups: ContentCompressionTypes[][] = []): Promise { + + if (!entry.key.startsWith('sha256:')) { + throw new TypeError(`Key must start with 'sha256:': ${entry.key}`); + } + + // Each encodingGroup is an array of Accept-Encodings that have the same q value, + // with the highest first + for (const encodingGroup of acceptEncodingsGroups) { + + let smallestSize: number | undefined = undefined; + let smallestEntry: KVAssetVariant | undefined = undefined; + + for (const encoding of encodingGroup) { + if (!entry.variants.includes(encoding)) { + continue; + } + + const variantKvStoreEntry = await this.loadKvAssetVariant(entry, encoding); + if (variantKvStoreEntry == null) { + continue; + } + if (smallestSize == null || variantKvStoreEntry.size < smallestSize) { + smallestSize = variantKvStoreEntry.size; + smallestEntry = variantKvStoreEntry; + } + } + + if (smallestEntry != null) { + return smallestEntry; + } + } + + const baseKvStoreEntry = await this.loadKvAssetVariant(entry, null); + if (baseKvStoreEntry == null) { + throw new TypeError('Key not found: ' + entry.key); + } + + return baseKvStoreEntry; + } + + async serveAsset(request: Request, asset: KVAssetEntry, init?: AssetInit): Promise { + + const headers = new Headers(init?.headers); + headers.set('Content-Type', asset.contentType); + + if (this.collectionNameHeader) { + headers.set(this.collectionNameHeader, this.activeCollectionName); + headers.append('Vary', this.collectionNameHeader); + } + + if (init?.cache != null) { + let cacheControlValue; + switch(init.cache) { + case 'extended': + cacheControlValue = 'max-age=31536000'; + break; + case 'never': + cacheControlValue = 'no-store'; + break; + } + headers.append('Cache-Control', cacheControlValue); + } + + const acceptEncodings = await this.findAcceptEncodingsGroups(request); + const kvAssetVariant = await this.findKVAssetVariantForAcceptEncodingsGroups(asset, acceptEncodings); + if (kvAssetVariant.contentEncoding != null) { + headers.append('Content-Encoding', kvAssetVariant.contentEncoding); + } + + headers.set('ETag', `"${kvAssetVariant.hash}"`); + if (asset.lastModifiedTime !== 0) { + headers.set('Last-Modified', (new Date( asset.lastModifiedTime * 1000 )).toUTCString()); + } + + const preconditionResponse = this.handlePreconditions(request, asset, headers); + if (preconditionResponse != null) { + return preconditionResponse; + } + + const kvStoreEntry = kvAssetVariant.kvStoreEntry; + return new Response( + kvStoreEntry.body, + { + status: init?.status ?? 200, + headers, + } + ); + } + + async serveRequest(request: Request): Promise { + + // Only handle GET and HEAD + if (request.method !== 'GET' && request.method !== 'HEAD') { + return null; + } + + const url = new URL(request.url); + const pathname = decodeURI(url.pathname); + const serverConfig = await this.getServerConfig(); + if (serverConfig == null) { + return new Response( + `Settings not found. You may need to publish your application.`, + { + status: 500, + headers: { + 'content-type': 'text/plain', + }, + }, + ); + } + + const asset = await this.getMatchingAsset(serverConfig.publicDirPrefix + pathname, true); + if (asset != null) { + return this.serveAsset(request, asset, { + cache: await this.testExtendedCache(pathname) ? 'extended' : null, + }); + } + + // fallback HTML responses, like SPA and "not found" pages + if (requestAcceptsTextHtml(request)) { + + const kvAssetsIndex = await this.getKvAssetsIndex(); + if (kvAssetsIndex == null) { + return null; + } + + // These are raw asset paths, not relative to public path + const { spaFile } = serverConfig; + + if (spaFile != null) { + const asset = kvAssetsIndex[spaFile]; + if (asset != null) { + return this.serveAsset(request, asset, { + cache: 'never', + }); + } + } + + const { notFoundPageFile } = serverConfig; + if (notFoundPageFile != null) { + const asset = kvAssetsIndex[notFoundPageFile]; + if (asset != null) { + return this.serveAsset(request, asset, { + status: 404, + cache: 'never', + }); + } + } + } + + return null; + } +} diff --git a/src/server/serve-preconditions/if-modified-since.ts b/src/server/publisher-server/serve-preconditions/if-modified-since.ts similarity index 80% rename from src/server/serve-preconditions/if-modified-since.ts rename to src/server/publisher-server/serve-preconditions/if-modified-since.ts index 2639d1c..029d2fb 100644 --- a/src/server/serve-preconditions/if-modified-since.ts +++ b/src/server/publisher-server/serve-preconditions/if-modified-since.ts @@ -1,6 +1,10 @@ -import { ContentAsset } from "../../types/index.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ // https://httpwg.org/specs/rfc9110.html#field.if-modified-since + export function getIfModifiedSinceHeader(request: Request): number | null { // A recipient MUST ignore the If-Modified-Since header field if the received // field value is not a valid HTTP-date, the field value has more than one @@ -19,11 +23,11 @@ export function getIfModifiedSinceHeader(request: Request): number | null { return Math.floor(dateValueMs / 1000); } -export function checkIfModifiedSince(asset: ContentAsset, ifModifiedSince: number): boolean { +export function checkIfModifiedSince(lastModifiedTime: number, ifModifiedSince: number): boolean { // 1. If the selected representation's last modification date is earlier or equal to the // date provided in the field value, the condition is false. - if (asset.getMetadata().lastModifiedTime <= ifModifiedSince) { + if (lastModifiedTime <= ifModifiedSince) { return false; } diff --git a/src/server/serve-preconditions/if-none-match.ts b/src/server/publisher-server/serve-preconditions/if-none-match.ts similarity index 91% rename from src/server/serve-preconditions/if-none-match.ts rename to src/server/publisher-server/serve-preconditions/if-none-match.ts index d92d7f2..2b11e35 100644 --- a/src/server/serve-preconditions/if-none-match.ts +++ b/src/server/publisher-server/serve-preconditions/if-none-match.ts @@ -1,4 +1,7 @@ -import { ContentAsset } from "../../types/index.js"; +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ // https://httpwg.org/specs/rfc9110.html#field.if-none-match diff --git a/src/server/util/cookies.ts b/src/server/util/cookies.ts new file mode 100644 index 0000000..34e5c75 --- /dev/null +++ b/src/server/util/cookies.ts @@ -0,0 +1,16 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export function getCookieValue(request: Request, name: string) { + const cookieHeader = request.headers.get('Cookie'); + if (!cookieHeader) { + return null; + } + + return cookieHeader + .split(';') + .map(v => v.trim().split('=')) + .find(([key]) => key === name)?.[1] ?? null; +} diff --git a/src/server/util/http.ts b/src/server/util/http.ts deleted file mode 100644 index 466ceff..0000000 --- a/src/server/util/http.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function buildHeadersSubset(responseHeaders: Record, keys: Readonly) { - const resultHeaders: Record = {}; - for (const value of keys) { - if (value in responseHeaders) { - resultHeaders[value] = responseHeaders[value]; - } - } - return resultHeaders; -} diff --git a/src/server/util/kv-store.ts b/src/server/util/kv-store.ts new file mode 100644 index 0000000..43e625b --- /dev/null +++ b/src/server/util/kv-store.ts @@ -0,0 +1,118 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +/// +import { KVStore, type KVStoreEntry } from 'fastly:kv-store'; + +// This is a custom KVStoreEntry implementation that is built by using a body and +// metadataText passed in through the constructor. We piggyback off Response +// for the utility functions such as json() and text(). +class CustomKVStoreEntry extends Response implements KVStoreEntry { + constructor(body: BodyInit, metadataText: string) { + super(body); + this.metadataTextValue = metadataText; + } + + private readonly metadataTextValue: string; + get body(): ReadableStream { + return super.body!; + } + + metadata(): ArrayBuffer | null { + return new TextEncoder().encode(this.metadataTextValue); + } + metadataText(): string | null { + return this.metadataTextValue; + } +} + +export async function getKVStoreEntry( + kvStore: KVStore, + key: string, +): Promise { + + const entry = await kvStore.get(key); + if (entry == null) { + return null; + } + + const metadataText = entry.metadataText() ?? ''; + if (metadataText === '') { + return entry; + } + + let metadata; + try { + metadata = JSON.parse(metadataText); + } catch { + return entry; + } + + if (!('numChunks' in metadata) || typeof metadata.numChunks !== 'number') { + return entry; + } + + if (metadata.numChunks < 2) { + return entry; + } + + const streams = [ + entry.body, + ]; + + for (let chunkIndex = 1; chunkIndex < metadata.numChunks; chunkIndex++) { + const chunkKey = `${key}_${chunkIndex}`; + const chunkEntry = await kvStore.get(chunkKey); + if (chunkEntry == null) { + throw new Error(`Missing chunk ${chunkKey}`); + } + streams.push(chunkEntry?.body); + } + + const combinedStream = concatReadableStreams(streams); + + return new CustomKVStoreEntry(combinedStream, metadataText); +} + +function concatReadableStreams(streams: ReadableStream[]): ReadableStream { + let currentStreamIndex = 0; + let currentReader: ReadableStreamDefaultReader | null = null; + + return new ReadableStream({ + async pull(controller) { + while (true) { + // If no current reader, get one from the next stream + if (!currentReader) { + if (currentStreamIndex >= streams.length) { + controller.close(); + return; + } + currentReader = streams[currentStreamIndex].getReader(); + } + + const { value, done } = await currentReader.read(); + + if (done) { + currentReader.releaseLock(); + currentReader = null; + currentStreamIndex++; + continue; // Go to next stream + } + + controller.enqueue(value); + return; // Let pull() be called again + } + }, + async cancel(reason) { + if (currentReader) { + try { + await currentReader.cancel(reason); + } catch { + // swallow + } + } + } + }); +} diff --git a/src/types/compute.ts b/src/types/compute.ts deleted file mode 100644 index da4d095..0000000 --- a/src/types/compute.ts +++ /dev/null @@ -1,12 +0,0 @@ -export interface StoreEntry { - readonly body: ReadableStream | null; - readonly bodyUsed: boolean; - arrayBuffer(): Promise; - // blob(): Promise; - // formData(): Promise; - json(): Promise; - text(): Promise; - readonly contentEncoding: string | null; - readonly hash: string; - readonly size: number; -} diff --git a/src/types/config-normalized.ts b/src/types/config-normalized.ts deleted file mode 100644 index 822012f..0000000 --- a/src/types/config-normalized.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { ContentAssetInclusionTest, ExcludeDirTest, ModuleAssetInclusionTest } from "./config.js"; -import type { ContentTypeDef } from "./content-types.js"; -import type { ContentCompressionTypes } from "../constants/compression.js"; - -export type ContentAssetInclusionResultNormalized = { - includeContent: boolean, - inline: boolean, -}; - -export type ModuleAssetInclusionResultNormalized = { - includeModule: boolean, - useStaticImport: boolean, -}; - -export type StaticPublisherConfigNormalized = { - rootDir: string, - staticContentRootDir: string, - kvStoreName: string | null, - excludeDirs: ExcludeDirTest[], - excludeDotFiles: boolean, - includeWellKnown: boolean, - contentAssetInclusionTest: ContentAssetInclusionTest | null; - contentCompression: ContentCompressionTypes[], - moduleAssetInclusionTest: ModuleAssetInclusionTest | null; - contentTypes: ContentTypeDef[], - server: PublisherServerConfigNormalized | null, -}; - -export type PublisherServerConfigNormalized = { - publicDirPrefix: string, - staticItems: string[], - compression: ContentCompressionTypes[], - spaFile: string | null, - notFoundPageFile: string | null, - autoExt: string[], - autoIndex: string[], - // modifyResponse: ModifyResponseFunction | null, -}; diff --git a/src/types/config.ts b/src/types/config.ts deleted file mode 100644 index a9b0381..0000000 --- a/src/types/config.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { ContentTypeDef } from "./content-types.js"; - -export interface ExcludeDirTest { - test(name: string): boolean; -} - -export type ContentAssetInclusionResult = { - // Content asset inclusion mode: - - // includeContent: - // true - Include in Wasm binary and serve using includeBytes() function, or - // serve from KV Store if enabled. Default. - // false - Exclude from this publish. - includeContent?: boolean, - - // inline: - // true - Use includeBytes() for this asset, even if KV Store is enabled. - // false - If KV Store is enabled, serve the file from it. Default. - inline?: boolean, -}; - -// Asset Inclusion test -export type ContentAssetInclusionTest = (assetKey: string, contentType?: string) => ContentAssetInclusionResult | true | 'inline' | false | null; - -export type ModuleAssetInclusionResult = { - // Module asset inclusion mode: - - // includeModule: - // true - Include in Wasm binary as a dynamically loaded module. - // false - Exclude from this publish. Default. - includeModule?: boolean, - - // useStaticImport: - // true - If isModule is true, then import the module statically instead of dynamically. - // The dynamic loader getModule() function will simply return a reference to this - // instance, in this case. - // false - Don't load the module statically. Default. - useStaticImport?: boolean, -}; - -// Asset Inclusion test -export type ModuleAssetInclusionTest = (assetKey: string, contentType?: string) => ModuleAssetInclusionResult | true | false | 'static-import' | null; - -export type StaticPublisherConfig = { - // Set to a directory that acts as the root of all files that will be included in this publish. - rootDir: string, - - // Set to a directory that will hold the static content built by Static Publisher - // These files should not be committed to source control. './src' if not specified. - staticContentRootDir: string, - - // Set to a non-null string equal to the _name_ of an existing KV Store to enable "KV Store mode" for this publish. - // Service ID must also be specified in fastly.toml, or this will be an error. - kvStoreName?: string | false | null, - - // An array of values used to exclude files and directories (as well as files within those directories) from being - // included in this publish. Each entry in the array can be a string or a RegExp and will be tested against the relative - // path from 'rootDir' of each file or directory. - // Defaults to [ './node_modules' ]. Set to an empty array or specifically to null to include all files. - excludeDirs?: (string | ExcludeDirTest)[] | string | ExcludeDirTest | null, - - // If true, then files whose names begin with a dot, as well as files in directories whose names begin with a .dot, - // are excluded from this publish. Defaults to true. - excludeDotFiles?: boolean, - - // If true, include .well-known even if excludeDotFiles is true. - // Defaults to true. - includeWellKnown?: boolean, - - // A test to run on each asset key to determine whether and how to include the file as a content asset and/or module asset. - contentAssetInclusionTest?: ContentAssetInclusionTest | null; - - // Pre-generate content in these formats as well and serve them in tandem with the - // compression setting in the server settings. Default value is [ 'br' | 'gzip' ]. - contentCompression?: ('br' | 'gzip')[], - - // A test to run on each asset key to determine whether and how to include the file as a content asset and/or module asset. - moduleAssetInclusionTest?: ModuleAssetInclusionTest | null; - - // Additional / override content types. - contentTypes?: ContentTypeDef[], - - // Server settings - server?: PublisherServerConfig | null, -}; - -// Modify response -// export type ModifyResponseFunction = (response: Response, assetKey: string) => Response; - -// Publisher Server configuration - -export type PublisherServerConfig = { - // Prefix to apply to web requests. Effectively, a directory within rootDir that is used - // by the web server to determine the asset to respond with. Defaults to the empty string. - publicDirPrefix?: string, - - // A test to apply to item names to decide whether to serve them as "static" files, in other - // words, with a long TTL. These are used for files that are not expected to change. - // They can be provided as a string or array of strings. - // Items that contain asterisks, are interpreted as glob patterns. - // Items that end with a trailing slash are interpreted as directory names, - // Items that don't contain asterisks and that do not end in slash are checked for exact match. - staticItems?: string[] | string | false | null, - - // Compression. If the request contains an Accept-Encoding header, they are checked in the order listed - // in the header for the values listed here. The compression algorithm represented by the first match is applied. - // Default value is [ 'br', 'gzip' ]. - compression?: ('br' | 'gzip')[], - - // Set to the asset key of a content item to serve this when a GET request comes in for an unknown asset, and - // the Accept header includes text/html. - spaFile?: string | false | null, - - // Set to the asset key of a content item to serve this when a request comes in for an unknown asset, and - // the Accept header includes text/html. - notFoundPageFile?: string | false | null, - - // When a file is not found, and it doesn't end in a slash, then try auto-ext: try to serve a file with the same name - // postfixed with the specified strings, tested in the order listed. - autoExt?: string[] | string | false | null, - - // When a file is not found, then try auto-index: treat it as a directory, then try to serve a file that has the - // specified strings, tested in the order listed. - autoIndex?: string[] | string | false | null, - - // Modify Response before it is served - // modifyResponse?: ModifyResponseFunction | null, -}; diff --git a/src/types/content-assets.ts b/src/types/content-assets.ts deleted file mode 100644 index 16cd310..0000000 --- a/src/types/content-assets.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { ContentCompressionTypes } from "../constants/compression.js"; -import type { StoreEntry } from "../types/compute.js"; - -export type CompressedFileInfos = Partial>; - -export type ContentFileInfo = { - hash: string, // same as hash of file - size: number, -}; - -// For runtime use e.g., testing -export type ContentFileInfoForBytes = ContentFileInfo & { - bytes: Uint8Array, -}; -export type ContentFileInfoForString = ContentFileInfo & { - content: string, -}; - -// For static publishing -export type ContentFileInfoForStaticPublishing = ContentFileInfo & { - staticFilePath: string, -}; - -export type ContentFileInfoForWasmInline = ContentFileInfoForStaticPublishing; - -export type ContentFileInfoForKVStore = ContentFileInfoForStaticPublishing & { - kvStoreKey: string, -}; - -export type ContentAssetMetadataMapEntryBase = { - assetKey: string, - contentType: string, - text: boolean, - lastModifiedTime: number, // as unix time - fileInfo: TFileInfo, - compressedFileInfos: CompressedFileInfos -}; - -export type ContentAssetMetadataMapEntryBytes = { - type: 'bytes', -} & ContentAssetMetadataMapEntryBase; -export type ContentAssetMetadataMapEntryString = { - type: 'string', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntryWasmInline = { - type: 'wasm-inline', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntryKVStore = { - type: 'kv-store', -} & ContentAssetMetadataMapEntryBase; - -export type ContentAssetMetadataMapEntry = - | ContentAssetMetadataMapEntryBytes - | ContentAssetMetadataMapEntryString - | ContentAssetMetadataMapEntryWasmInline - | ContentAssetMetadataMapEntryKVStore; - -export type ContentAssetMetadataMap = { - [assetKey: string]: ContentAssetMetadataMapEntry, -}; - -export interface ContentAsset { - readonly type: string; - readonly isLocal: boolean; - readonly assetKey: string; - getMetadata(): ContentAssetMetadataMapEntry; - getStoreEntry(acceptEncodingsGroups?: ContentCompressionTypes[][]): Promise; - getBytes(): Uint8Array; - getText(): string; - getJson(): T; -} diff --git a/src/types/content-types.ts b/src/types/content-types.ts deleted file mode 100644 index c813c07..0000000 --- a/src/types/content-types.ts +++ /dev/null @@ -1,24 +0,0 @@ -export type ContentTypeTest = (name: string) => boolean; - -// Content Type definition -export type ContentTypeDef = { - // A test on the asset key to perform on this content type. - test: RegExp | ContentTypeTest, - - // The Content-Type header value to provide for this content type. - contentType: string, - - // Whether this content type represents a text value encoded in utf-8. - // If so, conveniences can be provided. - text?: boolean, - - // Binary formats are usually not candidates for compression, but certain - // types can benefit from it. - precompressAsset?: boolean, -}; - -export type ContentTypeTestResult = { - contentType: string, - text: boolean, - precompressAsset: boolean, -}; diff --git a/src/types/index.ts b/src/types/index.ts deleted file mode 100644 index e538be3..0000000 --- a/src/types/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -export type { - StoreEntry, -} from './compute.js'; -export type { - StaticPublisherConfig, - ContentAssetInclusionTest, - ContentAssetInclusionResult, - ModuleAssetInclusionTest, - ModuleAssetInclusionResult, -} from './config.js'; -export type { - ContentAssetMetadataMap, - ContentAssetMetadataMapEntry, - ContentAsset, -} from './content-assets.js'; -export type { - ModuleAssetMap, - ModuleAssetMapEntry, - ModuleAsset, -} from './module-assets.js'; -export type { - ContentTypeTest, - ContentTypeDef, - ContentTypeTestResult, -} from './content-types.js'; -export type { - PublisherServerConfigNormalized, -} from './config-normalized.js'; diff --git a/src/types/module-assets.ts b/src/types/module-assets.ts deleted file mode 100644 index db4be9f..0000000 --- a/src/types/module-assets.ts +++ /dev/null @@ -1,15 +0,0 @@ -export type ModuleAssetMapEntry = { - isStaticImport: boolean; - module: any | null, - loadModule: () => Promise, -}; -export type ModuleAssetMap = { - [assetKey: string]: ModuleAssetMapEntry, -}; - -export interface ModuleAsset { - readonly assetKey: string; - getModule(): Promise; - getStaticModule(): any | null; - readonly isStaticImport: boolean; -} diff --git a/src/types/util.ts b/src/types/util.ts deleted file mode 100644 index d7cd5fa..0000000 --- a/src/types/util.ts +++ /dev/null @@ -1,7 +0,0 @@ -export type Merge = { - [TKey in keyof T]: TKey extends keyof U ? (T[TKey] | U[TKey]) : T[TKey]; -}; - -export type Unmerge = Required<{ - [TKey in keyof T]: TKey extends keyof U ? (Exclude) : T[TKey]; -}>; diff --git a/src/util/index.ts b/src/util/index.ts deleted file mode 100644 index 81f416b..0000000 --- a/src/util/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './content-types.js'; -export * from './metadata.js'; diff --git a/src/util/metadata.ts b/src/util/metadata.ts deleted file mode 100644 index 6cdd47d..0000000 --- a/src/util/metadata.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ContentAssetMetadataMap } from "../types/content-assets.js"; - -export function getKVStoreKeysFromMetadata(contentAssetMetadataMap: ContentAssetMetadataMap): Set { - - const results = new Set(); - - for (const metadata of Object.values(contentAssetMetadataMap)) { - if (metadata.type === 'kv-store') { - results.add(metadata.fileInfo.kvStoreKey); - for (const fileInfo of Object.values(metadata.compressedFileInfos)) { - results.add(fileInfo.kvStoreKey); - } - } - } - - return results; - -} From fcb161498a6883e7fee06443b013d3c59143582f Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 16 Apr 2025 17:38:46 +0900 Subject: [PATCH 02/20] Readme --- README.md | 902 +++++++++++++++++++++++++----------------------- README.short.md | 53 +++ 2 files changed, 523 insertions(+), 432 deletions(-) create mode 100644 README.short.md diff --git a/README.md b/README.md index 9402aba..5bd650b 100644 --- a/README.md +++ b/README.md @@ -1,606 +1,644 @@ # Static Publisher for JavaScript on Fastly Compute -Using a static site generator to build your website? Do you simply need to serve some static files? With `compute-js-static-publish`, now you can deploy and serve everything from Fastly's [Compute](https://developer.fastly.com/learning/compute/) platform. +## 📖 Table of Contents -## How it works +- [✨ Key Features](#-key-features) +- [🏁 Quick Start](#-quick-start) +- [⚙️ Configuring `static-publish.rc.js`](#️-configuring-static-publishrcjs) +- [🧾 Default Server Config: `publish-content.config.js`](#-default-server-config-publish-contentconfigjs) +- [📦 Collections (Preview, Deploy, Promote)](#-collections-preview-deploy-promote) +- [🧹 Cleaning Up](#-cleaning-up) +- [🔄 Content Compression](#-content-compression) +- [🧩 Using PublisherServer in Custom Apps](#-using-publisherserver-in-custom-apps) +- [📥 Using Published Assets in Your Code](#-using-published-assets-in-your-code) +- [📚 Next Steps](#-next-steps) -You have some HTML files, along with some accompanying CSS, JavaScript, image, and/or font files in a directory. Perhaps you've used a framework or static site generator to build these files. +Serve static websites and web apps at the edge — no backends and no CDN invalidation delays. -Assuming the root directory that contains your static files is `./public`: +`@fastly/compute-js-static-publish` helps you deploy static files to [Fastly Compute](https://developer.fastly.com/learning/compute/) using Fastly's KV Store for fast, cacheable, and content-addressed delivery. -### 1. Run `compute-js-static-publish` +## ✨ Key Features -```shell -npx @fastly/compute-js-static-publish@latest --root-dir=./public --kv-store-name=my-store -``` - -This will scaffold a Compute application at `./compute-js`. It will add a default `./compute-js/src/index.js` file that instantiates the [`PublisherServer`](#publisherserver) class and runs it to serve the static files from your project. - -You must also specify the name of a Fastly KV Store that you will be using when you deploy your application. The KV Store doesn't have to exist yet, we'll be doing that in the next step. - -> [!TIP] -> This process creates a `./compute-js/static-publish.rc.js` to hold your configuration. This, as well as the other files created in your new Compute program at `./compute-js`, can be committed to source control (except for the ones we specify in `.gitignore`!) - -### 2. Set up your Fastly Service and KV Store +- 📦 Easy to scaffold and deploy +- 🚀 Content stored in Fastly KV Store using hashed keys +- 🔁 Publish new content without deploying new Wasm binaries +- 🗂 Organize releases into named collections which can be previewed (e.g. `live`, `staging`, `preview-123`) +- 🧼 Cleanup tools to remove expired or orphaned files +- ⚙️ Configurable per-collection server configurations (e.g. fallback files) +- 🔒 Supports Brotli/gzip compression, conditional GET, and long cache TTLs +--- +## 🏁 Quick Start +### 1. Scaffold a Compute App +Create a directory for your project, place your static files in `./public`, then type: -The above step of generating your application that includes your files and a program that serves them, and needs to be run only once. To make modifications to your application, simply make changes to your static files and rebuild it. Read the rest of this section for more details. +```sh +npx @fastly/compute-js-static-publish \ + --root-dir=./public \ + --kv-store-name=site-content +``` +You get a Compute app in `./compute-js` with: -### 3. Test your application locally +- `fastly.toml` (service config) +- `src/index.js` (entry point) +- `static-publish.rc.js` (app config) +- `publish-content.config.js` (publish-time / runtime behavior) -Once your application has been generated and your KV Store is ready, it's ready to be run! +### 2. Preview Locally -The `start` script builds and runs your application using [Fastly's local development server](https://developer.fastly.com/learning/compute/testing/#running-a-local-testing-server). +Type the following — no Fastly account or service required yet! -```shell -cd ./compute-js +```sh +cd compute-js npm install npm run start ``` -The build process scans your `./public` directory to generate files in the `./compute-js/static-publisher` directory. These files are packaged into your application's Wasm binary. - -Once your application is running, your files will be served under `http://127.0.0.1:7676/` at the corresponding paths relative to the `./public` directory. For example, making a request to `http://127.0.0.1:7676/foo/bar.html` will attempt to serve the file at `./public/foo/bar.html`. +Fastly's [local development environment](https://www.fastly.com/documentation/guides/compute/testing/#running-a-local-testing-server) serves your static website at `http://127.0.0.1:7676`, powered by a simulated KV Store. -### 3. Make changes to your application +### 3. Deploy Your App -Now, you're free to make changes to your static files. Add, modify, or remove files in the `./public` directory, and then re-build and re-run your application by typing `npm run start` again. - -Each time you re-build the project, `compute-js-static-publish` will re-scan your `./public` directory and regenerate the files in the `./compute-js/static-publisher` directory. - -> [!TIP] -> You can make further customizations to the behavior of your application, such as specifying directories of your static files, specifying whether to use GZIP compression on your files, specifying custom MIME types of your files, and more. You can also run custom code alongside the default server behavior, or even access the contents of the files directly from custom code. See [Advanced Usages](#advanced-usages) below for details. Rebuilding will not modify the files in your `./compute-js/src` directory, so feel safe making customizations to your code. +Ready to go live? All you need is a [free Fastly account](https://www.fastly.com/signup/?tier=free)! -### 4. When you're ready to go live, deploy your Compute service - -The `deploy` script builds and [publishes your application to a Compute service in your Fastly account](https://developer.fastly.com/reference/cli/compute/publish/). - -```shell +```sh npm run deploy ``` -## Prerequisites - -Although your published application runs on a Fastly Compute service, the publishing process offered by this package requires Node.js 20.11 or newer. - -## Features - -- Simple to set up, with a built-in server module. -- Or, make file contents directly available to your application, so you can write your own server. -- Content and metadata are available to your application, accessible by files' pre-package file paths. -- Brotli and Gzip compression. -- Support for `If-None-Match` and `If-Modified-Since` request headers. -- Optionally use webpack as a module bundler. -- Files are kept in Fastly's [KV Store](#kv-store). It is also possible to selectively serve some files - embedded into your Wasm module. -- Supports loading JavaScript files as code into your Compute application. -- Presets for several static site generators. - -Some of these features are new! If you wish to update to this version, you may need to re-scaffold your application, or follow the steps outlined in [MIGRATING.md](./MIGRATING.md). +The command publishes your Compute app and creates the KV Store. (No content uploaded yet!) -## How does it work? Where are the files? +### 4. Publish Content -Once your application is scaffolded, `@fastly/compute-js-static-publish` integrates into your development process by -running as part of your build process. - -The files you have configured to be included (`--root-dir`) are enumerated and prepared. Their contents are uploaded into -Fastly's [KV Store](#kv-store). This process is called "publishing". - -Once the files are published, they are available to the other source files in the Compute application. For example, -the stock application runs the [PublisherServer](#publisherserver) class to serve these files. - -For more advanced uses, such as accessing the contents of these file in your application, see the -[Using the packaged objects in your own application](#using-published-assets-in-your-own-application) section below. +```sh +npm run publish-content +``` -Publishing is meant to run each time before building your Compute application into a Wasm file. -If the files in `--root-dir` have changed, then a new set of files will be published. +Upload static files to the KV Store and applies the server config. Your website is now up and live! -### Content Compression +--- -During publishing, this tool supports pre-compression of content. By default, your assets for select content types are -compressed using the Brotli and gzip algorithms, and then stored alongside the original files in your Wasm binary (or -[KV Store](#kv-store)). +## 🗂 Project Layout -The content types that are compressed include any that are considered `text` types, as well as certain binary types that -expect to see a benefit of being compressed. +Here's what your project might look like after scaffolding: -* Compressed text types: `.txt` `.html` `.xml` `.json` `.map` `.js` `.css` `.svg` -* Compressed binary types: `.bmp` `.tar` +``` +my-project/ +├── public/ # Your static site files (HTML, CSS, JS, etc.) +│ ├── index.html +│ ├── scripts.js +│ ├── styles.css +│ └── (... other static files ...) +└── compute-js/ # The scaffolded Compute application + ├── fastly.toml # Fastly service configuration + ├── package.json # Scaffolded package metadata + ├── package-lock.json # Dependency lockfile + ├── .gitignore # Ignores build artifacts by default + ├── static-publish.rc.js # App config + ├── publish-content.config.js # Publishing / runtime config + ├── static-publisher/ # ⚠️ Do not commit - generated content for local dev and publishing + │ ├── kvstore.json # Simulates KV Store content for local preview + │ └── kv-store-content/ # Preprocessed and compressed files for KV upload + └── src/ + └── index.js # Your Compute app entry point +``` -To configure these content types, use the `contentTypes` field of the [`static-publish.rc.js` config file](#static-publish-rc). +## ⚙️ Configuring `static-publish.rc.js` -> [!IMPORTANT] -> By default, pre-compressed content assets are not generated when the KV Store is not used. -This is done to prevent the inclusion multiple of copies of each asset from making the Wasm binary too large. -If you want to pre-compress assets when not using KV Store, add a value for 'contentCompression' to your -`static-publish.rc.js` file. +This file defines your compute-js-static-publish application's settings. A copy of this is also baked into the Wasm binary and loaded when running your Compute app locally or on the edge. -## CLI options +### Example: `static-publish.rc.js` -Except for `--root-dir`, most arguments are optional. +```js +const rc = { + kvStoreName: 'site-content', + defaultCollectionName: 'live', + publishId: 'default', +}; -```shell -npx @fastly/compute-js-static-publish \ - --root-dir=./build \ - --public-dir=./build/public \ - --static-dir=./build/public/static \ - --output=./compute-js \ - --spa=./build/spa.html +export default rc; ``` -If you provide options, they override the defaults described below. +### Fields: -Any configuration options will be written to a `static-publish.rc.js` file, and used each time you build your Compute -application. +- `kvStoreName` - The name of the KV Store used for publishing (required). +- `defaultCollectionName` - Collection to serve when none is specified (required). +- `publishId` - Unique prefix for all keys in the KV Store (required). Override only for advanced setups (e.g., multiple apps sharing the same KV Store). -On subsequent builds of your Compute application, `compute-js-static-publish` will run with a special flag, `build-static`, -reading from stored configuration, then scanning the `--public-dir` directory to recreate `./compute-js/static-publisher/statics.js`. +> [!NOTE] +> Changes to this file require rebuilding the Compute app, since a copy of it is baked into the Wasm binary. -Any relative file and directory paths passed at the command line are handled as relative to the current directory. +## 🧾 Default Server Config: `publish-content.config.js` -### Required options: +This file is included as part of the scaffolding. Every time you publish content, the publish settings in this file are used for publishing the content, and the server settings are taken from this file and saved as the settings used by the server for that collection. -| Option | Description | -|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------| -| `--root-dir` | The root of the directory that contains the files to include in the publishing. All files you wish to include must reside under this root. | -| `--kv-store-name` | The name of a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) to hold your assets. | +You can override this file for a single `publish-content` command by specifying an alternative using `--config` on the command line. -### Publishing options: +```js +const config = { + // these paths are relative to compute-js dir + rootDir: '../public', + staticPublisherWorkingDir: './static-publisher', + + // Include/exclude filters (optional): + excludeDirs: ['node_modules'], + excludeDotfiles: true, + includeWellKnown: true, + + // Advanced filtering (optional): + kvStoreAssetInclusionTest: (key, contentType) => { + return true; // include everything by default + }, + + // Override which compressed variants to create for each asset during publish (optional): + contentCompression: ['br', 'gzip'], + + // Content type definitions/overrides (optional): + contentTypes: [ + { test: /\.custom$/, contentType: 'application/x-custom', text: false }, + ], + + // Server settings + server: { + publicDir: './public', + spaFile: '/spa.html', + notFoundPageFile: '/404.html', + autoIndex: ['index.html'], + autoExt: ['.html'], + staticItems: ['/static/', '/assets/'], + allowedEncodings: ['br', 'gzip'], + } +}; -| Option | Default | Description | -|----------------------------------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `--output` | `./compute-js` | The directory in which to create the Compute application. | -| `--static-publisher-working-dir` | (output directory) + `/static-publisher` | The directory under the Compute application where asset files are written in preparation for upload to the KV Store and for serving for local mode. | +export default config; +``` -### Server options: +> [!NOTE] +> Changes to this file apply when content is published. + +### Fields: + +- `rootDir` - Directory to scan for content, relative to this file (required). +- `staticPublisherWorkingDir` - Directory to hold working files during publish (default: `'./static-publisher'`). +- `excludeDirs` - Array of directory names or regex patterns to exclude (default: `['./node_modules']`). +- `excludeDotFiles` - Exclude dotfiles and dot-named directories (default: true). +- `includeWellKnown` - Always include `.well-known` even if dotfiles are excluded (default: true). +- `kvStoreAssetInclusionTest` - Function to determine inclusion and variant behavior per file. +- `contentCompression` - Array of compression formats to pre-generate (`['br', 'gzip']` by default). +- `contentTypes` - Additional or override content type definitions. + +- `server` - Server runtime config that contains the following fields: + - `publicDir` - The 'public' directory. The Publisher Server will + resolve requests relative to this directory (default: same value as 'root-dir'). + - `spaFile` - Path to a file to be used to serve in a SPA application. + - `notFoundPageFile` - Path to a file to be used to serve as a 404 not found page. + - `autoIndex` - List of files to automatically use as index. + - `autoExt` - List of extensions to automatically apply to a path and retry when + the requested path is not found. + - `staticItems` - Directories to specify as containing 'static' files. The + Publisher Server will serve files from these directories with a long TTL. + - `allowedEncodings` - Specifies which compression formats the server is allowed + to serve based on the client's `Accept-Encoding` header. + +## 📦 Collections (Preview, Deploy, Promote) + +Collections are a powerful feature that allow you to publish and manage multiple versions of your site simultaneously. Each collection is a named set of: + +- Static files +- Server configuration (e.g., fallback file, static directories, etc.) +- An index file that maps paths to those hashes + +Collections are published using a `--collection-name`, and can be reused, updated, or deleted independently. For example, you can create a staging version of your site using: + +```sh +npx @fastly/compute-js-static-publish publish-content \ + --collection-name=staging \ + --expires-in=7d \ + --config=./publish-content.config.js +``` -Used to populate the `server` key under `static-publish.rc.js`. +You can overwrite or republish any collection at any time. Old file hashes will be reused automatically where contents match. -| Option | Default | Description | -|--------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--public-dir` | | The directory that contains your website's public files. | -| `--static-dir` | (None) | Any directories under `--public-dir` that contain the website's static assets that will be served with a very long TTL. You can specify as many such directories as you wish, by listing multiple items. | -| `--auto-ext` | `.html,.htm` | Specify automatic file extensions. | -| `--auto-index` | `index.html,index.htm` | Specify filenames for automatically serving an index file. | -| `--spa` | (None) | Path to a fallback file for SPA applications. | -| `--not-found-page` | `/404.html` | Path to a fallback file for 404 Not Found. | +### Expiration (Auto-cleanup) -See [PublisherServer](#publisherserver) for more information about these features. +Collections can expire automatically: -For backwards compatibility, if you do not specify a `--root-dir` but you have provided a `--public-dir`, then that value is used for `--root-dir`. +- Expired collections return 404s +- They’re ignored by the server +- Their files are cleaned up by `clean` -Note that the files referenced by `--spa` and `--not-found-page` do not necessarily have to reside inside `--public-dir`. +```sh +--expires-in=3d # relative (e.g. 1h, 2d, 1w) +--expires-at=2025-05-01T12:00Z # absolute (ISO 8601) +``` -### Fastly service options +### Switching the active collection -These arguments are used to populate the `fastly.toml` and `package.json` files of your Compute application. +By default, the app serves from the collection named in `static-publish +.rc.js` under `defaultCollectionName`. To switch the active collection, you add custom code to your Compute app that calls `publisherServer.setActiveCollectionName(name)`: -| Option | Default | Description | -|-------------------|--------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--name` | `name` from `package.json`, or `compute-js-static-site` | The name of your Compute application. | -| `--description` | `description` from `package.json`, or `Compute static site` | The description of your Compute application. | -| `--author` | `author` from `package.json`, or `you@example.com` | The author of your Compute application. | -| `--service-id` | (None) | The ID of an existing Fastly WASM service for your Compute application. | +```js +publisherServer.setActiveCollectionName("preview-42"); +``` -## PublisherServer +This only affects the current request (in Compute, requests do not share state). -`PublisherServer` is a simple yet powerful server that can be used out of the box to serve the files prepared by this tool. +#### Example: Subdomain-based Routing -This server handles the following automatically: +```js +import { PublisherServer, collectionSelector } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; -* Maps the path of your request to a path under `--public-dir` and serves the content of the asset -* Sources the content from the content packaged in the Wasm binary, or from the [KV Store](#kv-store), if configured. -* Applies long-lived cache headers to files served from `--static-dir` directories. Files under these directories will be cached by the browser for 1 year. (Use versioned or hashed filenames to avoid serving stale assets.) -* Performs Brotli and gzip compression as requested by the `Accept-Encoding` headers. -* Provides `Last-Modified` and `ETag` response headers, and uses them with `If-Modified-Since` and `If-None-Match` request headers to produce `304 Not Modified` responses. -* If an exact match is not found for the request path, applies automatic extensions (e.g., `.html`) and automatic index files (e.g., `index.html`). -* Can be configured to serve a fallback file for SPA apps. Useful for apps that use [client-side routing](https://create-react-app.dev/docs/deployment#serving-apps-with-client-side-routing). -* Can be configured to serve a 404 not found file. -* Returns `null` if nothing matches, so that you can add your own handling if necessary. +const publisherServer = PublisherServer.fromStaticPublishRc(rc); -During initial scaffolding, the configuration based on the command-line parameters is written to your `./static-publisher.rc.js` file under the `server` key. +addEventListener("fetch", event => { + const request = event.request; + const collectionName = collectionSelector.fromHostDomain(request, /^preview-(.*)\./); + if (collectionName != null) { + publisherServer.setActiveCollectionName(collectionName); + } -### Configuring PublisherServer + event.respondWith(publisherServer.serveRequest(request)); +}); +``` -You can further configure the server by making modifications to the `server` key under `./static-publisher.rc.js`. +### 🔀 Selecting a Collection at Runtime -| Key | Default | Description | -|--------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `publicDirPrefix` | `''` | Prefix to apply to web requests. Effectively, a directory within `rootDir` that is used by the web server to determine the asset to respond with. | -| `staticItems` | `[]` | A test to apply to item names to decide whether to serve them as "static" files, in other words, with a long TTL. These are used for files that are not expected to change. They can be provided as a string or array of strings. | -| `compression` | `[ 'br', 'gzip' ]` | If the request contains an `Accept-Encoding` header, they are checked for the values listed here. The compression algorithm that produces the smallest transfer size is applied. | -| `autoExt` | `[]` | When a file is not found, and it doesn't end in a slash, then try auto-ext: we try to serve a file with the same name post-fixed with the specified strings, tested in the order listed. These are tested before auto-index, if any. | -| `autoIndex` | `[]` | When a file is not found, then try auto-index: we treat it as a directory, then try to serve a file that has the specified strings, tested in the order listed. | -| `spaFile` | `null` | Asset key of a content item to serve with a status code of `200` when a GET request comes arrives for an unknown asset, and the Accept header includes text/html. | -| `notFoundPageFile` | `null` | Asset key of a content item to serve with a status code of `404` when a GET request comes arrives for an unknown asset, and the Accept header includes text/html. | +The `collectionSelector` module provides helpers to extract a collection name from different parts of a request: -For `staticItems`: -* The item name that is tested is the path of the request, without applying the publicDirPrefix. -* Items that contain asterisks are interpreted as glob patterns (for example, `/static/**/*.js`) -* Items that end with a trailing slash are interpreted as a directory name. -* Items that don't contain asterisks and that do not end in slash are checked for exact match. +```js +collectionSelector.fromHostDomain(request, /^preview-(.*)\./); +``` -For `compression`, the following values are allowed: -* `'br'` - Brotli -* `'gzip'` - Gzip +#### From the Request URL -## Associating your project with a Fastly Service +```js +collectionSelector.fromRequestUrl(request, url => url.pathname.split('/')[2]); +``` -The project created by this tool is a Fastly Compute JavaScript application, complete with a `fastly.toml` file that -describes your project to the Fastly CLI. +#### With a Custom Request Matcher -To deploy your project to production, you deploy it to a [Fastly service](https://developer.fastly.com/reference/glossary#term-service) -in your account. Usually, you create your service automatically as part of your first deployment of the project. +```js +collectionSelector.fromRequest(request, req => req.headers.get('x-collection') ?? 'live'); +``` -In this case, `fastly.toml` has no value for `service_id` at the time you deploy, so the Fastly CLI will prompt -you to create a Fastly service in your account, after which it will save the new service's ID to your `fastly.toml` file. +#### From a Cookie -Alternatively, you may deploy to a service that already exists. You can create this service using the -[Fastly CLI](https://developer.fastly.com/reference/cli/service/create/) or the [Fastly web app](https://manage.fastly.com/). -Note that since this is a Compute application, the service must be created as a Wasm service. +```js +const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); +``` -Before deploying your application, specify the service by setting the `service_id` value in the `fastly.toml` file to the -ID of the service. The Fastly CLI will deploy to the service identified by this value. +#### From a Fastly Config Store -To specify the service at the time you are scaffolding the project (for example, if you are running this tool and deploying -as part of a CI process), specify the `--service-id` command line argument to populate `fastly.toml` with this value. +```js +collectionSelector.fromConfigStore('my-config-store', 'collection-key'); +``` -## Using the KV Store +### 🚀 Promoting a Collection -
+If you're happy with a preview or staging collection and want to make it live, use the `collections promote` command: -// Also, allow populating kv-store and publish id from command line +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=staging +``` -Starting with v7, assets are uploaded to a [Fastly KV Store](https://developer.fastly.com/learning/concepts/data-stores/#kv-stores) -during the publishing process. In addition, the index file is also saved to the KV store. As a result, building this way -requires no deploy of your application to release a new version. +This copies all content and server settings from the `staging` collection to the collection named in your `defaultCollectionName`. To copy to a different name, add `--to=some-other-name`. -// You build your application as normal, and as a step during the build, your files -// are uploaded to the Fastly KV Store, and metadata in the application is marked to source them from there instead -// of from bytes in the Wasm binary. +You can also specify a new expiration: -You can enable the use of KV Store with `@fastly/compute-js-static-publish` as you scaffold your application, or -at any later time. +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=preview-42 \ + --to=staging \ + --expires-in=7d +``` -At the time you perform a publish: +> [!NOTE] +> The collection denoted by `defaultCollectionName` is exempt from expiration. -* Your Fastly service must already exist. See [Associating your project with a Fastly Service](#associating-your-project-with-a-fastly-service) above. +## 🛠 Development → Production Workflow -* Your [KV Store](https://www.fastly.com/documentation/guides/concepts/edge-state/data-stores/#kv-stores) must already - exist under the same Fastly account, and be linked to the service. The - [Resource Link](https://www.fastly.com/documentation/guides/concepts/resources/) must have the same name as the - KV Store. +During development, starting the local preview server (`npm run start`) will run `publish-content --local-only` automatically via a `prestart hook`. This simulates publishing by writing to `kvstore.json` instead of uploading to the actual KV Store. You can preview your site at `http://127.0.0.1:7676` - no Fastly account or service required. -To create your KV Store and link it to the service, follow these steps: +When you're ready for production: -```shell -# Create a KV Store -$ npx fastly kv-store create --name=example-store -SUCCESS: Created KV Store 'example-store' (onvbnv9ntdvlnldtn1qwnb) -``` +1. [Create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already. +2. Run `npm run deploy` + - This builds your Compute app into a Wasm binary + - Deploys it to a new or existing Fastly Compute service + - If creating a new service: + - you'll be prompted for backend info - **you can skip this**, as no backend is needed (all content is served from KV) + - KV Store will be created if necessary and automatically linked to your new service. -Make a note of the KV Store ID in this response (`onvbnv9ntdvlnldtn1qwnb` in the above example). Next, use this ID -value to create a Resource Link between your service and the KV Store. You **MUST** use the same name for your Resource -Link as you use for your KV Store. After linking the KV Store, activate the new version of the service. +Once deployed, publish content like so: -```shell -# Link the KV Store to a service provide the KV Store ID as resource-id -$ npx fastly resource-link create --version=latest --autoclone --resource-id=onvbnv9ntdvlnldtn1qwnb --name=example-store -$ npx fastly service-version activate --version=latest +```sh +npm run publish-content ``` -Once the KV Store is created and linked to your service, add the name of the KV Store (`example-store` in -　the above example) name to your `static-publish.rc.js` file under the `kvStoreName` key. - -To specify the KV Store at the time you are scaffolding the project (for example, if you are running this tool and -deploying as part of a CI process), specify the `--service-id` and `--kv-store-name` command line arguments to populate -the respective files with these values. - -After you have performed the above steps, go ahead and build your application as normal. -As a new step during the build process, the tool will send these files to the KV Store. +This: -> [!IMPORTANT] -> This step writes to your KV Store. When building your application, you must set the environment variable `FASTLY_API_TOKEN` to a Fastly API token that has access to write to this KV Store. -> -> Alternatively, if this environment variable is not found, the tool will attempt to detect an API token by calling `fastly profile token`. +- Uses the default collection name +- Uploads static files to the KV Store +- Stores server configuration for the collection > [!TIP] -> By running `npx @fastly/cli compute build --verbose` (or `npm run build` directly), you should see output in your logs saying that files are being sent to the KV Store. +> Upload to a specific collection by specifying the collection name when publishing content: +> ```sh +> npm run publish-content -- --collection-name=preview-42 +> ``` -The `statics-metadata.js` file should now show `"type": "kv-store"` for content assets. -Your Wasm binary should also be smaller, as the content of the files are no longer inlined in the build artifact. -You can deploy this and run it from Fastly, and the referenced files will be served from KV Store. +**No Wasm redeploy needed** unless you: -You will also see entries in `fastly.toml` that represent the local KV Store. -These enable the site to also run correctly when served using the local development environment. +- Modify `src/index.js` - such as when you update your custom routing logic (e.g. collection selection) or +- Change `static-publish.rc.js` -### Cleaning unused items from KV Store +If you do need to redeploy, simply run: -The files that are uploaded to the KV Store are submitted using keys of the following format: - -`:__` +```sh +npm run deploy +``` -For example: -`12345abcde67890ABCDE00:/public/index.html_br_aeed29478691e67f6d5s36b4ded20c17e9eae437614617067a8751882368b965` +## 🧹 Cleaning Up -Using such a key ensures that whenever the file contents are identical, the same key will be generated. -This enables to detect whether an unchanged file already exists in the KV Store, avoiding having to re-submit -files that have not changed. If the file contents have changed, then a new hash is generated. This ensures that -even during the brief amount of time between deploys, any request served by a prior version will still serve the same -corresponding previous version of the content. +Every time you publish, old files are left behind for safety. **However, files with the same content will be re-used across collections and publishing events** - they are only stored once in the KV Store using their content hash as a key. This ensures that unchanged files aren't duplicated, keeping storage efficient and deduplicated. To avoid bloat, use: -However, this system never deletes files automatically. After many deployments, extraneous files may be left over. +```sh +npx @fastly/compute-js-static-publish clean --delete-expired-collections +``` -`@fastly/compute-js-static-publish` includes a feature to delete these old versions of the files that are no longer being -used. To run it, type the following command: +This removes: -`npx @fastly/compute-js-static-publish --clean-kv-store` +- Expired collection index files (only if `--delete-expired-collections` is passed) +- Unused content blobs (no longer referenced) +- Orphaned server config files -It works by scanning `statics-metadata.js` for all currently-used keys. Then it enumerates all the existing -keys in the configured KV Store and that belong to this application (can do so by narrowing down all keys to the ones -that begin with the "publish id"). If any of the keys is not in the list of currently-used keys, then a request is made -to delete that KV Store value. +### 🔍 Dry Run Mode -And that's it! It should be possible to run this task to clean up once in a while. +Preview what will be deleted without making changes: -## Advanced Usages +```sh +npx @fastly/compute-js-static-publish clean --dry-run +``` -### The `static-publish.rc.js` config file +> ⚠️ Cleanup never deletes the default collection and never deletes content that’s still in use. -* `rootDir` - _Required._ All files under this root directory will be included by default in the publishing, - except for those that are excluded using some of the following features. Files outside this root cannot be - included in the publishing. +## 🔄 Content Compression -* `staticPublisherWorkingDir` - _Required._ Static asset loader and metadata files are created under this directory. +This project supports pre-compressing and serving assets in Brotli and Gzip formats. Compression is controlled at two different stages: -* `kvStoreName` - Set this value to the _name_ of an existing KV Store to enable uploading of content assets - to Fastly KV Store. See [Using the KV Store](#kv-store) for more information. +- **During publishing**, the `contentCompression` field in the `publish` section of `publish-content.config.js` defines which compressed variants (e.g., `br`, `gzip`) should be generated and uploaded to the KV Store. -* `excludeDirs` - Specifies names of files and directories within `rootDir` to exclude from the publishing. Each entry can - be a string or a JavaScript `RegExp` object. Every file and directory under `rootDir` is checked against each entry of - the array by testing its path relative to `rootDir`. The file or directory (included all children) and excluded if the - condition matches: - * If a string is specified, then an exact match is checked. - * If a `RegExp` is specified, then it is tested with the regular expression. - * If this setting is not set, then the default value is `['./node_modules']`. - * If you specifically set this to the empty array, then no files are excluded by this mechanism. +Assets are stored in multiple formats (uncompressed + compressed) if configured. The following file types are compressed by default: -* `excludeDotfiles` - Unless disabled, will exclude all files and directories (and their children) - whose names begin with a `'.''`. This is `true` by default. +- Text-based: `.html`, `.js`, `.css`, `.svg`, `.json`, `.txt`, `.xml`, `.map` +- Certain binary formats: `.bmp`, `.tar` -* `includeWellKnown` - Unless disabled, will include a file or directory called `.well-known` - even if `excludeDotfiles` would normally exclude it. This is `true` by default. +- **At runtime**, the `allowedEncodings` field in the `server` section of `publish-content.config.js` specifies which compression formats the server is allowed to serve based on the client's `Accept-Encoding` header. -* `kvStoreAssetInclusionTest` - Optionally specify a test function that can be run against each enumerated asset during - the publishing, to determine whether to include the asset. For every file, this function is passed - the [asset key](#asset-keys), as well as its content type (MIME type string). You may return one of three values from - this function: - * Boolean `true` - Include the file. It is uploaded to the KV Store. - * Boolean `false` - exclude the file. - * Object. Include the file. It is uploaded to the KV Store. This object may specify, optionally: - * `contentType` to override the Content type - * `contentCompression` an array of strings to override the content compression types. Specify an empty array for no compression. +`PublisherServer` will serve the smallest appropriate version based on the `Accept-Encoding` header. -* `contentCompression` - During the publishing, the tool will pre-generate compressed versions of content assets in these - formats and make them available to the Publisher Server or your application. Default value is [ 'br' | 'gzip' ]. +## 🧩 Using PublisherServer in Custom Apps -* `contentTypes` - Provide custom content types and/or override them. +You can combine PublisherServer with custom logic to support APIs, authentication, redirects, or A/B testing. `PublisherServer` returns `null` when it cannot handle a request, allowing you to chain in your own logic. - This tool comes with a [default set of content types](./src/util/content-types.ts) defined for many common - file extensions. This list can be used to add to and/or override items in the default list. - Content type definitions are checked in the provided order, and if none of them match, the default content types are - tested afterward. +```js +import { PublisherServer } from '@fastly/compute-js-static-publish'; +import rc from '../static-publish.rc.js'; - Provide these as an array of content type definition objects, each with the following keys and values: - * `test` - a RegExp or function to perform on the asset key. If the test succeeds, then the content asset is considered - to be of this content type definition. - * `contentType` - The content type header to apply when serving an asset of this content type definition. - * `text` - If `true`, this content type definition is considered to contain textual data. This makes `.text()` and `.json()` - available for calling on store entries. If not specified, this is treated as `false`. - * `precompressAsset` - When `true`, this tool generates pre-compressed versions of content assets and serves them to user - agents that assert an appropriate `Accept` header. See [Content Compression*](#content-compression) for details. If - not specified, this is `true` if `text` is `true`, and `false` if `text` is not `true`. - - For example, to add a custom content type `application/x-custom` for files that have a `.custom` extension, not treat - it as a text file, but precompress it during the generation of the application, add the following to your - `static-publish.rc.js` file: +const publisherServer = PublisherServer.fromStaticPublishRc(rc); - ```javascript - const config = { - /* ... other config ... */ - contentTypes: [ - { test: /\.custom$/, contentType: 'application/x-custom', text: false, precompressAsset: true }, - ], - }; - ``` +addEventListener("fetch", event => { + event.respondWith(handleRequest(event.request)); +}); - > Note that content types are tested at publishing time, not at runtime. +async function handleRequest(request) { + const response = await publisherServer.serveRequest(request); + if (response) { + return response; + } -* `server` - [Configuration of `PublisherServer()`](#configuring-publisherserver). - above. + // Add your custom logic here + if (request.url.endsWith('/api/hello')) { + return new Response(JSON.stringify({ greeting: "hi" }), { + headers: { 'content-type': 'application/json' } + }); + } -### Running custom code alongside Publisher Server + return new Response("Not Found", { status: 404 }); +} +``` -The generated `./src/index.js` program instantiates the server and simply asks it to respond to a request. +## 📥 Using Published Assets in Your Code -You are free to add code to this file. +To access files you've published, use the `getMatchingAsset()` and `loadKvAssetVariant()` methods on `publisherServer`. -For example, if the `PublisherServer` is unable to formulate a response to the request, then it returns `null`. You may -add your own code to handle these cases, such as to provide custom responses. +### Access Metadata for a File: ```js -import { getPublisherServer } from './statics.js'; -const publisherServer = getPublisherServer(); - -addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); -async function handleRequest(event) { - - const response = await publisherServer.serveRequest(event.request); - if (response != null) { - return response; - } - - // Do custom things here! - // Handle API requests, serve non-static responses, etc. - - return new Response('Not found', { status: 404 }); +const asset = await publisherServer.getMatchingAsset('/index.html'); +if (asset != null) { + // Asset exists with that path + asset.contentType; // e.g., 'text/html' + asset.lastModifiedTime; // Unix timestamp + asset.size; // Size in bytes of the base version + asset.hash; // SHA256 hash of base version + asset.variants; // Available variants (e.g., ['gzip']) } ``` -### Using published assets in your own application +### Load the File from KV Store: -Publishing, as described earlier, is the process of preparing files for inclusion into your application. -This process also makes metadata available about each of the files that are included, such as its content type, the last -modified date, the file hash, and so on. +```js +const kvAssetVariant = await publisherServer.loadKvAssetVariant(asset, null); // pass 'gzip' or 'br' for compressed -Publishing can also set up to embed content and modules into the Wasm binary of your Compute app if so configured, -although this usage will require your Wasm binary to be rebuilt and deployed to your service. +kvAssetVariant.kvStoreEntry; // KV Store entry +kvAssetVariant.size; // Size of the variant +kvAssetVariant.hash; // SHA256 of the variant +kvAssetVariant.contentEncoding; // 'gzip', 'br', or null +kvAssetVariant.numChunks; // Number of chunks (for large files) +``` -The [`PublisherServer` class](#publisherserver) used by the default scaffolded application is a simple application of this content -and metadata. By importing `./statics.js` into your Compute application, you can just as easily access this -information about the assets that were included during publishing. +You can stream `kvAssetVariant.kvStoreEntry.body` directly to a `Response`, or read it using `.text()`, `.json()`, or `.arrayBuffer()` depending on its content type. -> IMPORTANT: Use a static `import` statement, rather than using `await import()` to load `./statics.js`, in order to -ensure that its top-level code runs during the initialization phase of your Compute application. +--- -#### Assets +## 📚 CLI Reference -There are two categories of assets: Content Assets and Module Assets. +### Available Commands -* A Content Asset is a type of asset where your application or a user of your application is interested in the text or -binary contents of an asset. +#### Outside a Compute App Directory +- `npx @fastly/compute-js-static-publish [options]` - Scaffold a new Compute app - The data of each content asset can exist in one of two stores: - * Inline Store - this is a data store that exists within the Wasm binary. - * Fastly KV Store - Fastly's distributed edge data store. Data can be placed here without impacting the size of - your Wasm binary. +#### Inside a Compute App Directory +- `publish-content` - Publish static files to the KV Store under a named collection +- `clean` - Delete expired and unreferenced KV entries +- `collections list` - List all published collections +- `collections delete` - Delete a specific collection index +- `collections promote` - Copy a collection to another name +- `collections update-expiration` - Modify expiration time for an existing collection - Your application can stream the contents of these assets to a visitor, or read from the stream itself and access its - contents. +### 🛠 App Scaffolding -* A Module Asset is a type of asset where your application wants to load the asset as a module, and use it as part of its -running code. Their contents are actually built at publishing time and their built representation is included in the Wasm -binary. They can be imported statically at will, and your application is able to execute the code exported by these modules. +Run outside an existing Compute app directory: -##### Asset Keys +```sh +npx @fastly/compute-js-static-publish \ + --root-dir=./public \ + --kv-store-name=site-content \ + [--output=./compute-js] \ + [--publish-id=] \ + [--public-dir=./public] \ + [--static-dir=./public/static] \ + [--static-publisher-working-dir=./compute-js/static-publisher] \ + [--spa=./public/spa.html] \ + [--not-found-page=./public/404.html] \ + [--auto-index=index.html,index.htm] \ + [--auto-ext=.html,.htm] \ + [--name=my-site] \ + [--description='My Compute static site'] \ + [--author='you@example.com'] \ + [--service-id=your-fastly-service-id] +``` -When working with content assets or module assets from your application, they are referenced by their asset key, which -is the relative path of the file from `rootDir`, including the leading slash. +#### Options: + +**Used to generate the Compute app:** +- `--kv-store-name`: Required. Name of KV Store to use. +- `--output`: Compute app destination (default: `./compute-js`) +- `--name`: Application name to insert into `fastly.toml` +- `--description`: Application description to insert into `fastly.toml` +- `--author`: Author to insert into `fastly.toml` +- `--service-id`: Optional existing Fastly Compute service ID + +**Used in building config files:** +- `--root-dir`: Required. Directory of static site content. +- `--publish-id`: Optional key prefix for KV entries (default: `'default'`). +- `--static-publisher-working-dir`: Directory to hold working files (default: `/static-publisher`). +- `--public-dir`: Public files base directory (default: same as `--root-dir`). +- `--static-dir`: One or more directories to serve with long cache TTLs. +- `--spa`: SPA fallback file path (e.g., `./public/spa.html`). +- `--not-found-page`: 404 fallback file path (e.g., `./public/404.html`). +- `--auto-index`: List of filenames to use as index files. +- `--auto-ext`: Extensions to try when resolving URLs. + +### 🚀 Inside a Compute App Directory + +Once you're in the scaffolded Compute app directory (with `static-publish.rc.js` present), you can run these subcommands: + +#### `publish-content` + +```sh +npx @fastly/compute-js-static-publish publish-content \ + [--root-dir=./public] \ + [--collection-name=preview-42] \ + [--config=./publish-content.config.js] \ + [--expires-in=7d | --expires-at=2025-05-01T12:00Z] \ + [--local-only | --no-local] \ + [--fastly-api-token=...] +``` -#### Content Assets +Publishes your static files and server config for a given collection. -You can obtain the content assets included in publishing by importing the `contentAssets` object exported from -`./statics.js`. +#### Options: -```js -import { contentAssets } from './statics'; +**Configuration:** -// Obtain a content asset named '/public/index.html' -const asset = contentAssets.getAsset('/public/index.html'); +- `--config`: Path to a config file to configure server behavior for this collection (default: `./publish-content.config.js`) -// 'wasm-inline' if object's data is 'inlined' into Wasm binary -// 'kv-store' if object's data exists in Fastly KV Store -asset.type; +**Content and Collection:** -// Get the "store entry" -const storeEntry = await asset.getStoreEntry(); +- `--root-dir`: Source directory to read files from (overrides value in `publish-content.config.js`) +- `--collection-name`: Name of the collection to create/update (default: value in `static-publish.rc.js`) +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format: `2025-05-01T12:00Z`) *(Only one of **`--expires-in`** or **`--expires-at`** may be specified)* -storeEntry.contentEncoding; // null, 'br', 'gzip' -``` +**Mode:** -Regardless of which store these objects come from, they implement the `Body` interface as defined by `@fastly/js-compute`. -As such, you are able to work with them in the same way to obtain their contents: +- `--local-only`: Do not upload files to Fastly KV Store; only simulate KV Store locally +- `--no-local`: Do not prepare files for local simulated KV Store; upload to real KV Store only -```js -storeEntry.body; // ReadableStream -storeEntry.bodyUsed; // true if consumed or distrubed +**Auth:** -// Get the data as ArrayBuffer, parsed JSON, or string -// The latter two are only available if the data is a text type -const arrayBuffer = await storeEntry.arrayBuffer(); -const json = await storeEntry.json(); -const text = await storeEntry.text(); -``` +- `--fastly-api-token`: API token to use when publishing\ + *(Overrides **`FASTLY_API_TOKEN`** environment variable and **`fastly profile token`**)* +- Stores server config per collection +- Supports expiration settings -Or, if you don't care about the contents but just want to stream it to the visitor, you can pass the `.body` field directly -to the Response constructor: +#### `clean` -```js -const response = new Response(storeEntry.body, { status: 200 }); +```sh +npx @fastly/compute-js-static-publish clean \ + [--delete-expired-collections] \ + [--dry-run] ``` -> IMPORTANT: Once a store entry is consumed, its body cannot be read from again. If you need to access the contents of the -same asset more than once, you may obtain another store entry, as in the following example: -```js -import { contentAssets } from './statics'; -const asset = contentAssets.getAsset('/public/index.html'); -const entry1 = await asset.getStoreEntry(); // Get a new store entry -const json1a = await entry1.json(); -const json1b = await entry1.json(); // Can't do this, the body has already been consumed! - -const entry2 = await asset.getStoreEntry(); // Get a new store entry for same asset -const json2a = await entry2.json(); // This will work. -``` +Removes unreferenced content from the KV Store. -#### Module Assets +#### Options: -Module assets are useful when an asset includes executable JavaScript code that you may want to execute at runtime. +- `--delete-expired-collections`: Also delete collection index files that have expired +- `--dry-run`: Show what would be deleted without actually removing anything -You can obtain the module assets included in publishing by importing the `moduleAssets` object exported from -`./statics.js`. Keep in mind that by default, no modules are included in `moduleAssets`. If you wish to include module -assets, you must configure your publishing to include them. See [`moduleAssetInclusionTest` in the `static-publish.rc.js` -config file](#static-publish-rc) for more details. +#### `collections list` -`/module/hello.js` -```js -export function hello() { - console.log('Hello, World!'); -} +```sh +npx @fastly/compute-js-static-publish collections list ``` +Lists all known collection names and metadata. -```js -import { moduleAssets } from './statics'; - -// Obtain a module asset named '/module/hello.js' -const asset = moduleAssets.getAsset('/module/hello.js'); +#### `collections promote` -// Load the module -const helloModule = await asset.getModule(); - -helloModule.hello(); // Will print "Hello, World!" +```sh +npx @fastly/compute-js-static-publish collections promote \ + --collection-name=preview-42 \ + --to=live \ + [--expires-in=7d | --expires-at=2025-06-01T00:00Z] ``` +Copies an existing collection (content + config) to a new collection name. -#### Metadata - -In some use cases, you may have a use case where you need to know about the files that were included during publishing, -but not in the context of Compute. (e.g., a tool that runs in Node.js that performs some maintenance task on assets). - -You cannot import `./statics.js` from a Node.js application, as it holds dependencies on Compute. +#### Options: +- `--collection-name`: The name of the source collection to promote (required) +- `--to`: The name of the new (target) collection to create or overwrite (required) +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format) -Instead, you can import `./statics-metadata.js`, a companion file that is generated in the same directory. This file -exposes plain JavaScript objects that contain the metadata about your content assets that were included in the final -publishing event. +*Exactly one of **`--expires-in`** or **`--expires-at`** may be provided.* -See the definition of `ContentAssetMetadataMapEntry` in the [`types/content-assets` file](./src/types/content-assets.ts) for more details. +#### `collections update-expiration` -## Migrating +```sh +npx @fastly/compute-js-static-publish collections update-expiration \ + --collection-name=preview-42 \ + --expires-in=3d | --expires-at=2025-06-01T00:00Z +``` +Sets or updates the expiration time of a collection. -See [MIGRATING.md](./MIGRATING.md). +#### Options: +- `--collection-name`: The name of the collection to update (required) +- `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) +- `--expires-at`: Absolute expiration time (ISO format) -## Issues +*Exactly one of **`--expires-in`** or **`--expires-at`** must be provided.* -If you encounter any non-security-related bug or unexpected behavior, please [file an issue][bug]. +#### `collections delete` -[bug]: https://github.com/fastly/compute-js-static-publish/issues/new?labels=bug +```sh +npx @fastly/compute-js-static-publish collections delete \ + --collection-name=preview-42 +``` +Deletes a specific collection’s index and associated settings. -### Security issues +#### Options: +- `--collection-name`: The name of the collection to delete (required) -Please see our [SECURITY.md](./SECURITY.md) for guidance on reporting security-related issues. +--- -## License +## 📚 Next Steps -[MIT](./LICENSE). +- View CLI command help: `npx @fastly/compute-js-static-publish --help` +- Use in CI to automate branch previews +- Visit [https://developer.fastly.com](https://developer.fastly.com) for Compute platform docs diff --git a/README.short.md b/README.short.md new file mode 100644 index 0000000..947713f --- /dev/null +++ b/README.short.md @@ -0,0 +1,53 @@ +# @fastly/compute-js-static-publish + +Fastly Compute + KV Store for static websites and web apps. + +This CLI tool helps you: + +- ✅ Deploy static sites to Fastly Compute with zero backend +- 📦 Store files in Fastly KV Store efficiently +- 🗂 Publish to named collections (`live`, `preview-42`, etc.) +- 🔄 Switch between collections at runtime +- 🧹 Clean up old or expired assets + +--- + +## Quick Start + +Create a directory for your project, place your static files in `./public`, then type: + +```sh +npx @fastly/compute-js-static-publish --root-dir=./public --kv-store-name=site-content +``` + +### 🔧 Local Preview + +```sh +cd compute-js +npm install +npm run start # preview locally +``` + +Serves your app at `http://127.0.0.1:7676`, powered by a simulated KV Store. + +### 🚀 Deploy to Production + +When you're ready to go live, [create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already, and then: + +```sh +cd compute-js +npm run deploy # deploy the app +npm run publish-content # upload your static files +``` + +## Features + +- Named collections for previews, staging, production +- SPA + fallback handling +- Precompressed Brotli/gzip support +- CLI tools for publish, promote, and cleanup + +## Documentation + +📘 Full documentation available on GitHub: +[https://github.com/fastly/compute-js-static-publish](https://github.com/fastly/compute-js-static-publish) From 0f18f557ae6aab343dced8876f1f1433433e191c Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 16 Apr 2025 17:38:39 +0900 Subject: [PATCH 03/20] v7.0.0-beta.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 116acb8..23fef9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "6.3.0", + "version": "7.0.0-beta.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "6.3.0", + "version": "7.0.0-beta.0", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index ecaf569..af50b84 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "6.3.0", + "version": "7.0.0-beta.0", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From 0128972fa015849e9409c21b4bf7dd1bb631317c Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 17 Apr 2025 12:39:28 +0900 Subject: [PATCH 04/20] Additional v7 updates --- src/cli/commands/init-app.ts | 12 +++++++++--- src/server/publisher-server/index.ts | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/cli/commands/init-app.ts b/src/cli/commands/init-app.ts index a553b98..4239914 100644 --- a/src/cli/commands/init-app.ts +++ b/src/cli/commands/init-app.ts @@ -598,7 +598,7 @@ export async function action(argv: string[]) { scripts: { prestart: "npx @fastly/compute-js-static-publish publish-content --local-only", start: "fastly compute serve", - deploy: "fastly compute publish", + "publish-service": "fastly compute publish", "publish-content": 'npx @fastly/compute-js-static-publish publish-content', build: 'js-compute-runtime ./src/index.js ./bin/main.wasm' }, @@ -724,6 +724,7 @@ export default config; // src/index.js resourceFiles['./src/index.js'] = /* language=text */ `\ /// +import { env } from 'fastly:env'; import { PublisherServer } from '@fastly/compute-js-static-publish'; import rc from '../static-publish.rc.js'; const publisherServer = PublisherServer.fromStaticPublishRc(rc); @@ -732,7 +733,11 @@ const publisherServer = PublisherServer.fromStaticPublishRc(rc); addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); async function handleRequest(event) { - const response = await publisherServer.serveRequest(event.request); + console.log('FASTLY_SERVICE_VERSION', env('FASTLY_SERVICE_VERSION')); + + const request = event.request; + + const response = await publisherServer.serveRequest(request); if (response != null) { return response; } @@ -767,6 +772,7 @@ async function handleRequest(event) { console.log('To build and deploy to your Compute service:'); console.log(''); console.log(' cd ' + COMPUTE_JS_DIR); - console.log(' npm run deploy'); + console.log(' npm run publish-service'); + console.log(' npm run publish-content'); console.log(''); } diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts index 1f7b956..9161bc3 100644 --- a/src/server/publisher-server/index.ts +++ b/src/server/publisher-server/index.ts @@ -101,7 +101,6 @@ export class PublisherServer { } // Server config is obtained from the KV Store, and cached for the duration of this object. - // TODO get from simple cache async getServerConfig() { const settingsFileKey = `${this.publishId}_settings_${this.activeCollectionName}`; const kvStore = new KVStore(this.kvStoreName); @@ -465,6 +464,12 @@ export class PublisherServer { const url = new URL(request.url); const pathname = decodeURI(url.pathname); + + // Custom health check route + if (pathname === '/healthz') { + return new Response("OK", { status: 200 }); + } + const serverConfig = await this.getServerConfig(); if (serverConfig == null) { return new Response( From 87584175e65970e6b50fc937285c651963e43d2d Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 17 Apr 2025 12:39:36 +0900 Subject: [PATCH 05/20] Additional Readme updates --- README.md | 12 ++++++------ README.short.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5bd650b..d40e1fa 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ - [🏁 Quick Start](#-quick-start) - [⚙️ Configuring `static-publish.rc.js`](#️-configuring-static-publishrcjs) - [🧾 Default Server Config: `publish-content.config.js`](#-default-server-config-publish-contentconfigjs) -- [📦 Collections (Preview, Deploy, Promote)](#-collections-preview-deploy-promote) +- [📦 Collections (Publish, Preview, Promote)](#-collections-publish-preview-promote) - [🧹 Cleaning Up](#-cleaning-up) - [🔄 Content Compression](#-content-compression) - [🧩 Using PublisherServer in Custom Apps](#-using-publisherserver-in-custom-apps) @@ -65,7 +65,7 @@ Fastly's [local development environment](https://www.fastly.com/documentation/gu Ready to go live? All you need is a [free Fastly account](https://www.fastly.com/signup/?tier=free)! ```sh -npm run deploy +npm run publish-service ``` The command publishes your Compute app and creates the KV Store. (No content uploaded yet!) @@ -202,7 +202,7 @@ export default config; - `allowedEncodings` - Specifies which compression formats the server is allowed to serve based on the client's `Accept-Encoding` header. -## 📦 Collections (Preview, Deploy, Promote) +## 📦 Collections (Publish, Preview, Promote) Collections are a powerful feature that allow you to publish and manage multiple versions of your site simultaneously. Each collection is a named set of: @@ -326,7 +326,7 @@ During development, starting the local preview server (`npm run start`) will run When you're ready for production: 1. [Create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already. -2. Run `npm run deploy` +2. Run `npm run publish-service` - This builds your Compute app into a Wasm binary - Deploys it to a new or existing Fastly Compute service - If creating a new service: @@ -356,10 +356,10 @@ This: - Modify `src/index.js` - such as when you update your custom routing logic (e.g. collection selection) or - Change `static-publish.rc.js` -If you do need to redeploy, simply run: +If you do need to rebuild and redeploy the Compute app, simply run: ```sh -npm run deploy +npm run publish-service ``` ## 🧹 Cleaning Up diff --git a/README.short.md b/README.short.md index 947713f..4dbd7b8 100644 --- a/README.short.md +++ b/README.short.md @@ -36,8 +36,8 @@ When you're ready to go live, [create a free Fastly account](https://www.fastly. ```sh cd compute-js -npm run deploy # deploy the app -npm run publish-content # upload your static files +npm run publish-service # deploy the app (publish the "service") +npm run publish-content # upload your static files (publish the "content") ``` ## Features From d2303418e9f9666516657fff0f7bd016fcbdbef6 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 17 Apr 2025 16:29:36 +0900 Subject: [PATCH 06/20] v7.0.0-beta.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 23fef9e..128beb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.0", + "version": "7.0.0-beta.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.0", + "version": "7.0.0-beta.1", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index af50b84..acfd297 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "7.0.0-beta.0", + "version": "7.0.0-beta.1", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From dbc78775ea7f5053e63f71d22e255ef061f0ee3b Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Mon, 21 Apr 2025 14:38:45 +0900 Subject: [PATCH 07/20] Additional v7 updates --- package-lock.json | 14 +- package.json | 3 +- src/cli/commands/clean.ts | 161 ----------- src/cli/commands/collections/list.ts | 80 ------ src/cli/commands/manage/clean.ts | 254 +++++++++++++++++ .../{ => manage}/collections/delete.ts | 93 +++++-- .../{ => manage}/collections/index.ts | 66 ++--- src/cli/commands/manage/collections/list.ts | 156 +++++++++++ .../commands/manage/collections/promote.ts | 187 +++++++++++++ .../manage/collections/update-expiration.ts | 159 +++++++++++ src/cli/commands/{ => manage}/index.ts | 76 +++--- .../commands/{ => manage}/publish-content.ts | 257 ++++++++++++------ .../{init-app.ts => scaffold/index.ts} | 97 +++++-- src/cli/fastly-api/api-token.ts | 13 +- src/cli/fastly-api/kv-store.ts | 4 +- src/cli/index.ts | 17 +- src/cli/util/args.ts | 72 ++++- src/cli/util/fastly-toml.ts | 17 ++ src/cli/util/kv-store-items.ts | 41 ++- src/cli/util/node.ts | 11 + src/models/server/index.ts | 4 + src/models/time/index.ts | 66 +++++ src/server/publisher-server/index.ts | 45 ++- 23 files changed, 1401 insertions(+), 492 deletions(-) delete mode 100644 src/cli/commands/clean.ts delete mode 100644 src/cli/commands/collections/list.ts create mode 100644 src/cli/commands/manage/clean.ts rename src/cli/commands/{ => manage}/collections/delete.ts (53%) rename src/cli/commands/{ => manage}/collections/index.ts (73%) create mode 100644 src/cli/commands/manage/collections/list.ts create mode 100644 src/cli/commands/manage/collections/promote.ts create mode 100644 src/cli/commands/manage/collections/update-expiration.ts rename src/cli/commands/{ => manage}/index.ts (76%) rename src/cli/commands/{ => manage}/publish-content.ts (72%) rename src/cli/commands/{init-app.ts => scaffold/index.ts} (88%) create mode 100644 src/cli/util/fastly-toml.ts create mode 100644 src/cli/util/node.ts create mode 100644 src/models/server/index.ts create mode 100644 src/models/time/index.ts diff --git a/package-lock.json b/package-lock.json index 128beb2..0f8b5ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "dependencies": { "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", - "glob-to-regexp": "^0.4.1" + "glob-to-regexp": "^0.4.1", + "toml": "^3.0.0" }, "bin": { "compute-js-static-publish": "build/cli/index.js" @@ -2483,6 +2484,12 @@ "dev": true, "license": "MIT" }, + "node_modules/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -4049,6 +4056,11 @@ "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", "dev": true }, + "toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, "tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", diff --git a/package.json b/package.json index acfd297..8c03c9d 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "dependencies": { "@fastly/cli": "^11.2.0", "command-line-args": "^5.2.1", - "glob-to-regexp": "^0.4.1" + "glob-to-regexp": "^0.4.1", + "toml": "^3.0.0" }, "peerDependencies": { "@fastly/js-compute": "^3.33.2" diff --git a/src/cli/commands/clean.ts b/src/cli/commands/clean.ts deleted file mode 100644 index daf2a00..0000000 --- a/src/cli/commands/clean.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright Fastly, Inc. - * Licensed under the MIT license. See LICENSE file for details. - */ - -import commandLineArgs, { type OptionDefinition } from 'command-line-args'; - -import { type KVAssetEntryMap } from '../../models/assets/kvstore-assets.js'; -import { LoadConfigError, loadStaticPublisherRcFile } from '../util/config.js'; -import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteFile } from '../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../fastly-api/api-token.js'; - -export async function action(argv: string[]) { - - const optionDefinitions: OptionDefinition[] = [ - { name: 'verbose', type: Boolean }, - { name: 'fastly-api-token', type: String, }, - ]; - - const commandLineValues = commandLineArgs(optionDefinitions, { argv }); - const { - verbose, - ['fastly-api-token']: fastlyApiToken, - } = commandLineValues; - - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); - process.exitCode = 1; - return; - } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); - - // #### load config - let staticPublisherRc; - try { - staticPublisherRc = await loadStaticPublisherRcFile(); - } catch (err) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - if (err instanceof LoadConfigError) { - for (const error of err.errors) { - console.error(error); - } - } - process.exitCode = 1; - return; - } - - const publishId = staticPublisherRc.publishId; - console.log(`✔️ Publish ID: ${publishId}`); - - const kvStoreName = staticPublisherRc.kvStoreName; - console.log(`✔️ Using KV Store: ${kvStoreName}`); - - const defaultCollectionName = staticPublisherRc.defaultCollectionName; - console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); - - // ### KVStore Keys to delete - const kvKeysToDelete = new Set(); - - // ### List all indexes ### - const indexesPrefix = publishId + '_index_'; - const indexKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - indexesPrefix, - ); - if (indexKeys == null) { - throw new Error(`Can't query indexes in KV Store`); - } - - // ### Found collections ### - const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); - console.log('Found collections'); - for (const collection of foundCollections) { - console.log(collection); - } - - // TODO ### Determine which ones are not expired, based on an expiration meta - const liveCollections = foundCollections; - - // ### List all settings ### - const settingsPrefix = publishId + '_index_'; - const settingsKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - settingsPrefix, - ); - if (settingsKeys == null) { - throw new Error(`Can't query settings in KV Store`); - } - - // If a settings object is found that doesn't match a live collection name then - // mark it for deletion - const foundSettingsCollections = settingsKeys.map(key => ({ key, name: key.slice(settingsPrefix.length) })); - for (const foundSettings of foundSettingsCollections) { - if (!foundCollections.includes(foundSettings.name)) { - kvKeysToDelete.add(foundSettings.key); - } - } - - // ### Go through the index files and make a list of all keys (hashes) that we are keeping - const assetsIdsInUse = new Set(); - for (const collection of liveCollections) { - - // TODO deal with when the index file is > 20MB - const kvAssetsIndexResponse = await getKvStoreEntry( - fastlyApiContext, - kvStoreName, - indexesPrefix + collection - ); - if (!kvAssetsIndexResponse) { - throw new Error(`Can't load KV Store entry ${indexesPrefix + collection}`); - } - const kvAssetsIndex = (await kvAssetsIndexResponse.response.json()) as KVAssetEntryMap; - for (const [_assetKey, assetEntry] of Object.entries(kvAssetsIndex)) { - if (assetEntry.key.startsWith('sha256:')) { - assetsIdsInUse.add(`sha256_${assetEntry.key.slice('sha256:'.length)}`); - } - } - } - - // ### Obtain the assets in the KV Store and find the ones that are not in use - const assetPrefix = publishId + '_files_'; - const assetKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - assetPrefix, - ); - if (assetKeys == null) { - throw new Error(`Can't query assets in KV Store`); - } - - for (const assetKey of assetKeys) { - let assetId = assetKey.slice(assetPrefix.length); - if (assetId.startsWith('sha256_')) { - assetId = assetId.slice(0, 'sha256_'.length + 64); - } else { - // If we don't know what the prefix is, we ignore it - continue; - } - - if (assetsIdsInUse.has(assetId)) { - console.log('Asset ID ' + assetId + ' in use'); - } else { - kvKeysToDelete.add(assetKey); - console.log('Asset ID ' + assetId + ' not in use'); - } - } - - // ### Delete items that have been flagged - for (const key of kvKeysToDelete) { - console.log("Deleting item: " + key); - await kvStoreDeleteFile(fastlyApiContext, kvStoreName, key); - } - - console.log("✅ Completed.") -} diff --git a/src/cli/commands/collections/list.ts b/src/cli/commands/collections/list.ts deleted file mode 100644 index 7d32a63..0000000 --- a/src/cli/commands/collections/list.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Fastly, Inc. - * Licensed under the MIT license. See LICENSE file for details. - */ - -import commandLineArgs, { type OptionDefinition } from 'command-line-args'; - -import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; -import { getKVStoreKeys } from '../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; - -export async function action(argv: string[]) { - - const optionDefinitions: OptionDefinition[] = [ - { name: 'verbose', type: Boolean }, - { name: 'fastly-api-token', type: String, }, - ]; - - const commandLineValues = commandLineArgs(optionDefinitions, { argv }); - const { - verbose, - ['fastly-api-token']: fastlyApiToken, - } = commandLineValues; - - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); - process.exitCode = 1; - return; - } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); - - // #### load config - let staticPublisherRc; - try { - staticPublisherRc = await loadStaticPublisherRcFile(); - } catch (err) { - console.error("❌ Can't load static-publish.rc.js"); - console.error("Run this from a compute-js-static-publish compute-js directory."); - if (err instanceof LoadConfigError) { - for (const error of err.errors) { - console.error(error); - } - } - process.exitCode = 1; - return; - } - - const publishId = staticPublisherRc.publishId; - console.log(`✔️ Publish ID: ${publishId}`); - - const kvStoreName = staticPublisherRc.kvStoreName; - console.log(`✔️ Using KV Store: ${kvStoreName}`); - - const defaultCollectionName = staticPublisherRc.defaultCollectionName; - console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); - - // ### List all indexes ### - const indexesPrefix = publishId + '_index_'; - const indexKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - indexesPrefix, - ); - if (indexKeys == null) { - throw new Error(`Can't query indexes in KV Store`); - } - - // ### Found collections ### - const foundCollections = indexKeys.map(key => ({ key, name: key.slice(indexesPrefix.length), })); - if (foundCollections.length === 0) { - console.log('No collections found.'); - } else { - console.log(`Found collections: ${foundCollections.map(x => `'${x.name}'`).join(', ')}`); - } - - console.log("✅ Completed.") -} diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts new file mode 100644 index 0000000..5a50bfb --- /dev/null +++ b/src/cli/commands/manage/clean.ts @@ -0,0 +1,254 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap } from '../../../models/assets/kvstore-assets.js'; +import { type IndexMetadata } from '../../../models/server/index.js'; +import { isExpired } from '../../../models/time/index.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; +import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteEntry } from '../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; +import { parseCommandLine } from '../../util/args.js'; +import { doKvStoreItemsOperation } from "../../util/kv-store-items.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish clean [options] + +Description: + Cleans up expired or unreferenced items in the Fastly KV Store. + This includes expired collection indexes and orphaned content assets. + +Options: + --delete-expired-collections If set, expired collection index files will be deleted. + --dry-run Show what would be deleted without performing any deletions. + + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'delete-expired-collections', type: Boolean }, + { name: 'dry-run', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['delete-expired-collections']: deleteExpiredCollections, + ['dry-run']: dryRun, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + // ### KVStore Keys to delete + const kvKeysToDelete = new Set(); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + const indexKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + indexesPrefix, + ); + if (indexKeys == null) { + throw new Error(`Can't query indexes in KV Store`); + } + + // ### Go through the index files, make lists + const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); + + // All collection names that we are keeping + const liveCollections = new Set(); + + // All asset keys (hashes) that we are keeping + const assetsIdsInUse = new Set(); + + console.log('Processing collections:'); + await doKvStoreItemsOperation( + foundCollections.map(collection => ({ + key: indexesPrefix + collection, + collection, + })), + async({collection}, indexKey) => { + console.log(`Collection: ${collection}`); + + // TODO deal with when the index file is > 20MB + const kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext, + kvStoreName, + indexKey, + ); + if (!kvAssetsIndexResponse) { + throw new Error(`Can't load KV Store entry ${indexesPrefix + collection}`); + } + + let indexMetadata: IndexMetadata = {}; + if (kvAssetsIndexResponse.metadata != null) { + try { + indexMetadata = JSON.parse(kvAssetsIndexResponse.metadata) as IndexMetadata; + } catch { + } + } + + let isLive = true; + if (indexMetadata.expirationTime == null) { + console.log(' Collection has no expiration time set'); + } else { + console.log(' ⏰ Expiration: ' + new Date(indexMetadata.expirationTime * 1000)); + if (collection === defaultCollectionName) { + console.log(` ✅ Expiration time not enforced for default collection.`); + } else if (isExpired(indexMetadata.expirationTime)) { + if (!deleteExpiredCollections) { + console.log(` ⚠️ Use --delete-expired-collections to delete expired collections.`); + } else { + console.log(` 🗑️ Marking expired collection for deletion.`); + kvKeysToDelete.add(indexKey); + isLive = false; + + } + } + } + if (!isLive) { + return; + } + + liveCollections.add(collection); + const kvAssetsIndex = (await kvAssetsIndexResponse.response.json()) as KVAssetEntryMap; + for (const [_assetKey, assetEntry] of Object.entries(kvAssetsIndex)) { + if (assetEntry.key.startsWith('sha256:')) { + assetsIdsInUse.add(`sha256_${assetEntry.key.slice('sha256:'.length)}`); + } + } + } + ) + console.log(''); + + // ### List all settings ### + console.log('Enumerating settings:'); + const settingsPrefix = publishId + '_settings_'; + const settingsKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + settingsPrefix, + ); + if (settingsKeys == null) { + throw new Error(`Can't query settings in KV Store`); + } + + // If a settings object is found that doesn't match a live collection name then + // mark it for deletion + const foundSettingsCollections = settingsKeys.map(key => ({ key, name: key.slice(settingsPrefix.length) })); + for (const foundSettings of foundSettingsCollections) { + console.log(`Settings: ${foundSettings.name}`); + if (!liveCollections.has(foundSettings.name)) { + console.log(` 🗑️ Does not match a live index, marking for deletion.`); + kvKeysToDelete.add(foundSettings.key); + } + } + console.log(''); + + // ### Obtain the assets in the KV Store and find the ones that are not in use + console.log('Enumerating assets:'); + const assetPrefix = publishId + '_files_'; + const assetKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + assetPrefix, + ); + if (assetKeys == null) { + throw new Error(`Can't query assets in KV Store`); + } + + for (const assetKey of assetKeys) { + let assetId = assetKey.slice(assetPrefix.length); + if (assetId.startsWith('sha256_')) { + assetId = assetId.slice(0, 'sha256_'.length + 64); + } else { + // If we don't know what the prefix is, we ignore it + continue; + } + + if (assetsIdsInUse.has(assetId)) { + console.log(` ${assetId}: in use`); + } else { + kvKeysToDelete.add(assetKey); + console.log(` ${assetId}: not in use - ✅ marking for deletion`); + } + } + console.log(''); + + // ### Delete items that have been flagged + const items = [...kvKeysToDelete].map(key => ({key})); + await doKvStoreItemsOperation( + items, + async(_, key) => { + if (dryRun) { + console.log(`[DRY RUN] Deleting item: ${key}`); + } else { + console.log(`Deleting item from KV Store: ${key}`); + await kvStoreDeleteEntry(fastlyApiContext, kvStoreName, key); + } + } + ); + + console.log('✅ Completed.') +} diff --git a/src/cli/commands/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts similarity index 53% rename from src/cli/commands/collections/delete.ts rename to src/cli/commands/manage/collections/delete.ts index 4dfb61f..b026b87 100644 --- a/src/cli/commands/collections/delete.ts +++ b/src/cli/commands/manage/collections/delete.ts @@ -3,34 +3,63 @@ * Licensed under the MIT license. See LICENSE file for details. */ -import commandLineArgs, { type OptionDefinition } from 'command-line-args'; - -import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; -import { getKVStoreKeys, kvStoreDeleteFile } from '../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; +import { type OptionDefinition } from 'command-line-args'; + +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { getKVStoreKeys, kvStoreDeleteEntry } from '../../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { parseCommandLine } from "../../../util/args.js"; +import { doKvStoreItemsOperation } from "../../../util/kv-store-items.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections delete \\ + --collection-name \\ + [options] + +Description: + Deletes a collection index from the KV Store. The content files will remain but will no longer be referenced. + +Options: + --collection-name (Required) The name of the collection to delete + + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} -export async function action(argv: string[]) { +export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String }, { name: 'fastly-api-token', type: String, }, - { name: 'collection-name', type: String, multiple: true } ]; - const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + const { verbose, ['fastly-api-token']: fastlyApiToken, ['collection-name']: collectionNameValue, - } = commandLineValues; - - const collectionNamesAsArray = (Array.isArray(collectionNameValue) ? collectionNameValue : [ collectionNameValue ]) - .filter(x => typeof x === 'string') - .map(x => x.split(',')) - .flat() - .map(x => x.trim()) - .filter(Boolean); - if (collectionNamesAsArray.length === 0) { + } = parsed.commandLineOptions; + + if (collectionNameValue == null) { console.error("❌ Required argument '--collection-name' not specified."); process.exitCode = 1; return; @@ -63,16 +92,22 @@ export async function action(argv: string[]) { } const publishId = staticPublisherRc.publishId; - console.log(`✔️ Publish ID: ${publishId}`); + console.log(` | Publish ID: ${publishId}`); const kvStoreName = staticPublisherRc.kvStoreName; - console.log(`✔️ Using KV Store: ${kvStoreName}`); + console.log(` | Using KV Store: ${kvStoreName}`); const defaultCollectionName = staticPublisherRc.defaultCollectionName; - console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + console.log(` | Default Collection Name: ${defaultCollectionName}`); - console.log(`✔️ Collections to delete: ${collectionNamesAsArray.map(x => `'${x}'`).join(', ')}`) - const collectionNames = new Set(collectionNamesAsArray); + const collectionName = collectionNameValue; + if (collectionName === defaultCollectionName) { + console.error(`❌ Cannot delete default collection: ${collectionName}`); + process.exitCode = 1; + return; + } + + console.log(`✔️ Collection to delete: ${collectionName}`); const kvKeysToDelete = new Set(); @@ -94,7 +129,7 @@ export async function action(argv: string[]) { } else { console.log(`Found collections: ${foundCollections.map(x => `'${x.name}'`).join(', ')}`); for (const collection of foundCollections) { - if (collectionNames.has(collection.name)) { + if (collection.name === collectionName) { console.log(`Flagging collection '${collection.name}' for deletion: ${collection.key}`); kvKeysToDelete.add(collection.key); } @@ -102,10 +137,14 @@ export async function action(argv: string[]) { } // ### Delete items that have been flagged - for (const key of kvKeysToDelete) { - console.log("Deleting key from KV Store: " + key); - await kvStoreDeleteFile(fastlyApiContext, kvStoreName, key); - } + const items = [...kvKeysToDelete].map(key => ({key})); + await doKvStoreItemsOperation( + items, + async(_, key) => { + console.log(`Deleting key from KV Store: ${key}`); + await kvStoreDeleteEntry(fastlyApiContext, kvStoreName, key); + } + ); console.log("✅ Completed.") } diff --git a/src/cli/commands/collections/index.ts b/src/cli/commands/manage/collections/index.ts similarity index 73% rename from src/cli/commands/collections/index.ts rename to src/cli/commands/manage/collections/index.ts index ef4b81f..adb6325 100644 --- a/src/cli/commands/collections/index.ts +++ b/src/cli/commands/manage/collections/index.ts @@ -3,41 +3,13 @@ * Licensed under the MIT license. See LICENSE file for details. */ -import { getCommandAndArgs } from '../../util/args.js'; -import * as collectionsDelete from './delete.js'; -import * as collectionsList from './list.js'; - -export async function action(actionArgs: string[]) { - - const modes = { - 'delete': collectionsDelete, - 'list': collectionsList, - }; - - const commandAndArgs = getCommandAndArgs( - actionArgs, - Object.keys(modes) as (keyof typeof modes)[], - ); - - if (commandAndArgs.needHelp) { - if (commandAndArgs.command != null) { - console.error(`Unknown subcommand: ${commandAndArgs.command}`); - console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); - console.error(); - process.exitCode = 1; - } - - help(); - return; - } - - const { command, argv, } = commandAndArgs; - await modes[command].action(argv); - -} +import { getCommandAndArgs } from '../../../util/args.js'; +import * as deleteCommand from './delete.js'; +import * as listCommand from './list.js'; +import * as promoteCommand from './promote.js'; +import * as updateExpirationCommand from './update-expiration.js'; function help() { - console.log(`\ Usage: @@ -49,6 +21,7 @@ Description: Available Subcommands: list List all published collections delete Delete a specific collection index + promote Copies an existing collection (content + config) to a new collection name update-expiration Modify expiration time for an existing collection Global Options: @@ -62,5 +35,32 @@ Examples: npx @fastly/compute-js-static-publish collections list npx @fastly/compute-js-static-publish collections delete --collection-name=preview-42 `); +} + +export async function action(actionArgs: string[]) { + + const modes = { + 'delete': deleteCommand, + 'list': listCommand, + 'promote': promoteCommand, + 'update-expiration': updateExpirationCommand, + }; + + const commandAndArgs = getCommandAndArgs(actionArgs, modes); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.command != null) { + console.error(`Unknown subcommand: ${commandAndArgs.command}`); + console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); } diff --git a/src/cli/commands/manage/collections/list.ts b/src/cli/commands/manage/collections/list.ts new file mode 100644 index 0000000..6f9e1fe --- /dev/null +++ b/src/cli/commands/manage/collections/list.ts @@ -0,0 +1,156 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type OptionDefinition } from 'command-line-args'; + +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { getKvStoreEntry, getKVStoreKeys } from '../../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { parseCommandLine } from '../../../util/args.js'; +import type { IndexMetadata } from "../../../../models/server/index.js"; +import { isExpired } from "../../../../models/time/index.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections list [options] + +Description: + Lists all collections currently published in the KV Store. + +Options: + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + // ### List all indexes ### + const indexesPrefix = publishId + '_index_'; + const indexKeys = await getKVStoreKeys( + fastlyApiContext, + kvStoreName, + indexesPrefix, + ); + if (indexKeys == null) { + throw new Error(`❌ Can't query indexes in KV Store`); + } + + // ### Found collections ### + const foundCollections = indexKeys.map(x => x.slice(indexesPrefix.length)); + if (foundCollections.length === 0) { + console.log('No collections found.'); + } else { + console.log(`Found collections:`); + for (const collection of foundCollections) { + if (collection === defaultCollectionName) { + console.log(` ${collection} *DEFAULT*`); + } else { + console.log(` ${collection}`); + } + const indexKey = indexesPrefix + collection; + const kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext, + kvStoreName, + indexKey, + ); + if (!kvAssetsIndexResponse) { + throw new Error(`❌ Can't load KV Store entry ${indexesPrefix + collection}`); + } + let indexMetadata: IndexMetadata | undefined; + if (kvAssetsIndexResponse.metadata != null) { + try { + indexMetadata = JSON.parse(kvAssetsIndexResponse.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata == null) { + console.log(` No metadata found.`); + continue; + } + if (indexMetadata.publishedTime == null) { + console.log(' Published : unknown'); + } else { + console.log(` Published : ${new Date(indexMetadata.publishedTime * 1000)}`); + } + + if (indexMetadata.expirationTime == null) { + console.log(' Expiration : not set'); + } else { + console.log(` Expiration : ${new Date(indexMetadata.expirationTime * 1000)}`); + if (collection === defaultCollectionName) { + console.log(` ✅ Expiration time not enforced for default collection.`); + } else if (isExpired(indexMetadata.expirationTime)) { + console.log(` ⚠️ EXPIRED - Use 'clean --delete-expired-collections' to`); + console.log(` remove expired collections.`); + } + } + } + } + + console.log("✅ Completed.") +} diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts new file mode 100644 index 0000000..65f0b78 --- /dev/null +++ b/src/cli/commands/manage/collections/promote.ts @@ -0,0 +1,187 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type OptionDefinition } from 'command-line-args'; + +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { calcExpirationTime } from '../../../../models/time/index.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { + getKvStoreEntry, + kvStoreSubmitEntry, +} from '../../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { parseCommandLine } from '../../../util/args.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections promote \\ + --collection-name \\ + --to \\ + [options] + +Description: + Copies an existing collection (content + config) to a new collection name. + +Options: + --collection-name (Required) The name of the collection to promote + --to (Required) The name of the new (target) collection to create or overwrite + --expires-in Set new expiration relative to now (e.g., 7d, 1h). + --expires-at Set new expiration using an absolute ISO 8601 timestamp. + + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String }, + { name: 'to', type: String }, + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['collection-name']: collectionNameValue, + ['to']: toCollectionNameValue, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + if (collectionNameValue == null) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + if (toCollectionNameValue == null) { + console.error("❌ Required argument '--to' not specified."); + process.exitCode = 1; + return; + } + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + let expirationTime: number | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const sourceCollectionName = collectionNameValue; + console.log(`✔️ Collection to copy: ${sourceCollectionName}`); + + const targetCollectionName = toCollectionNameValue; + console.log(`✔️ Collection to promote to: ${targetCollectionName}`) + + if (expirationTime != null) { + console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } else { + console.log(`✔️ Not updating expiration timestamp.`); + } + if (targetCollectionName === defaultCollectionName && expirationTime != null) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + + const sourceCollectionIndexKey = `${publishId}_index_${sourceCollectionName}`; + const targetCollectionIndexKey = `${publishId}_index_${targetCollectionName}`; + + const sourceCollectionSettingsKey = `${publishId}_settings_${collectionNameValue}`; + const targetCollectionSettingsKey = `${publishId}_settings_${targetCollectionName}`; + + const [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ + getKvStoreEntry(fastlyApiContext, kvStoreName, sourceCollectionIndexKey), + getKvStoreEntry(fastlyApiContext, kvStoreName, sourceCollectionSettingsKey), + ]); + if (!indexEntryInfo) { + throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); + } + if (!settingsEntryInfo) { + throw new Error(`Error querying settings for '${collectionNameValue}' in KV Store`); + } + + let indexMetadata: IndexMetadata = {}; + if (indexEntryInfo.metadata != null) { + try { + indexMetadata = JSON.parse(indexEntryInfo.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata.publishedTime == null) { + indexMetadata.publishedTime = Math.floor(Date.now() / 1000); + } + if (expirationTime != null) { + indexMetadata.expirationTime = expirationTime; + } + + console.log(`Uploading to KV Store: '${targetCollectionName}'`); + + await Promise.all([ + kvStoreSubmitEntry(fastlyApiContext, kvStoreName, targetCollectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)), + kvStoreSubmitEntry(fastlyApiContext, kvStoreName, targetCollectionSettingsKey, settingsEntryInfo.response.body!, undefined), + ]); + + console.log("✅ Completed."); +} diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts new file mode 100644 index 0000000..2d51f9a --- /dev/null +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -0,0 +1,159 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import { type OptionDefinition } from 'command-line-args'; + +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { calcExpirationTime } from '../../../../models/time/index.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { + getKvStoreEntry, + kvStoreSubmitEntry, +} from '../../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { parseCommandLine } from '../../../util/args.js'; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish collections update-expiration [options] + +Description: + Updates the expiration time of an existing collection. + +Options: + --collection-name (Required) The name of the collection to modify + --expires-in Set new expiration relative to now (e.g., 7d, 1h) + --expires-at Set new expiration using an absolute ISO 8601 timestamp + + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} + +export async function action(actionArgs: string[]) { + + const optionDefinitions: OptionDefinition[] = [ + { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String, }, + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, + { name: 'fastly-api-token', type: String, }, + ]; + + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { + verbose, + ['collection-name']: collectionNameValue, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; + + if (collectionNameValue == null) { + console.error("❌ Required argument '--collection-name' not specified."); + process.exitCode = 1; + return; + } + + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // #### load config + let staticPublisherRc; + try { + staticPublisherRc = await loadStaticPublisherRcFile(); + } catch (err) { + console.error("❌ Can't load static-publish.rc.js"); + console.error("Run this from a compute-js-static-publish compute-js directory."); + if (err instanceof LoadConfigError) { + for (const error of err.errors) { + console.error(error); + } + } + process.exitCode = 1; + return; + } + + let expirationTime: number | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + + const publishId = staticPublisherRc.publishId; + console.log(` | Publish ID: ${publishId}`); + + const kvStoreName = staticPublisherRc.kvStoreName; + console.log(` | Using KV Store: ${kvStoreName}`); + + const defaultCollectionName = staticPublisherRc.defaultCollectionName; + console.log(` | Default Collection Name: ${defaultCollectionName}`); + + const collectionName = collectionNameValue; + console.log(`✔️ Collection to update: ${collectionName}`); + + if (expirationTime != null) { + console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } else { + console.log(`✔️ Not updating expiration timestamp.`); + } + if (collectionName === defaultCollectionName && expirationTime != null) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + + const collectionIndexKey = `${publishId}_index_${collectionName}`; + + const indexEntryInfo = await getKvStoreEntry(fastlyApiContext, kvStoreName, collectionIndexKey); + if (!indexEntryInfo) { + throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); + } + + let indexMetadata: IndexMetadata = {}; + if (indexEntryInfo.metadata != null) { + try { + indexMetadata = JSON.parse(indexEntryInfo.metadata) as IndexMetadata; + } catch { + } + } + if (indexMetadata.publishedTime == null) { + indexMetadata.publishedTime = Math.floor(Date.now() / 1000); + } + if (expirationTime != null) { + indexMetadata.expirationTime = expirationTime; + } + + console.log(`Uploading to KV Store: '${collectionName}'`); + + await kvStoreSubmitEntry(fastlyApiContext, kvStoreName, collectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)); + + console.log("✅ Completed."); +} diff --git a/src/cli/commands/index.ts b/src/cli/commands/manage/index.ts similarity index 76% rename from src/cli/commands/index.ts rename to src/cli/commands/manage/index.ts index 793e48f..05ece00 100644 --- a/src/cli/commands/index.ts +++ b/src/cli/commands/manage/index.ts @@ -3,51 +3,12 @@ * Licensed under the MIT license. See LICENSE file for details. */ -import fs from 'node:fs'; - -import { getCommandAndArgs } from '../util/args.js'; -import * as initApp from './init-app.js'; -import * as publishContent from './publish-content.js'; -import * as collections from './collections/index.js'; - -export async function action(actionArgs: string[]) { - - if (!fs.existsSync('./static-publish.rc.js')) { - - await initApp.action(actionArgs); - return; - - } - - const modes = { - 'publish-content': publishContent, - 'collections': collections, - }; - - const commandAndArgs = getCommandAndArgs( - actionArgs, - Object.keys(modes) as (keyof typeof modes)[], - ); - - if (commandAndArgs.needHelp) { - if (commandAndArgs.command != null) { - console.error(`Unknown command: ${commandAndArgs.command}`); - console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); - console.error(); - process.exitCode = 1; - } - - help(); - return; - } - - const { command, argv, } = commandAndArgs; - await modes[command].action(argv); - -} +import { getCommandAndArgs } from '../../util/args.js'; +import * as cleanCommand from './clean.js'; +import * as publishContentCommand from './publish-content.js'; +import * as collectionsCommands from './collections/index.js'; function help() { - console.log(`\ Usage: @@ -55,7 +16,8 @@ Usage: Description: Manage and publish static content to Fastly Compute using KV Store-backed collections. - If run outside a scaffolded project, this tool will automatically enter project initialization mode. + + Note: If run outside a scaffolded project, this tool will automatically enter scaffolding mode. Available Commands: publish-content Publish static files to the KV Store under a named collection @@ -81,5 +43,31 @@ Examples: npx @fastly/compute-js-static-publish collections list npx @fastly/compute-js-static-publish clean --dry-run `); +} + +export async function action(actionArgs: string[]) { + + const modes = { + 'clean': cleanCommand, + 'publish-content': publishContentCommand, + 'collections': collectionsCommands, + }; + + const commandAndArgs = getCommandAndArgs(actionArgs, modes); + + if (commandAndArgs.needHelp) { + if (commandAndArgs.command != null) { + console.error(`Unknown command: ${commandAndArgs.command}`); + console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } + + const { command, argv, } = commandAndArgs; + await modes[command].action(argv); } diff --git a/src/cli/commands/publish-content.ts b/src/cli/commands/manage/publish-content.ts similarity index 72% rename from src/cli/commands/publish-content.ts rename to src/cli/commands/manage/publish-content.ts index 78e9cf8..b219d70 100644 --- a/src/cli/commands/publish-content.ts +++ b/src/cli/commands/manage/publish-content.ts @@ -6,22 +6,30 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; -import commandLineArgs, { type OptionDefinition } from 'command-line-args'; - -import { type KVAssetEntryMap, type KVAssetVariantMetadata, isKVAssetVariantMetadata } from '../../models/assets/kvstore-assets.js'; -import { type ContentCompressionTypes } from '../../models/compression/index.js'; -import { type PublisherServerConfigNormalized } from '../../models/config/publisher-server-config.js'; -import { type ContentTypeDef } from '../../models/config/publish-content-config.js'; -import { type FastlyApiContext, FetchError, loadApiToken } from '../fastly-api/api-token.js'; -import { getKvStoreEntryInfo } from '../fastly-api/kv-store.js'; -import { mergeContentTypes, testFileContentType } from '../util/content-types.js'; -import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../util/config.js'; -import { applyDefaults } from '../util/data.js'; -import { calculateFileSizeAndHash, enumerateFiles, getFileSize, rootRelative } from '../util/files.js'; -import { applyKVStoreEntriesChunks, type KVStoreItemDesc, uploadFilesToKVStore } from '../util/kv-store-items.js'; -import { writeKVStoreEntriesForLocal } from '../util/kv-store-local-server.js'; -import { attemptWithRetries } from '../util/retryable.js'; -import { ensureVariantFileExists, type Variants } from '../util/variants.js'; +import { type OptionDefinition } from 'command-line-args'; + +import { type KVAssetEntryMap, type KVAssetVariantMetadata, isKVAssetVariantMetadata } from '../../../models/assets/kvstore-assets.js'; +import { type ContentCompressionTypes } from '../../../models/compression/index.js'; +import { type PublisherServerConfigNormalized } from '../../../models/config/publisher-server-config.js'; +import { type ContentTypeDef } from '../../../models/config/publish-content-config.js'; +import { type IndexMetadata } from '../../../models/server/index.js'; +import { calcExpirationTime } from '../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; +import { getKvStoreEntryInfo, kvStoreSubmitEntry } from '../../fastly-api/kv-store.js'; +import { parseCommandLine } from '../../util/args.js'; +import { mergeContentTypes, testFileContentType } from '../../util/content-types.js'; +import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../../util/config.js'; +import { applyDefaults } from '../../util/data.js'; +import { readServiceId } from '../../util/fastly-toml.js'; +import { calculateFileSizeAndHash, enumerateFiles, getFileSize, rootRelative } from '../../util/files.js'; +import { + applyKVStoreEntriesChunks, + doKvStoreItemsOperation, + type KVStoreItemDesc, +} from '../../util/kv-store-items.js'; +import { writeKVStoreEntriesForLocal } from '../../util/kv-store-local-server.js'; +import { isNodeError } from '../../util/node.js'; +import { ensureVariantFileExists, type Variants } from '../../util/variants.js'; // KV Store key format: // _index_.json @@ -31,16 +39,42 @@ import { ensureVariantFileExists, type Variants } from '../util/variants.js'; // split large files into 20MiB chunks const KV_STORE_CHUNK_SIZE = 1_024 * 1_024 * 20; -export async function action(argv: string[]) { +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish publish-content [options] + +Description: + Publishes static files from your root directory into the configured Fastly KV Store, + under a named collection (e.g., "live", "staging", "preview-123"). + Automatically skips uploading content that already exists in KV Store. + +Options: + --config Path to a publish-content.config.js file + (default: publish-content.config.js in the current directory) + --root-dir Override the root directory to publish from + --collection-name Publish under a specific collection name (defaults to value in static-publish.rc.js). + --expires-in Set expiration for the collection relative to now (e.g., 3d, 6h, 1w). + --expires-at Set expiration using an absolute ISO 8601 timestamp. + --local-only Write content to local KV Store for testing only. No remote uploads. + --no-local Skip local KV Store writes and upload only to the Fastly KV Store. + --kv-overwrite When Fastly KV Store is used, always overwrite existing items in the store. + + --fastly-api-token Fastly API token used for KV Store access. If not provided, + the tool will try: + 1. FASTLY_API_TOKEN environment variable + 2. fastly profile token (via CLI) + -h, --help Show help for this command. +`); +} + +export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean, }, - // Fastly API Token to use for this publishing. - { name: 'fastly-api-token', type: String, }, - - // Collection name to be used for this publishing. - { name: 'collection-name', type: String, }, + { name: 'config', type: String }, // The 'root' directory for the publishing. // All assets are expected to exist under this root. Required. @@ -48,27 +82,44 @@ export async function action(argv: string[]) { // then the value of 'public-dir' is used. { name: 'root-dir', type: String, }, - { name: 'force-upload', type: Boolean }, + // Collection name to be used for this publishing. + { name: 'collection-name', type: String, }, - { name: 'no-local', type: Boolean }, + { name: 'expires-in', type: String }, + { name: 'expires-at', type: String }, { name: 'local-only', type: Boolean }, + { name: 'no-local', type: Boolean }, + { name: 'kv-overwrite', type: Boolean }, - { name: 'config', type: String }, + // Fastly API Token to use for this publishing. + { name: 'fastly-api-token', type: String, }, ]; - const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } const { verbose, - ['fastly-api-token']: fastlyApiToken, - ['collection-name']: collectionNameValue, + ['config']: configFilePathValue, ['root-dir']: rootDir, - ['force-upload']: forceUpload, - ['no-local']: noLocalMode, + ['collection-name']: collectionNameValue, + ['expires-in']: expiresIn, + ['expires-at']: expiresAt, ['local-only']: localOnlyMode, - ['config']: configFilePathValue, - } = commandLineValues; + ['no-local']: noLocalMode, + ['kv-overwrite']: overwriteKvStoreItems, + ['fastly-api-token']: fastlyApiToken, + } = parsed.commandLineOptions; // no-local and local-only are mutually exclusive if (noLocalMode && localOnlyMode) { @@ -77,14 +128,48 @@ export async function action(argv: string[]) { return; } + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + // Create local files unless 'no-local' is set - const createLocalFiles = !noLocalMode; + let createLocalFiles = false; + if (noLocalMode) { + console.log(`ℹ️ 'No local mode' - skipping creation of files for local simulated KV Store`); + } else { + createLocalFiles = true; + } + // Use the KV Store unless 'local-only' is set - const useKvStore = !localOnlyMode; + let useKvStore = false; + if (serviceId == null) { + console.log(`ℹ️ 'service_id' not set in 'fastly.toml' - skipping creation of files for Fastly KV Store`); + } else if (localOnlyMode) { + console.log(`ℹ️ 'Local only mode' - skipping creation of files for Fastly KV Store`); + } else { + useKvStore = true; + } const segments: string[] = []; if (createLocalFiles) { - segments.push('for local simluated KV Store'); + segments.push('for local simulated KV Store'); } if (useKvStore) { segments.push('to the Fastly KV Store'); @@ -105,10 +190,6 @@ export async function action(argv: string[]) { console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); } - // compute-js-static-publisher cli is always run from the Compute application directory - // in other words, the directory that contains `fastly.toml`. - const computeAppDir = path.resolve(); - // #### load config let staticPublisherRc; try { @@ -150,16 +231,26 @@ export async function action(argv: string[]) { } } + let expirationTime: number | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + console.log(`✔️ Public directory '${rootRelative(publicDirRoot)}'.`); const publishId = staticPublisherRc.publishId; - console.log(`✔️ Publish ID: ${publishId}`); + console.log(` | Publish ID: ${publishId}`); const kvStoreName = staticPublisherRc.kvStoreName; - console.log(`✔️ Using KV Store: ${kvStoreName}`); + console.log(` | Using KV Store: ${kvStoreName}`); const defaultCollectionName = staticPublisherRc.defaultCollectionName; - console.log(`✔️ Default Collection Name: ${defaultCollectionName}`); + console.log(` | Default Collection Name: ${defaultCollectionName}`); // The Static Content Root Dir, which will hold loaders and content generated by this publishing. const staticPublisherWorkingDir = publishContentConfig.staticPublisherWorkingDir; @@ -189,6 +280,20 @@ export async function action(argv: string[]) { console.log(`✔️ (.well-known is exempt from exclusion.)`); } + // ### Collection name that we're currently publishing + // for example, live, staging + const collectionName = collectionNameValue ?? process.env.PUBLISHER_COLLECTION_NAME ?? defaultCollectionName; + console.log(`✔️ Collection Name: ${collectionName}`); + + if (expirationTime != null) { + console.log(`✔️ Publishing with expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } else { + console.log(`✔️ Publishing with no expiration timestamp.`); + } + if (collectionName === defaultCollectionName && expirationTime != null) { + console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); + } + // files to be included in the build/publish const files = enumerateFiles({ publicDirRoot, @@ -197,10 +302,6 @@ export async function action(argv: string[]) { includeWellKnown, }); - // ### Collection name that we're currently publishing - // for example, live, staging - const collectionName = collectionNameValue ?? process.env.PUBLISHER_COLLECTION_NAME ?? defaultCollectionName; - const stats = { kvStore: 0, }; @@ -303,9 +404,14 @@ export async function action(argv: string[]) { let kvStoreItemMetadata: KVAssetVariantMetadata | null = null; - if (useKvStore && !forceUpload) { - await attemptWithRetries( - async () => { + if (useKvStore && !overwriteKvStoreItems) { + const items = [{ + key: variantKey, + }]; + + await doKvStoreItemsOperation( + items, + async(_, variantKey) => { // fastlyApiContext is non-null if useKvStore is true const kvStoreEntryInfo = await getKvStoreEntryInfo(fastlyApiContext!, kvStoreName, variantKey); if (!kvStoreEntryInfo) { @@ -347,23 +453,7 @@ export async function action(argv: string[]) { }; } } - }, - { - onAttempt(attempt) { - if (attempt > 0) { - console.log(`Attempt ${attempt + 1} for: ${variantKey}`); - } - }, - onRetry(attempt, err, delay) { - let statusMessage = 'unknown'; - if (err instanceof FetchError) { - statusMessage = `HTTP ${err.status}`; - } else if (err instanceof TypeError) { - statusMessage = 'transport'; - } - console.log(`Attempt ${attempt + 1} for ${variantKey} gave retryable error (${statusMessage}), delaying ${delay} ms`); - }, - }, + } ); } @@ -464,20 +554,22 @@ export async function action(argv: string[]) { const indexFileName = `index_${collectionName}.json`; const indexFileKey = `${publishId}_index_${collectionName}`; - // Metadata can have build time, expiration date, build name - // const indexMetadata = {}; - const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); fs.writeFileSync(indexFilePath, JSON.stringify(kvAssetsIndex)); const indexFileSize = getFileSize(indexFilePath); + const indexMetadata: IndexMetadata = { + publishedTime: Math.floor(Date.now() / 1000), + expirationTime, + }; + kvStoreItemDescriptions.push({ write: true, size: indexFileSize, key: indexFileKey, filePath: indexFilePath, - // metadata: indexMetadata, + metadataJson: indexMetadata, }); console.log(`✅ Index has been saved.`) @@ -561,25 +653,32 @@ export async function action(argv: string[]) { key: settingsFileKey, filePath: settingsFilePath, }); - console.log(`✅ Settings have been saved.`) + console.log(`✅ Settings have been saved.`); - console.log(`🍪 Chunking large files...`) + console.log(`🍪 Chunking large files...`); await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); - console.log(`✅ Large files have been chunked.`) + console.log(`✅ Large files have been chunked.`); if (useKvStore) { - console.log(`📤 Uploading entries to KV Store.`) + console.log(`📤 Uploading entries to KV Store.`); // fastlyApiContext is non-null if useKvStore is true - await uploadFilesToKVStore(fastlyApiContext!, kvStoreName, kvStoreItemDescriptions); - console.log(`✅ Uploaded entries to KV Store.`) + await doKvStoreItemsOperation( + kvStoreItemDescriptions.filter(x => x.write), + async ({ filePath, metadataJson }, key) => { + const fileBytes = fs.readFileSync(filePath); + await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); + console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) + } + ); + console.log(`✅ Uploaded entries to KV Store.`); } if (createLocalFiles) { - console.log(`📝 Writing local server KV Store entries.`) + console.log(`📝 Writing local server KV Store entries.`); const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); - console.log(`✅ Wrote KV Store entries for local server.`) + console.log(`✅ Wrote KV Store entries for local server.`); } - console.log(`🎉 Completed.`) + console.log(`🎉 Completed.`); } diff --git a/src/cli/commands/init-app.ts b/src/cli/commands/scaffold/index.ts similarity index 88% rename from src/cli/commands/init-app.ts rename to src/cli/commands/scaffold/index.ts index 4239914..6877451 100644 --- a/src/cli/commands/init-app.ts +++ b/src/cli/commands/scaffold/index.ts @@ -11,10 +11,51 @@ import * as child_process from 'node:child_process'; import * as path from 'node:path'; import * as fs from 'node:fs'; -import commandLineArgs, { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; - -import { dotRelative, rootRelative } from '../util/files.js'; -import { findComputeJsStaticPublisherVersion, type PackageJson } from '../util/package.js'; +import { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; + +import { dotRelative, rootRelative } from '../../util/files.js'; +import { findComputeJsStaticPublisherVersion, type PackageJson } from '../../util/package.js'; +import { parseCommandLine } from "../../util/args.js"; + +function help() { + console.log(`\ + +Usage: + npx @fastly/compute-js-static-publish [options] + +Description: + Scaffold a new Compute app configured for static publishing. + + Note: If run inside a scaffolded project, this tool will automatically enter project + management mode. + +Options: + --kv-store-name (required) KV Store name for content storage + --root-dir (required) Path to static content (e.g., ./public) + -o, --output Output directory for Compute app (default: ./compute-js) + --publish-id Advanced. Prefix for KV keys (default: "default") + --static-publisher-working-dir Working directory for build artifacts + (default: ./compute-js/static-publisher) + +Compute Service Metadata: + --name App name (for fastly.toml) + --description App description (for fastly.toml) + --author App author (for fastly.toml) + --service-id (optional) Fastly service ID + +Server Config: + --public-dir Base dir for content (default: root-dir) + --static-dir , Directory served with long TTL (can repeat) + --spa SPA fallback file (e.g., /index.html) + --not-found-page 404 fallback file (e.g., /404.html) + --auto-index , Index filename (e.g., index.html,index.htm) + --auto-ext , Automatic extensions (e.g., .html,.htm) + +Other: + --verbose Enable verbose output + -h, --help Show help +`); +} export type InitAppOptions = { outputDir: string | undefined, @@ -54,7 +95,7 @@ const defaultOptions: InitAppOptions = { function buildOptions( packageJson: PackageJson | null, - commandLineValues: CommandLineOptions, + commandLineOptions: CommandLineOptions, ): InitAppOptions { // Applied in this order for proper overriding @@ -76,7 +117,7 @@ function buildOptions( { let outputDir: string | undefined; - const outputDirValue = commandLineValues['output']; + const outputDirValue = commandLineOptions['output']; if (outputDirValue == null || typeof outputDirValue === 'string') { outputDir = outputDirValue; } @@ -87,7 +128,7 @@ function buildOptions( { let rootDir: string | undefined; - const rootDirValue = commandLineValues['root-dir']; + const rootDirValue = commandLineOptions['root-dir']; if (rootDirValue == null || typeof rootDirValue === 'string') { rootDir = rootDirValue; } @@ -98,7 +139,7 @@ function buildOptions( { let publicDir: string | undefined; - const publicDirValue = commandLineValues['public-dir']; + const publicDirValue = commandLineOptions['public-dir']; if (publicDirValue == null || typeof publicDirValue === 'string') { publicDir = publicDirValue; } @@ -109,7 +150,7 @@ function buildOptions( { let staticDirs: string[] | undefined; - const staticDirsValue = commandLineValues['static-dir']; + const staticDirsValue = commandLineOptions['static-dir']; const asArray = Array.isArray(staticDirsValue) ? staticDirsValue : [ staticDirsValue ]; if (asArray.every((x: any) => typeof x === 'string')) { @@ -122,7 +163,7 @@ function buildOptions( { let staticPublisherWorkingDir: string | undefined; - const staticPublisherWorkingDirValue = commandLineValues['static-publisher-working-dir']; + const staticPublisherWorkingDirValue = commandLineOptions['static-publisher-working-dir']; if (staticPublisherWorkingDirValue == null || typeof staticPublisherWorkingDirValue === 'string') { staticPublisherWorkingDir = staticPublisherWorkingDirValue; } @@ -133,7 +174,7 @@ function buildOptions( { let spa: string | undefined; - const spaValue = commandLineValues['spa']; + const spaValue = commandLineOptions['spa']; if (spaValue === null) { // If 'spa' is provided with a null value, then the flag was provided // with no value. Assumed to be './index.html' relative to the public directory. @@ -149,7 +190,7 @@ function buildOptions( { let notFoundPage: string | undefined; - const notFoundPageValue = commandLineValues['not-found-page']; + const notFoundPageValue = commandLineOptions['not-found-page']; if (notFoundPageValue === null) { // If 'spa' is provided with a null value, then the flag was provided // with no value. Assumed to be './404.html' relative to the public directory. @@ -165,7 +206,7 @@ function buildOptions( { let autoIndex: string[] | undefined; - const autoIndexValue = commandLineValues['auto-index']; + const autoIndexValue = commandLineOptions['auto-index']; const asArray = Array.isArray(autoIndexValue) ? autoIndexValue : [ autoIndexValue ]; if (asArray.every((x: any) => typeof x === 'string')) { @@ -192,7 +233,7 @@ function buildOptions( { let autoExt: string[] = []; - const autoExtValue = commandLineValues['auto-ext']; + const autoExtValue = commandLineOptions['auto-ext']; const asArray = Array.isArray(autoExtValue) ? autoExtValue : [ autoExtValue ]; if (asArray.every((x: any) => typeof x === 'string')) { @@ -220,7 +261,7 @@ function buildOptions( { let name: string | undefined; - const nameValue = commandLineValues['name']; + const nameValue = commandLineOptions['name']; if (nameValue == null || typeof nameValue === 'string') { name = nameValue; } @@ -231,7 +272,7 @@ function buildOptions( { let author: string | undefined; - const authorValue = commandLineValues['author']; + const authorValue = commandLineOptions['author']; if (authorValue == null || typeof authorValue === 'string') { author = authorValue; } @@ -242,7 +283,7 @@ function buildOptions( { let description: string | undefined; - const descriptionValue = commandLineValues['description']; + const descriptionValue = commandLineOptions['description']; if (descriptionValue == null || typeof descriptionValue === 'string') { description = descriptionValue; } @@ -253,7 +294,7 @@ function buildOptions( { let serviceId: string | undefined; - const serviceIdValue = commandLineValues['service-id']; + const serviceIdValue = commandLineOptions['service-id']; if (serviceIdValue == null || typeof serviceIdValue === 'string') { serviceId = serviceIdValue; } @@ -264,7 +305,7 @@ function buildOptions( { let publishId: string | undefined; - const publishIdValue = commandLineValues['publish-id']; + const publishIdValue = commandLineOptions['publish-id']; if (publishIdValue == null || typeof publishIdValue === 'string') { publishId = publishIdValue; } @@ -275,7 +316,7 @@ function buildOptions( { let kvStoreName: string | undefined; - const kvStoreNameValue = commandLineValues['kv-store-name']; + const kvStoreNameValue = commandLineOptions['kv-store-name']; if (kvStoreNameValue == null || typeof kvStoreNameValue === 'string') { kvStoreName = kvStoreNameValue; } @@ -298,7 +339,7 @@ function processPublicDirToken(filepath: string, publicDir: string) { return path.resolve(publicDir, processedPath) } -export async function action(argv: string[]) { +export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, @@ -372,7 +413,17 @@ export async function action(argv: string[]) { { name: 'auto-ext', type: String, multiple: true, }, ]; - const commandLineValues = commandLineArgs(optionDefinitions, { argv }); + const parsed = parseCommandLine(actionArgs, optionDefinitions); + if (parsed.needHelp) { + if (parsed.error != null) { + console.error(String(parsed.error)); + console.error(); + process.exitCode = 1; + } + + help(); + return; + } let packageJson; try { @@ -385,7 +436,7 @@ export async function action(argv: string[]) { const options = buildOptions( packageJson, - commandLineValues, + parsed.commandLineOptions, ); const COMPUTE_JS_DIR = options.outputDir; diff --git a/src/cli/fastly-api/api-token.ts b/src/cli/fastly-api/api-token.ts index b5a6c11..ea22826 100644 --- a/src/cli/fastly-api/api-token.ts +++ b/src/cli/fastly-api/api-token.ts @@ -10,7 +10,7 @@ import { makeRetryable } from '../util/retryable.js'; export interface FastlyApiContext { apiToken: string, -}; +} export type LoadApiTokenResult = { apiToken: string, @@ -83,6 +83,15 @@ export class FetchError extends Error { status: number; } +function isReadableStream(data: unknown): data is ReadableStream { + return ( + typeof data === 'object' && + data !== null && + typeof (data as ReadableStream).getReader === 'function' && + typeof (data as ReadableStream).tee === 'function' + ); +} + export async function callFastlyApi( fastlyApiContext: FastlyApiContext, endpoint: string, @@ -105,7 +114,9 @@ export async function callFastlyApi( ...requestInit, headers, redirect: 'error', + ...(isReadableStream(requestInit?.body) ? { duplex: 'half' } as RequestInit : null), }); + let response; try { response = await fetch(request); diff --git a/src/cli/fastly-api/kv-store.ts b/src/cli/fastly-api/kv-store.ts index ca78c87..6238e13 100644 --- a/src/cli/fastly-api/kv-store.ts +++ b/src/cli/fastly-api/kv-store.ts @@ -177,7 +177,7 @@ export async function getKvStoreEntry( } const encoder = new TextEncoder(); -export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: Uint8Array | string, metadata: string | undefined) { +export async function kvStoreSubmitEntry(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: ReadableStream | Uint8Array | string, metadata: string | undefined) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { @@ -198,7 +198,7 @@ export async function kvStoreSubmitFile(fastlyApiContext: FastlyApiContext, kvSt } -export async function kvStoreDeleteFile(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { +export async function kvStoreDeleteEntry(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { diff --git a/src/cli/index.ts b/src/cli/index.ts index 5e090f0..6a6471d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -4,8 +4,19 @@ * Licensed under the MIT license. See LICENSE file for details. */ -import { action } from './commands/index.js'; +import fs from 'node:fs'; -console.log("🧑‍💻 Fastly Compute JavaScript Static Publisher"); +import * as scaffoldCommand from './commands/scaffold/index.js'; +import * as manageCommands from './commands/manage/index.js'; -await action(process.argv); +if (!fs.existsSync('./static-publish.rc.js')) { + + console.log("🧑‍💻Fastly Compute JavaScript Static Publisher (Scaffolding mode)"); + await scaffoldCommand.action(process.argv); + +} else { + + console.log("🧑‍💻Fastly Compute JavaScript Static Publisher (Management mode)"); + await manageCommands.action(process.argv); + +} diff --git a/src/cli/util/args.ts b/src/cli/util/args.ts index a096c4c..15cddea 100644 --- a/src/cli/util/args.ts +++ b/src/cli/util/args.ts @@ -3,17 +3,19 @@ * Licensed under the MIT license. See LICENSE file for details. */ -import commandLineArgs from 'command-line-args'; +import commandLineArgs, { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; export type ModeAction = (argv: string[]) => void | Promise; export type ActionModule = { action: ModeAction }; +export type ActionTable = Record; + +const helpOptionsDefs: OptionDefinition[] = [ + { name: 'help', alias: 'h', type: Boolean }, +]; export function isHelpArgs(argv: string[]) { - const helpDefinitions = [ - { name: 'help', type: Boolean, }, - ]; - const helpOptions = commandLineArgs(helpDefinitions, { argv, stopAtFirstUnknown: true }); + const helpOptions = commandLineArgs(helpOptionsDefs, { argv, stopAtFirstUnknown: true }); return !!helpOptions['help']; } @@ -35,22 +37,23 @@ export function findMainCommandNameAndArgs(argv: string[]): [string | null, stri } -export type CommandAndArgs = -| { +export type CommandAndArgs = { needHelp: false, command: T, argv: string[], -} -| { +}; + +export type CommandAndArgsHelp = { needHelp: true, command: string | null, } -; + +export type CommandAndArgsResult = CommandAndArgs | CommandAndArgsHelp; export function getCommandAndArgs( argv: string[], - modes: T[], -): CommandAndArgs { + actions: ActionTable, +): CommandAndArgsResult { if (isHelpArgs(argv)) { return { needHelp: true, @@ -62,7 +65,7 @@ export function getCommandAndArgs( const [ command, actionArgv ] = result; if (command != null) { - for (const modeName of modes) { + for (const modeName of Object.keys(actions)) { if (command === modeName) { return { needHelp: false, @@ -84,3 +87,46 @@ export function getCommandAndArgs( command, }; } + +export type CommandLineParsed = { + needHelp: false, + commandLineOptions: CommandLineOptions, +}; + +export type CommandLineHelp = { + needHelp: true, + error: Error | null, +}; + +export type CommandLineResult = CommandLineParsed | CommandLineHelp; + +export function parseCommandLine( + argv: string[], + optionDefinitions: OptionDefinition[], +): CommandLineResult { + let commandLineOptions; + try { + commandLineOptions = commandLineArgs([ + ...helpOptionsDefs, + ...optionDefinitions, + ], { argv }); + } catch(err) { + const error = err instanceof Error ? err : new Error(String(err)); + return { + needHelp: true, + error, + } + } + + if (!!commandLineOptions['help']) { + return { + needHelp: true, + error: null, + }; + } + + return { + needHelp: false, + commandLineOptions, + }; +} diff --git a/src/cli/util/fastly-toml.ts b/src/cli/util/fastly-toml.ts new file mode 100644 index 0000000..7bf8a11 --- /dev/null +++ b/src/cli/util/fastly-toml.ts @@ -0,0 +1,17 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +import fs from 'fs'; +import toml from 'toml'; + +export function readServiceId(filePath: string) { + const text = fs.readFileSync(filePath, 'utf8'); + const parsed = toml.parse(text); + const serviceId = parsed.service_id; + if (typeof serviceId === 'string') { + return serviceId; + } + return undefined; +} diff --git a/src/cli/util/kv-store-items.ts b/src/cli/util/kv-store-items.ts index f219b46..ad5f2a9 100644 --- a/src/cli/util/kv-store-items.ts +++ b/src/cli/util/kv-store-items.ts @@ -4,39 +4,30 @@ */ import fs from 'node:fs'; -import { directoryExists, getFileSize, rootRelative } from './files.js'; +import { directoryExists, getFileSize } from './files.js'; import { attemptWithRetries } from './retryable.js'; -import { type FastlyApiContext, FetchError } from '../fastly-api/api-token.js'; -import { kvStoreSubmitFile } from '../fastly-api/kv-store.js'; +import { FetchError } from '../fastly-api/api-token.js'; -export type KVStoreItemDesc = { - write: boolean, - size: number, - key: string, - filePath: string, - metadataJson?: Record, -}; +export async function doKvStoreItemsOperation( + objects: TObject[], + fn: (obj: TObject, key: string, index: number) => Promise, + maxConcurrent: number = 12, +) { -export async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, kvStoreName: string, kvStoreItemDescriptions: KVStoreItemDesc[]) { - - const maxConcurrent = 12; let index = 0; // Shared among workers async function worker() { - while (index < kvStoreItemDescriptions.length) { + while (index < objects.length) { const currentIndex = index; index = index + 1; - const { write, key, filePath, metadataJson } = kvStoreItemDescriptions[currentIndex]; - if (!write) { - continue; - } + + const object = objects[currentIndex]; + const { key } = object; try { await attemptWithRetries( async() => { - const fileBytes = fs.readFileSync(filePath); - await kvStoreSubmitFile(fastlyApiContext, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); - console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) + await fn(object, key, currentIndex); }, { onAttempt(attempt) { @@ -67,6 +58,14 @@ export async function uploadFilesToKVStore(fastlyApiContext: FastlyApiContext, k await Promise.all(workers); } +export type KVStoreItemDesc = { + write: boolean, + size: number, + key: string, + filePath: string, + metadataJson?: Record, +}; + export function shouldRecreateChunks(chunksDir: string, numChunks: number, item: KVStoreItemDesc, chunkSize: number) { console.log(` 📄 '${item.key}' - ${item.size} bytes → ${numChunks} chunks`) if (!directoryExists(chunksDir)) { diff --git a/src/cli/util/node.ts b/src/cli/util/node.ts new file mode 100644 index 0000000..2d279be --- /dev/null +++ b/src/cli/util/node.ts @@ -0,0 +1,11 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +export function isNodeError(err: unknown): err is NodeJS.ErrnoException { + if (err == null || typeof err !== 'object' || !('code' in err)) { + return false; + } + return true; +} diff --git a/src/models/server/index.ts b/src/models/server/index.ts new file mode 100644 index 0000000..db6add5 --- /dev/null +++ b/src/models/server/index.ts @@ -0,0 +1,4 @@ +export type IndexMetadata = { + publishedTime?: number, + expirationTime?: number, +}; diff --git a/src/models/time/index.ts b/src/models/time/index.ts new file mode 100644 index 0000000..8d54245 --- /dev/null +++ b/src/models/time/index.ts @@ -0,0 +1,66 @@ +/* + * Copyright Fastly, Inc. + * Licensed under the MIT license. See LICENSE file for details. + */ + +const DURATION_MS: Record = { + w: 604_800_000, + d: 86_400_000, + h: 3_600_000, + m: 60_000, + s: 1_000, + ms: 1, +}; + +// Parses something like "1d2h30m" or "5m15s" or "150ms" +function parseDuration(durationStr: string) { + const regex = /(\d+(?:\.\d+)?)(ms|[dhms])/g; + let totalMs = 0; + let matchedLength = 0; + + for (const match of durationStr.matchAll(regex)) { + const [res, valueStr, unit] = match; + const value = parseFloat(valueStr); + totalMs += value * DURATION_MS[unit]; + matchedLength += res.length; + } + + if (matchedLength !== durationStr.length) { + throw new Error(`Invalid duration format: '${durationStr}'`); + } + + return totalMs; +} + +export type CalcExpirationTimeArg = { + expiresIn?: string, + expiresAt?: string, +}; + +export function calcExpirationTime({ expiresIn, expiresAt }: CalcExpirationTimeArg) { + if (expiresIn && expiresAt) { + throw new Error('Only one of expiresIn or expiresAt may be provided'); + } + + if (expiresIn) { + const deltaMs = parseDuration(expiresIn); + console.log(` ⏳ Expiration duration '${expiresIn}' = ${deltaMs} ms`); + return Math.floor((Date.now() + deltaMs) / 1000); // return UNIX timestamp + } + + if (expiresAt) { + const time = Date.parse(expiresAt); + if (isNaN(time)) { + throw new Error(`Invalid expiresAt value: '${expiresAt}'`); + } + console.log(` ⏰️ Expiration timestamp '${expiresAt}' = '${new Date(time)}'`); + console.log(` → ${new Date(time).toISOString()}`); + return Math.floor(time / 1000); + } + + return undefined; +} + +export function isExpired(unixTime: number) { + return unixTime < Math.floor(Date.now() / 1000); +} diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts index 9161bc3..6072586 100644 --- a/src/server/publisher-server/index.ts +++ b/src/server/publisher-server/index.ts @@ -21,9 +21,11 @@ import { type KVAssetEntryMap, type KVAssetVariantMetadata, } from '../../models/assets/kvstore-assets.js'; +import { type IndexMetadata } from '../../models/server/index.js'; import { getKVStoreEntry } from '../util/kv-store.js'; import { checkIfModifiedSince, getIfModifiedSinceHeader } from './serve-preconditions/if-modified-since.js'; import { checkIfNoneMatch, getIfNoneMatchHeader } from './serve-preconditions/if-none-match.js'; +import { isExpired } from "../../models/time/index.js"; type KVAssetVariant = { kvStoreEntry: KVStoreEntry, @@ -92,8 +94,16 @@ export class PublisherServer { activeCollectionName: string; collectionNameHeader: string | null; + // Cached settings + settingsCached: PublisherServerConfigNormalized | null | undefined; + + // Cached index + kvAssetsIndex: KVAssetEntryMap | null | undefined; + setActiveCollectionName(collectionName: string) { this.activeCollectionName = collectionName; + this.settingsCached = undefined; + this.kvAssetsIndex = undefined; } setCollectionNameHeader(collectionHeader: string | null) { @@ -102,15 +112,20 @@ export class PublisherServer { // Server config is obtained from the KV Store, and cached for the duration of this object. async getServerConfig() { + if (this.settingsCached !== undefined) { + return this.settingsCached; + } const settingsFileKey = `${this.publishId}_settings_${this.activeCollectionName}`; const kvStore = new KVStore(this.kvStoreName); const settingsFile = await getKVStoreEntry(kvStore, settingsFileKey); if (settingsFile == null) { console.error(`Settings File not found at ${settingsFileKey}.`); console.error(`You may need to publish your application.`); - return null; + this.settingsCached = null; + } else { + this.settingsCached = (await settingsFile.json()) as PublisherServerConfigNormalized; } - return (await settingsFile.json()) as PublisherServerConfigNormalized; + return this.settingsCached; } async getStaticItems() { @@ -134,15 +149,39 @@ export class PublisherServer { } async getKvAssetsIndex() { + if (this.kvAssetsIndex !== undefined) { + return this.kvAssetsIndex; + } const indexFileKey = `${this.publishId}_index_${this.activeCollectionName}`; const kvStore = new KVStore(this.kvStoreName); const indexFile = await getKVStoreEntry(kvStore, indexFileKey); if (indexFile == null) { console.error(`Index File not found at ${indexFileKey}.`); console.error(`You may need to publish your application.`); + this.kvAssetsIndex = null; return null; } - return (await indexFile.json()) as KVAssetEntryMap; + + let collectionIsExpired = false; + if (this.activeCollectionName !== this.defaultCollectionName) { + const metadataText = indexFile.metadataText() + if (metadataText != null) { + try { + const metadata = JSON.parse(metadataText) as IndexMetadata; + collectionIsExpired = metadata.expirationTime != null && isExpired(metadata.expirationTime); + } catch { + } + } + } + + if (collectionIsExpired) { + console.error(`Requested collection expired at ${indexFileKey}.`); + this.kvAssetsIndex = null; + return null; + } + + this.kvAssetsIndex = (await indexFile.json()) as KVAssetEntryMap; + return this.kvAssetsIndex; } async getMatchingAsset(assetKey: string, applyAuto: boolean = false): Promise { From ca5db5e37ea85af79a67c54eb4abf6b51d2d2450 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Tue, 22 Apr 2025 15:16:02 +0900 Subject: [PATCH 08/20] v7.0.0-beta.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0f8b5ba..6e1b5d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.1", + "version": "7.0.0-beta.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.1", + "version": "7.0.0-beta.2", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index 8c03c9d..15daf73 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "7.0.0-beta.1", + "version": "7.0.0-beta.2", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From dae336016da6c518cdb7de69528aa2230fd48176 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 00:50:35 +0900 Subject: [PATCH 09/20] More v7 updates --- src/cli/commands/manage/clean.ts | 170 ++++++++++++--- src/cli/commands/manage/collections/delete.ts | 124 ++++++++--- src/cli/commands/manage/collections/index.ts | 22 +- src/cli/commands/manage/collections/list.ts | 123 ++++++++--- .../commands/manage/collections/promote.ts | 203 ++++++++++++++---- .../manage/collections/update-expiration.ts | 171 ++++++++++++--- src/cli/commands/manage/index.ts | 15 +- src/cli/commands/manage/publish-content.ts | 198 +++++++++-------- src/cli/commands/scaffold/index.ts | 21 +- src/cli/{fastly-api => util}/api-token.ts | 2 +- src/cli/util/config.ts | 40 ++-- src/cli/util/kv-store-items.ts | 2 +- src/cli/util/kv-store-local-server.ts | 116 +++++++++- src/cli/{fastly-api => util}/kv-store.ts | 12 +- src/models/config/publish-content-config.ts | 5 - src/models/config/static-publish-rc.ts | 7 +- src/models/time/index.ts | 11 +- 17 files changed, 923 insertions(+), 319 deletions(-) rename src/cli/{fastly-api => util}/api-token.ts (98%) rename src/cli/{fastly-api => util}/kv-store.ts (95%) diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts index 5a50bfb..ad31bb7 100644 --- a/src/cli/commands/manage/clean.ts +++ b/src/cli/commands/manage/clean.ts @@ -3,16 +3,25 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import path from 'node:path'; + import { type OptionDefinition } from 'command-line-args'; import { type KVAssetEntryMap } from '../../../models/assets/kvstore-assets.js'; import { type IndexMetadata } from '../../../models/server/index.js'; import { isExpired } from '../../../models/time/index.js'; import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; -import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteEntry } from '../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; +import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteEntry } from '../../util/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../util/api-token.js'; import { parseCommandLine } from '../../util/args.js'; +import { readServiceId } from '../../util/fastly-toml.js'; import { doKvStoreItemsOperation } from "../../util/kv-store-items.js"; +import { isNodeError } from '../../util/node.js'; +import { + getLocalKvStoreEntry, + getLocalKVStoreKeys, + localKvStoreDeleteEntry +} from "../../util/kv-store-local-server.js"; function help() { console.log(`\ @@ -26,13 +35,20 @@ Description: Options: --delete-expired-collections If set, expired collection index files will be deleted. + --dry-run Show what would be deleted without performing any deletions. - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. `); } @@ -40,8 +56,11 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + { name: 'delete-expired-collections', type: Boolean }, { name: 'dry-run', type: Boolean }, + + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -61,18 +80,54 @@ export async function action(actionArgs: string[]) { verbose, ['delete-expired-collections']: deleteExpiredCollections, ['dry-run']: dryRun, + local: localMode, ['fastly-api-token']: fastlyApiToken, } = parsed.commandLineOptions; - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); process.exitCode = 1; return; } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + console.log(`🧹 Cleaning KV Store entries...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } // #### load config let staticPublisherRc; @@ -99,16 +154,29 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + // ### KVStore Keys to delete const kvKeysToDelete = new Set(); // ### List all indexes ### const indexesPrefix = publishId + '_index_'; - const indexKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - indexesPrefix, - ); + let indexKeys; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } if (indexKeys == null) { throw new Error(`Can't query indexes in KV Store`); } @@ -131,12 +199,19 @@ export async function action(actionArgs: string[]) { async({collection}, indexKey) => { console.log(`Collection: ${collection}`); - // TODO deal with when the index file is > 20MB - const kvAssetsIndexResponse = await getKvStoreEntry( - fastlyApiContext, - kvStoreName, - indexKey, - ); + let kvAssetsIndexResponse; + if (localMode) { + kvAssetsIndexResponse = await getLocalKvStoreEntry( + storeFile, + indexKey, + ); + } else { + kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + indexKey, + ); + } if (!kvAssetsIndexResponse) { throw new Error(`Can't load KV Store entry ${indexesPrefix + collection}`); } @@ -185,11 +260,19 @@ export async function action(actionArgs: string[]) { // ### List all settings ### console.log('Enumerating settings:'); const settingsPrefix = publishId + '_settings_'; - const settingsKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - settingsPrefix, - ); + let settingsKeys; + if (localMode) { + settingsKeys = await getLocalKVStoreKeys( + storeFile, + settingsPrefix, + ); + } else { + settingsKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + settingsPrefix, + ); + } if (settingsKeys == null) { throw new Error(`Can't query settings in KV Store`); } @@ -209,11 +292,19 @@ export async function action(actionArgs: string[]) { // ### Obtain the assets in the KV Store and find the ones that are not in use console.log('Enumerating assets:'); const assetPrefix = publishId + '_files_'; - const assetKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - assetPrefix, - ); + let assetKeys; + if (localMode) { + assetKeys = await getLocalKVStoreKeys( + storeFile, + assetPrefix, + ); + } else { + assetKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + assetPrefix, + ); + } if (assetKeys == null) { throw new Error(`Can't query assets in KV Store`); } @@ -245,7 +336,18 @@ export async function action(actionArgs: string[]) { console.log(`[DRY RUN] Deleting item: ${key}`); } else { console.log(`Deleting item from KV Store: ${key}`); - await kvStoreDeleteEntry(fastlyApiContext, kvStoreName, key); + if (localMode) { + await localKvStoreDeleteEntry( + storeFile, + key, + ); + } else { + await kvStoreDeleteEntry( + fastlyApiContext!, + kvStoreName, + key, + ); + } } } ); diff --git a/src/cli/commands/manage/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts index b026b87..3ce3a15 100644 --- a/src/cli/commands/manage/collections/delete.ts +++ b/src/cli/commands/manage/collections/delete.ts @@ -3,33 +3,46 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import path from 'node:path'; + import { type OptionDefinition } from 'command-line-args'; -import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; -import { getKVStoreKeys, kvStoreDeleteEntry } from '../../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; import { parseCommandLine } from "../../../util/args.js"; -import { doKvStoreItemsOperation } from "../../../util/kv-store-items.js"; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; +import { getKVStoreKeys, kvStoreDeleteEntry } from '../../../util/kv-store.js'; +import { doKvStoreItemsOperation } from '../../../util/kv-store-items.js'; +import { getLocalKVStoreKeys, localKvStoreDeleteEntry } from '../../../util/kv-store-local-server.js'; +import { isNodeError } from '../../../util/node.js'; function help() { console.log(`\ Usage: - npx @fastly/compute-js-static-publish collections delete \\ - --collection-name \\ - [options] + npx @fastly/compute-js-static-publish collections delete --collection-name= [options] Description: - Deletes a collection index from the KV Store. The content files will remain but will no longer be referenced. + Deletes a collection index from the KV Store. The content files will remain but will no + longer be referenced. + + Use the 'npx @fastly/compute-js-static-publish clean' command afterward to remove content + files that are no longer referenced by any collection. -Options: - --collection-name (Required) The name of the collection to delete +Required: + --collection-name= The name of the collection to delete - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. `); } @@ -37,7 +50,10 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String }, + + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -55,25 +71,61 @@ export async function action(actionArgs: string[]) { const { verbose, - ['fastly-api-token']: fastlyApiToken, ['collection-name']: collectionNameValue, + ['fastly-api-token']: fastlyApiToken, + local: localMode, } = parsed.commandLineOptions; + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + if (collectionNameValue == null) { console.error("❌ Required argument '--collection-name' not specified."); process.exitCode = 1; return; } - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); process.exitCode = 1; return; } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + console.log(`🧹 Cleaning KV Store entries...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } // #### load config let staticPublisherRc; @@ -100,6 +152,11 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + const collectionName = collectionNameValue; if (collectionName === defaultCollectionName) { console.error(`❌ Cannot delete default collection: ${collectionName}`); @@ -109,15 +166,24 @@ export async function action(actionArgs: string[]) { console.log(`✔️ Collection to delete: ${collectionName}`); + // ### KVStore Keys to delete const kvKeysToDelete = new Set(); // ### List all indexes ### const indexesPrefix = publishId + '_index_'; - const indexKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - indexesPrefix, - ); + let indexKeys: string[] | null; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } if (indexKeys == null) { throw new Error(`Can't query indexes in KV Store`); } @@ -142,7 +208,11 @@ export async function action(actionArgs: string[]) { items, async(_, key) => { console.log(`Deleting key from KV Store: ${key}`); - await kvStoreDeleteEntry(fastlyApiContext, kvStoreName, key); + if (localMode) { + await localKvStoreDeleteEntry(storeFile, key); + } else { + await kvStoreDeleteEntry(fastlyApiContext!, kvStoreName, key); + } } ); diff --git a/src/cli/commands/manage/collections/index.ts b/src/cli/commands/manage/collections/index.ts index adb6325..c34f438 100644 --- a/src/cli/commands/manage/collections/index.ts +++ b/src/cli/commands/manage/collections/index.ts @@ -19,17 +19,23 @@ Description: Manage named collections within a Compute application built with @fastly/compute-js-static-publish. Available Subcommands: - list List all published collections - delete Delete a specific collection index - promote Copies an existing collection (content + config) to a new collection name - update-expiration Modify expiration time for an existing collection + list List all published collections + delete Delete a specific collection index + promote Copies an existing collection (content + config) + to a new collection name + update-expiration Modify expiration time for an existing collection Global Options: - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command or any subcommand + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. Examples: npx @fastly/compute-js-static-publish collections list diff --git a/src/cli/commands/manage/collections/list.ts b/src/cli/commands/manage/collections/list.ts index 6f9e1fe..0b3d65a 100644 --- a/src/cli/commands/manage/collections/list.ts +++ b/src/cli/commands/manage/collections/list.ts @@ -3,14 +3,19 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import path from 'node:path'; + import { type OptionDefinition } from 'command-line-args'; -import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; -import { getKvStoreEntry, getKVStoreKeys } from '../../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +import { type IndexMetadata } from '../../../../models/server/index.js'; +import { isExpired } from '../../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; import { parseCommandLine } from '../../../util/args.js'; -import type { IndexMetadata } from "../../../../models/server/index.js"; -import { isExpired } from "../../../../models/time/index.js"; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; +import { getKvStoreEntry, getKVStoreKeys } from '../../../util/kv-store.js'; +import { isNodeError } from '../../../util/node.js'; +import { getLocalKvStoreEntry, getLocalKVStoreKeys } from "../../../util/kv-store-local-server.js"; function help() { console.log(`\ @@ -21,12 +26,17 @@ Usage: Description: Lists all collections currently published in the KV Store. -Options: - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. `); } @@ -34,6 +44,8 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -51,18 +63,54 @@ export async function action(actionArgs: string[]) { const { verbose, + local: localMode, ['fastly-api-token']: fastlyApiToken, } = parsed.commandLineOptions; - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); process.exitCode = 1; return; } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + console.log(`📃 Listing collections...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } // #### load config let staticPublisherRc; @@ -89,13 +137,26 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + // ### List all indexes ### const indexesPrefix = publishId + '_index_'; - const indexKeys = await getKVStoreKeys( - fastlyApiContext, - kvStoreName, - indexesPrefix, - ); + let indexKeys: string[] | null; + if (localMode) { + indexKeys = await getLocalKVStoreKeys( + storeFile, + indexesPrefix, + ); + } else { + indexKeys = await getKVStoreKeys( + fastlyApiContext!, + kvStoreName, + indexesPrefix, + ); + } if (indexKeys == null) { throw new Error(`❌ Can't query indexes in KV Store`); } @@ -113,12 +174,20 @@ export async function action(actionArgs: string[]) { console.log(` ${collection}`); } const indexKey = indexesPrefix + collection; - const kvAssetsIndexResponse = await getKvStoreEntry( - fastlyApiContext, - kvStoreName, - indexKey, - ); - if (!kvAssetsIndexResponse) { + let kvAssetsIndexResponse; + if (localMode) { + kvAssetsIndexResponse = await getLocalKvStoreEntry( + storeFile, + indexKey, + ); + } else { + kvAssetsIndexResponse = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + indexKey, + ); + } + if (kvAssetsIndexResponse == null) { throw new Error(`❌ Can't load KV Store entry ${indexesPrefix + collection}`); } let indexMetadata: IndexMetadata | undefined; diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts index 65f0b78..120ec57 100644 --- a/src/cli/commands/manage/collections/promote.ts +++ b/src/cli/commands/manage/collections/promote.ts @@ -11,33 +11,56 @@ import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config import { getKvStoreEntry, kvStoreSubmitEntry, -} from '../../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +} from '../../../util/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; import { parseCommandLine } from '../../../util/args.js'; +import path from "node:path"; +import { readServiceId } from "../../../util/fastly-toml.js"; +import { isNodeError } from "../../../util/node.js"; +import { getLocalKvStoreEntry, localKvStoreSubmitEntry } from "../../../util/kv-store-local-server.js"; +import fs from "node:fs"; function help() { console.log(`\ Usage: - npx @fastly/compute-js-static-publish collections promote \\ - --collection-name \\ - --to \\ + npx @fastly/compute-js-static-publish collections promote + --collection-name= \\ + --to=name> \\ [options] Description: Copies an existing collection (content + config) to a new collection name. -Options: - --collection-name (Required) The name of the collection to promote - --to (Required) The name of the new (target) collection to create or overwrite - --expires-in Set new expiration relative to now (e.g., 7d, 1h). - --expires-at Set new expiration using an absolute ISO 8601 timestamp. +Required: + --collection-name= The name of the collection to promote + --to= The name of the new (target) collection to create or overwrite - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify no more than one. If not provided, then the + existing expiration rule of the collection being + promoted is used. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. `); } @@ -45,10 +68,15 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String }, { name: 'to', type: String }, + { name: 'expires-in', type: String }, { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, + + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -67,12 +95,18 @@ export async function action(actionArgs: string[]) { const { verbose, ['collection-name']: collectionNameValue, - ['to']: toCollectionNameValue, + to: toCollectionNameValue, ['expires-in']: expiresIn, ['expires-at']: expiresAt, + ['expires-never']: expiresNever, + local: localMode, ['fastly-api-token']: fastlyApiToken, } = parsed.commandLineOptions; + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + if (collectionNameValue == null) { console.error("❌ Required argument '--collection-name' not specified."); process.exitCode = 1; @@ -85,15 +119,56 @@ export async function action(actionArgs: string[]) { return; } - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + let expirationTime: number | null | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); process.exitCode = 1; return; } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.log(`📃 Promoting collection...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } // #### load config let staticPublisherRc; @@ -111,16 +186,6 @@ export async function action(actionArgs: string[]) { return; } - let expirationTime: number | undefined; - try { - expirationTime = calcExpirationTime({expiresIn, expiresAt}); - } catch(err: unknown) { - console.error(`❌ Cannot process expiration time`); - console.error(String(err)); - process.exitCode = 1; - return; - } - const publishId = staticPublisherRc.publishId; console.log(` | Publish ID: ${publishId}`); @@ -130,18 +195,25 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + const sourceCollectionName = collectionNameValue; console.log(`✔️ Collection to copy: ${sourceCollectionName}`); const targetCollectionName = toCollectionNameValue; console.log(`✔️ Collection to promote to: ${targetCollectionName}`) - if (expirationTime != null) { - console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); - } else { + if (expirationTime === undefined) { console.log(`✔️ Not updating expiration timestamp.`); + } else if (expirationTime === null) { + console.log(`✔️ Updating expiration timestamp: never`); + } else { + console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); } - if (targetCollectionName === defaultCollectionName && expirationTime != null) { + if (targetCollectionName === defaultCollectionName && expirationTime !== undefined) { console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); } @@ -151,10 +223,18 @@ export async function action(actionArgs: string[]) { const sourceCollectionSettingsKey = `${publishId}_settings_${collectionNameValue}`; const targetCollectionSettingsKey = `${publishId}_settings_${targetCollectionName}`; - const [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ - getKvStoreEntry(fastlyApiContext, kvStoreName, sourceCollectionIndexKey), - getKvStoreEntry(fastlyApiContext, kvStoreName, sourceCollectionSettingsKey), - ]); + let indexEntryInfo, settingsEntryInfo; + if (localMode) { + [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ + getLocalKvStoreEntry(storeFile, sourceCollectionIndexKey), + getLocalKvStoreEntry(storeFile, sourceCollectionSettingsKey), + ]); + } else { + [ indexEntryInfo, settingsEntryInfo ] = await Promise.all([ + getKvStoreEntry(fastlyApiContext!, kvStoreName, sourceCollectionIndexKey), + getKvStoreEntry(fastlyApiContext!, kvStoreName, sourceCollectionSettingsKey), + ]); + } if (!indexEntryInfo) { throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); } @@ -172,16 +252,49 @@ export async function action(actionArgs: string[]) { if (indexMetadata.publishedTime == null) { indexMetadata.publishedTime = Math.floor(Date.now() / 1000); } - if (expirationTime != null) { - indexMetadata.expirationTime = expirationTime; + if (expirationTime !== undefined) { + if (expirationTime === null) { + delete indexMetadata.expirationTime; + } else { + indexMetadata.expirationTime = expirationTime; + } } console.log(`Uploading to KV Store: '${targetCollectionName}'`); - await Promise.all([ - kvStoreSubmitEntry(fastlyApiContext, kvStoreName, targetCollectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)), - kvStoreSubmitEntry(fastlyApiContext, kvStoreName, targetCollectionSettingsKey, settingsEntryInfo.response.body!, undefined), - ]); + if (localMode) { + + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + const indexFileName = `index_${collectionNameValue}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + const indexBody = await indexEntryInfo.response.arrayBuffer(); + fs.writeFileSync(indexFilePath, Buffer.from(indexBody)); + await localKvStoreSubmitEntry( + storeFile, + targetCollectionIndexKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + + const settingsFileName = `settings_${collectionNameValue}.json`; + const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); + const settingsBody = await settingsEntryInfo.response.arrayBuffer(); + fs.writeFileSync(settingsFilePath, Buffer.from(settingsBody)); + await localKvStoreSubmitEntry( + storeFile, + targetCollectionSettingsKey, + path.relative(computeAppDir, settingsFilePath), + undefined, + ); + + } else { + await Promise.all([ + kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, targetCollectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)), + kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, targetCollectionSettingsKey, settingsEntryInfo.response.body!, undefined), + ]); + } console.log("✅ Completed."); } diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts index 2d51f9a..0009bbf 100644 --- a/src/cli/commands/manage/collections/update-expiration.ts +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -11,29 +11,52 @@ import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config import { getKvStoreEntry, kvStoreSubmitEntry, -} from '../../../fastly-api/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../../fastly-api/api-token.js'; +} from '../../../util/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; import { parseCommandLine } from '../../../util/args.js'; +import { readServiceId } from "../../../util/fastly-toml.js"; +import { isNodeError } from "../../../util/node.js"; +import path from "node:path"; +import fs from "node:fs"; +import { getLocalKvStoreEntry, localKvStoreSubmitEntry } from "../../../util/kv-store-local-server.js"; function help() { console.log(`\ Usage: - npx @fastly/compute-js-static-publish collections update-expiration [options] + npx @fastly/compute-js-static-publish collections update-expiration \\ + --collection-name= \\ + [options] Description: Updates the expiration time of an existing collection. -Options: - --collection-name (Required) The name of the collection to modify - --expires-in Set new expiration relative to now (e.g., 7d, 1h) - --expires-at Set new expiration using an absolute ISO 8601 timestamp +Required: + --collection-name= The name of the collection to modify - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify exactly one. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. `); } @@ -41,9 +64,14 @@ export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ { name: 'verbose', type: Boolean }, + { name: 'collection-name', type: String, }, + { name: 'expires-in', type: String }, { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, + + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -64,24 +92,76 @@ export async function action(actionArgs: string[]) { ['collection-name']: collectionNameValue, ['expires-in']: expiresIn, ['expires-at']: expiresAt, + ['expires-never']: expiresNever, + local: localMode, ['fastly-api-token']: fastlyApiToken, } = parsed.commandLineOptions; + // compute-js-static-publisher cli is always run from the Compute application directory + // in other words, the directory that contains `fastly.toml`. + const computeAppDir = path.resolve(); + if (collectionNameValue == null) { console.error("❌ Required argument '--collection-name' not specified."); process.exitCode = 1; return; } - const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); - if (apiTokenResult == null) { - console.error("❌ Fastly API Token not provided."); - console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + let expirationTime: number | null | undefined; + try { + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); + } catch(err: unknown) { + console.error(`❌ Cannot process expiration time`); + console.error(String(err)); + process.exitCode = 1; + return; + } + if (expirationTime === undefined) { + console.error("❌ Exactly one of '--expires-in', '--expires-at', or '--expires-never' is required."); + process.exitCode = 1; + return; + } + + // Check to see if we have a service ID listed in `fastly.toml`. + // If we do NOT, then we do not use the KV Store. + let serviceId: string | undefined; + try { + serviceId = readServiceId(path.resolve(computeAppDir, './fastly.toml')); + } catch(err: unknown) { + if (isNodeError(err) && err.code === 'ENOENT') { + console.warn(`❌ ERROR: can't find 'fastly.toml'.`); + process.exitCode = 1; + return; + } + + console.warn(`❌ ERROR: can't read or parse 'fastly.toml'.`); process.exitCode = 1; return; } - const fastlyApiContext = { apiToken: apiTokenResult.apiToken } satisfies FastlyApiContext; - console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + + console.log(`📃 Promoting collection...`); + + // Verify targets + let fastlyApiContext: FastlyApiContext | undefined = undefined; + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } + const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); + if (apiTokenResult == null) { + console.error("❌ Fastly API Token not provided."); + console.error("Set the FASTLY_API_TOKEN environment variable to an API token that has write access to the KV Store."); + process.exitCode = 1; + return; + } + fastlyApiContext = { apiToken: apiTokenResult.apiToken }; + console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); + } // #### load config let staticPublisherRc; @@ -99,16 +179,6 @@ export async function action(actionArgs: string[]) { return; } - let expirationTime: number | undefined; - try { - expirationTime = calcExpirationTime({expiresIn, expiresAt}); - } catch(err: unknown) { - console.error(`❌ Cannot process expiration time`); - console.error(String(err)); - process.exitCode = 1; - return; - } - const publishId = staticPublisherRc.publishId; console.log(` | Publish ID: ${publishId}`); @@ -118,13 +188,18 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); + const collectionName = collectionNameValue; console.log(`✔️ Collection to update: ${collectionName}`); - if (expirationTime != null) { + if (expirationTime !== null) { console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); } else { - console.log(`✔️ Not updating expiration timestamp.`); + console.log(`✔️ Updating expiration timestamp: never`); } if (collectionName === defaultCollectionName && expirationTime != null) { console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); @@ -132,7 +207,19 @@ export async function action(actionArgs: string[]) { const collectionIndexKey = `${publishId}_index_${collectionName}`; - const indexEntryInfo = await getKvStoreEntry(fastlyApiContext, kvStoreName, collectionIndexKey); + let indexEntryInfo; + if (localMode) { + indexEntryInfo = await getLocalKvStoreEntry( + storeFile, + collectionIndexKey, + ); + } else { + indexEntryInfo = await getKvStoreEntry( + fastlyApiContext!, + kvStoreName, + collectionIndexKey, + ); + } if (!indexEntryInfo) { throw new Error(`Error querying index for '${collectionNameValue}' in KV Store`); } @@ -147,13 +234,33 @@ export async function action(actionArgs: string[]) { if (indexMetadata.publishedTime == null) { indexMetadata.publishedTime = Math.floor(Date.now() / 1000); } - if (expirationTime != null) { + if (expirationTime === null) { + delete indexMetadata.expirationTime; + } else { indexMetadata.expirationTime = expirationTime; } console.log(`Uploading to KV Store: '${collectionName}'`); - await kvStoreSubmitEntry(fastlyApiContext, kvStoreName, collectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)); + if (localMode) { + + const staticPublisherKvStoreContent = `${staticPublisherWorkingDir}/kv-store-content`; + fs.mkdirSync(staticPublisherKvStoreContent, { recursive: true }); + + const indexFileName = `index_${collectionNameValue}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + const indexBody = await indexEntryInfo.response.arrayBuffer(); + fs.writeFileSync(indexFilePath, Buffer.from(indexBody)); + await localKvStoreSubmitEntry( + storeFile, + collectionIndexKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + + } else { + await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, collectionIndexKey, indexEntryInfo.response.body!, JSON.stringify(indexMetadata)); + } console.log("✅ Completed."); } diff --git a/src/cli/commands/manage/index.ts b/src/cli/commands/manage/index.ts index 05ece00..19ff824 100644 --- a/src/cli/commands/manage/index.ts +++ b/src/cli/commands/manage/index.ts @@ -28,18 +28,23 @@ Available Commands: collections update-expiration Modify expiration time for an existing collection Global Options: - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command or any subcommand + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. Automatic Project Initialization: If run in a directory that does not contain a \`static-publish.rc.js\` file, this tool will scaffold a new Compute application for you, including Fastly configuration, default routes, and publishing setup. Examples: - npx @fastly/compute-js-static-publish publish-content --collection-name live + npx @fastly/compute-js-static-publish publish-content --collection-name=live npx @fastly/compute-js-static-publish collections list npx @fastly/compute-js-static-publish clean --dry-run `); diff --git a/src/cli/commands/manage/publish-content.ts b/src/cli/commands/manage/publish-content.ts index b219d70..214ffed 100644 --- a/src/cli/commands/manage/publish-content.ts +++ b/src/cli/commands/manage/publish-content.ts @@ -14,8 +14,8 @@ import { type PublisherServerConfigNormalized } from '../../../models/config/pub import { type ContentTypeDef } from '../../../models/config/publish-content-config.js'; import { type IndexMetadata } from '../../../models/server/index.js'; import { calcExpirationTime } from '../../../models/time/index.js'; -import { type FastlyApiContext, loadApiToken } from '../../fastly-api/api-token.js'; -import { getKvStoreEntryInfo, kvStoreSubmitEntry } from '../../fastly-api/kv-store.js'; +import { type FastlyApiContext, loadApiToken } from '../../util/api-token.js'; +import { getKvStoreEntryInfo, kvStoreSubmitEntry } from '../../util/kv-store.js'; import { parseCommandLine } from '../../util/args.js'; import { mergeContentTypes, testFileContentType } from '../../util/content-types.js'; import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../../util/config.js'; @@ -43,56 +43,78 @@ function help() { console.log(`\ Usage: - npx @fastly/compute-js-static-publish publish-content [options] + npx @fastly/compute-js-static-publish publish-content [--collection-name=] [options] Description: - Publishes static files from your root directory into the configured Fastly KV Store, - under a named collection (e.g., "live", "staging", "preview-123"). - Automatically skips uploading content that already exists in KV Store. - -Options: - --config Path to a publish-content.config.js file - (default: publish-content.config.js in the current directory) - --root-dir Override the root directory to publish from - --collection-name Publish under a specific collection name (defaults to value in static-publish.rc.js). - --expires-in Set expiration for the collection relative to now (e.g., 3d, 6h, 1w). - --expires-at Set expiration using an absolute ISO 8601 timestamp. - --local-only Write content to local KV Store for testing only. No remote uploads. - --no-local Skip local KV Store writes and upload only to the Fastly KV Store. - --kv-overwrite When Fastly KV Store is used, always overwrite existing items in the store. - - --fastly-api-token Fastly API token used for KV Store access. If not provided, - the tool will try: + Publishes static files from your local root directory into a named collection, + either in the Fastly KV Store (default) or to a local dev directory (--local). + Files that already exist with the same hash are skipped automatically. + + After this process is complete, the PublisherServer object in the Compute application + will see the updated index of files and updated server settings from the + publish-content.config.js file. + +Optional: + --collection-name= Name of the collection to publish into. + Default: value from static-publisher.rc.js (defaultCollectionName) + + --config= Path to a publish-content.config.js file. + Default: ./publish-content.config.js + + --root-dir= Directory to publish from. Overrides the config file setting. + Default: rootDir from publish-content.config.js + + --kv-overwrite Cannot be used with --local. + When using Fastly KV Store, always overwrite + existing entries, even if unchanged. + +Expiration: + --expires-in= Expiration duration from now. + Examples: 3d, 12h, 15m, 1w + + --expires-at= Absolute expiration in ISO 8601 format. + Example: 2025-05-01T00:00:00Z + + --expires-never Prevent this collection from expiring. + + ⚠ These three options are mutually exclusive. + Specify only one. + +Global Options: + --local Instead of working with the Fastly KV Store, operate on + local files that will be used to simulate the KV Store + with the local development environment. + + --fastly-api-token= Fastly API token for KV Store access. + If not set, the tool will check: 1. FASTLY_API_TOKEN environment variable - 2. fastly profile token (via CLI) - -h, --help Show help for this command. + 2. Logged-in Fastly CLI profile + + -h, --help Show this help message and exit. + +Examples: + npx @fastly/compute-js-static-publish publish-content --collection-name=preview-456 + npx @fastly/compute-js-static-publish publish-content --expires-in=7d --kv-overwrite + npx @fastly/compute-js-static-publish publish-content --expires-never --local + `); } export async function action(actionArgs: string[]) { const optionDefinitions: OptionDefinition[] = [ - { name: 'verbose', type: Boolean, }, + { name: 'verbose', type: Boolean }, { name: 'config', type: String }, - - // The 'root' directory for the publishing. - // All assets are expected to exist under this root. Required. - // For backwards compatibility, if this value is not provided, - // then the value of 'public-dir' is used. - { name: 'root-dir', type: String, }, - - // Collection name to be used for this publishing. { name: 'collection-name', type: String, }, + { name: 'root-dir', type: String, }, + { name: 'kv-overwrite', type: Boolean }, { name: 'expires-in', type: String }, { name: 'expires-at', type: String }, + { name: 'expires-never', type: Boolean }, - { name: 'local-only', type: Boolean }, - { name: 'no-local', type: Boolean }, - { name: 'kv-overwrite', type: Boolean }, - - // Fastly API Token to use for this publishing. + { name: 'local', type: Boolean }, { name: 'fastly-api-token', type: String, }, ]; @@ -110,24 +132,17 @@ export async function action(actionArgs: string[]) { const { verbose, - ['config']: configFilePathValue, - ['root-dir']: rootDir, + config: configFilePathValue, ['collection-name']: collectionNameValue, + ['root-dir']: rootDir, + ['kv-overwrite']: overwriteKvStoreItems, ['expires-in']: expiresIn, ['expires-at']: expiresAt, - ['local-only']: localOnlyMode, - ['no-local']: noLocalMode, - ['kv-overwrite']: overwriteKvStoreItems, + ['expires-never']: expiresNever, + local: localMode, ['fastly-api-token']: fastlyApiToken, } = parsed.commandLineOptions; - // no-local and local-only are mutually exclusive - if (noLocalMode && localOnlyMode) { - console.error("❌ '--no-local' and '--local-only' are mutually exclusive."); - process.exitCode = 1; - return; - } - // compute-js-static-publisher cli is always run from the Compute application directory // in other words, the directory that contains `fastly.toml`. const computeAppDir = path.resolve(); @@ -149,36 +164,18 @@ export async function action(actionArgs: string[]) { return; } - // Create local files unless 'no-local' is set - let createLocalFiles = false; - if (noLocalMode) { - console.log(`ℹ️ 'No local mode' - skipping creation of files for local simulated KV Store`); - } else { - createLocalFiles = true; - } - - // Use the KV Store unless 'local-only' is set - let useKvStore = false; - if (serviceId == null) { - console.log(`ℹ️ 'service_id' not set in 'fastly.toml' - skipping creation of files for Fastly KV Store`); - } else if (localOnlyMode) { - console.log(`ℹ️ 'Local only mode' - skipping creation of files for Fastly KV Store`); - } else { - useKvStore = true; - } - - const segments: string[] = []; - if (createLocalFiles) { - segments.push('for local simulated KV Store'); - } - if (useKvStore) { - segments.push('to the Fastly KV Store'); - } - - console.log(`🚀 Publishing content ${segments.join(' and ')}...`); + console.log(`🚀 Publishing content...`); + // Verify targets let fastlyApiContext: FastlyApiContext | undefined = undefined; - if (useKvStore) { + if (localMode) { + console.log(` Working on local simulated KV Store...`); + } else { + if (serviceId === null) { + console.log(`❌️ 'service_id' not set in 'fastly.toml' - Deploy your Compute app to Fastly before publishing.`); + process.exitCode = 1; + return; + } const apiTokenResult = loadApiToken({ commandLine: fastlyApiToken }); if (apiTokenResult == null) { console.error("❌ Fastly API Token not provided."); @@ -188,6 +185,7 @@ export async function action(actionArgs: string[]) { } fastlyApiContext = { apiToken: apiTokenResult.apiToken }; console.log(`✔️ Fastly API Token: ${fastlyApiContext.apiToken.slice(0, 4)}${'*'.repeat(fastlyApiContext.apiToken.length-4)} from '${apiTokenResult.source}'`); + console.log(` Working on the Fastly KV Store...`); } // #### load config @@ -231,9 +229,9 @@ export async function action(actionArgs: string[]) { } } - let expirationTime: number | undefined; + let expirationTime: number | null | undefined; try { - expirationTime = calcExpirationTime({expiresIn, expiresAt}); + expirationTime = calcExpirationTime({expiresIn, expiresAt, expiresNever}); } catch(err: unknown) { console.error(`❌ Cannot process expiration time`); console.error(String(err)); @@ -252,8 +250,10 @@ export async function action(actionArgs: string[]) { const defaultCollectionName = staticPublisherRc.defaultCollectionName; console.log(` | Default Collection Name: ${defaultCollectionName}`); - // The Static Content Root Dir, which will hold loaders and content generated by this publishing. - const staticPublisherWorkingDir = publishContentConfig.staticPublisherWorkingDir; + const staticPublisherWorkingDir = staticPublisherRc.staticPublisherWorkingDir; + console.log(` | Static publisher working directory: ${staticPublisherWorkingDir}`); + + const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); // Load content types const contentTypes: ContentTypeDef[] = mergeContentTypes(publishContentConfig.contentTypes); @@ -290,7 +290,13 @@ export async function action(actionArgs: string[]) { } else { console.log(`✔️ Publishing with no expiration timestamp.`); } - if (collectionName === defaultCollectionName && expirationTime != null) { + if (expirationTime == null) { + // includes null and undefined + console.log(`✔️ Publishing with no expiration timestamp.`); + } else { + console.log(`✔️ Updating expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); + } + if (collectionName === defaultCollectionName && expirationTime !== undefined) { console.log(` ⚠️ NOTE: Expiration time not enforced for default collection.`); } @@ -404,7 +410,7 @@ export async function action(actionArgs: string[]) { let kvStoreItemMetadata: KVAssetVariantMetadata | null = null; - if (useKvStore && !overwriteKvStoreItems) { + if (!localMode && !overwriteKvStoreItems) { const items = [{ key: variantKey, }]; @@ -477,7 +483,7 @@ export async function action(actionArgs: string[]) { variant, file, ); - if (useKvStore) { + if (!localMode) { console.log(` ・ Flagging asset for upload to KV Store with key "${variantKey}".`); } @@ -517,8 +523,8 @@ export async function action(actionArgs: string[]) { }, }); - if (createLocalFiles) { - // Although we already know the size and hash of the file, the local server + if (localMode) { + // Although we already know the size and hash of the variant, the local server // needs a copy of the file so we create it if it doesn't exist. // This may happen for example if files were uploaded to the KV Store in a previous // publishing, but local static content files have been removed since. @@ -549,6 +555,12 @@ export async function action(actionArgs: string[]) { } console.log(`✅ Scan complete.`) + // TODO: fix this bug: + // Technically it's a bug to WRITE the index and settings json files to the + // staticPublisherKvStoreContent dir when target === 'fastly'. + // For these files we should not be creating the files using + // kvStoreItemDescriptions, but rather creating them directly. + // #### INDEX FILE console.log(`🗂️ Creating Index...`); const indexFileName = `index_${collectionName}.json`; @@ -561,7 +573,7 @@ export async function action(actionArgs: string[]) { const indexMetadata: IndexMetadata = { publishedTime: Math.floor(Date.now() / 1000), - expirationTime, + expirationTime: expirationTime ?? undefined, }; kvStoreItemDescriptions.push({ @@ -659,12 +671,16 @@ export async function action(actionArgs: string[]) { await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); console.log(`✅ Large files have been chunked.`); - if (useKvStore) { + if (localMode) { + console.log(`📝 Writing local server KV Store entries.`); + writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); + console.log(`✅ Wrote KV Store entries for local server.`); + } else { console.log(`📤 Uploading entries to KV Store.`); // fastlyApiContext is non-null if useKvStore is true await doKvStoreItemsOperation( kvStoreItemDescriptions.filter(x => x.write), - async ({ filePath, metadataJson }, key) => { + async ({filePath, metadataJson}, key) => { const fileBytes = fs.readFileSync(filePath); await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) @@ -672,12 +688,6 @@ export async function action(actionArgs: string[]) { ); console.log(`✅ Uploaded entries to KV Store.`); } - if (createLocalFiles) { - console.log(`📝 Writing local server KV Store entries.`); - const storeFile = path.resolve(staticPublisherWorkingDir, `./kvstore.json`); - writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); - console.log(`✅ Wrote KV Store entries for local server.`); - } console.log(`🎉 Completed.`); diff --git a/src/cli/commands/scaffold/index.ts b/src/cli/commands/scaffold/index.ts index 6877451..3bdbb8e 100644 --- a/src/cli/commands/scaffold/index.ts +++ b/src/cli/commands/scaffold/index.ts @@ -33,8 +33,8 @@ Options: --kv-store-name (required) KV Store name for content storage --root-dir (required) Path to static content (e.g., ./public) -o, --output Output directory for Compute app (default: ./compute-js) + --static-publisher-working-dir Working directory for build artifacts (default: /static-publisher) --publish-id Advanced. Prefix for KV keys (default: "default") - --static-publisher-working-dir Working directory for build artifacts (default: ./compute-js/static-publisher) Compute Service Metadata: @@ -647,11 +647,11 @@ export async function action(actionArgs: string[]) { }, private: true, scripts: { - prestart: "npx @fastly/compute-js-static-publish publish-content --local-only", - start: "fastly compute serve", - "publish-service": "fastly compute publish", - "publish-content": 'npx @fastly/compute-js-static-publish publish-content', - build: 'js-compute-runtime ./src/index.js ./bin/main.wasm' + 'dev:publish': 'npx @fastly/compute-js-static-publish publish-content --local', + 'dev:start': 'fastly compute serve', + 'fastly:deploy': 'fastly compute publish', + 'fastly:publish': 'npx @fastly/compute-js-static-publish publish-content', + 'build': 'js-compute-runtime ./src/index.js ./bin/main.wasm' }, }, undefined, 2); @@ -690,6 +690,7 @@ const rc = { kvStoreName: ${JSON.stringify(kvStoreName)}, publishId: ${JSON.stringify(publishId)}, defaultCollectionName: ${JSON.stringify(defaultCollectionName)}, + staticPublisherWorkingDir: ${JSON.stringify(dotRelative(computeJsDir, staticPublisherWorkingDir))}, }; export default rc; @@ -747,7 +748,6 @@ export default rc; /** @type {import('@fastly/compute-js-static-publish').PublishContentConfig} */ const config = { rootDir: ${JSON.stringify(dotRelative(computeJsDir, rootDir))}, - staticPublisherWorkingDir: ${JSON.stringify(dotRelative(computeJsDir, staticPublisherWorkingDir))}, // excludeDirs: [ './node_modules' ], // excludeDotFiles: true, // includeWellKnown: true, @@ -818,12 +818,13 @@ async function handleRequest(event) { console.log('To run your Compute application locally:'); console.log(''); console.log(' cd ' + COMPUTE_JS_DIR); - console.log(' npm run start'); + console.log(' npm run dev:publish'); + console.log(' npm run dev:start'); console.log(''); console.log('To build and deploy to your Compute service:'); console.log(''); console.log(' cd ' + COMPUTE_JS_DIR); - console.log(' npm run publish-service'); - console.log(' npm run publish-content'); + console.log(' npm run fastly:deploy'); + console.log(' npm run fastly:publish'); console.log(''); } diff --git a/src/cli/fastly-api/api-token.ts b/src/cli/util/api-token.ts similarity index 98% rename from src/cli/fastly-api/api-token.ts rename to src/cli/util/api-token.ts index ea22826..9f9353b 100644 --- a/src/cli/fastly-api/api-token.ts +++ b/src/cli/util/api-token.ts @@ -6,7 +6,7 @@ import { spawnSync } from 'node:child_process'; import cli from '@fastly/cli'; -import { makeRetryable } from '../util/retryable.js'; +import { makeRetryable } from './retryable.js'; export interface FastlyApiContext { apiToken: string, diff --git a/src/cli/util/config.ts b/src/cli/util/config.ts index 7e7029c..a179a9a 100644 --- a/src/cli/util/config.ts +++ b/src/cli/util/config.ts @@ -65,6 +65,7 @@ export const normalizeStaticPublisherRc = buildNormalizeFunctionForObject( objects: TObject[], diff --git a/src/cli/util/kv-store-local-server.ts b/src/cli/util/kv-store-local-server.ts index 5750ba8..758463b 100644 --- a/src/cli/util/kv-store-local-server.ts +++ b/src/cli/util/kv-store-local-server.ts @@ -8,10 +8,120 @@ import path from 'node:path'; import { type KVStoreItemDesc } from './kv-store-items.js'; -export function writeKVStoreEntriesForLocal(storeFile: string, computeAppDir: string, kvStoreItemDescriptions: KVStoreItemDesc[]) { +type KVStoreLocalServerEntry = ({ data: string, } | { file: string }) & { metadata?:string }; +type KVStoreLocalServerData = Record; - type KVStoreLocalServerEntry = ({ data: string, } | { file: string }) & { metadata?:string }; - type KVStoreLocalServerData = Record; +export async function getLocalKVStoreKeys( + storeFile: string, + prefix?: string, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + let keys = Object.keys(store); + + if (prefix != null) { + keys = keys.filter(key => key.startsWith(prefix)); + } + + return keys; + +} + +export async function getLocalKvStoreEntry( + storeFile: string, + key: string, + metadataOnly?: boolean, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + const obj = store[key]; + if (obj == null) { + return null; + } + + let response; + if (metadataOnly) { + response = new Response(null); + } else if ('data' in obj) { + response = new Response(obj.data); + } else { + const fileData = fs.readFileSync(obj.file); + response = new Response(fileData); + } + return { + metadata: obj.metadata ?? null, + generation: null, + response, + }; + +} + +export async function localKvStoreSubmitEntry( + storeFile: string, + key: string, + file: string, + metadata: string | undefined, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + store[key] = { + file, + metadata, + }; + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} + +export async function localKvStoreDeleteEntry( + storeFile: string, + key: string, +) { + + let store: KVStoreLocalServerData; + try { + // If the local KV store file exists, we have to add to it. + const storeFileJson = fs.readFileSync(storeFile, 'utf-8') + store = JSON.parse(storeFileJson); + } catch { + store = {}; + } + + delete store[key]; + + fs.writeFileSync(storeFile, JSON.stringify(store)); + +} + +export function writeKVStoreEntriesForLocal( + storeFile: string, + computeAppDir: string, + kvStoreItemDescriptions: KVStoreItemDesc[], +) { let store: KVStoreLocalServerData; try { diff --git a/src/cli/fastly-api/kv-store.ts b/src/cli/util/kv-store.ts similarity index 95% rename from src/cli/fastly-api/kv-store.ts rename to src/cli/util/kv-store.ts index 6238e13..cb9c0c4 100644 --- a/src/cli/fastly-api/kv-store.ts +++ b/src/cli/util/kv-store.ts @@ -149,7 +149,7 @@ export async function getKvStoreEntry( const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { - return false; + return null; } const endpoint = `/resources/stores/kv/${encodeURIComponent(kvStoreId)}/keys/${encodeURIComponent(key)}`; @@ -161,7 +161,7 @@ export async function getKvStoreEntry( } catch(err) { if (err instanceof FetchError && err.status === 404) { - return false; + return null; } throw err; } @@ -177,7 +177,13 @@ export async function getKvStoreEntry( } const encoder = new TextEncoder(); -export async function kvStoreSubmitEntry(fastlyApiContext: FastlyApiContext, kvStoreName: string, key: string, data: ReadableStream | Uint8Array | string, metadata: string | undefined) { +export async function kvStoreSubmitEntry( + fastlyApiContext: FastlyApiContext, + kvStoreName: string, + key: string, + data: ReadableStream | Uint8Array | string, + metadata: string | undefined, +) { const kvStoreId = await getKVStoreIdForName(fastlyApiContext, kvStoreName); if (kvStoreId == null) { diff --git a/src/models/config/publish-content-config.ts b/src/models/config/publish-content-config.ts index b7fb365..2e069e2 100644 --- a/src/models/config/publish-content-config.ts +++ b/src/models/config/publish-content-config.ts @@ -46,10 +46,6 @@ export type PublishContentConfig = { // Set to a directory that acts as the root of all files that will be included in this publish. rootDir: string, - // Set to a directory that will hold the working files built by compute-js-static-publish - // These files should not be committed to source control. - staticPublisherWorkingDir: string, - // An array of values used to exclude files and directories (as well as files within those directories) from being // included in this publish. Each entry in the array can be a string or a RegExp and will be tested against the relative // path from 'rootDir' of each file or directory. @@ -80,7 +76,6 @@ export type PublishContentConfig = { export type PublishContentConfigNormalized = { rootDir: string, - staticPublisherWorkingDir: string, excludeDirs: ExcludeDirTest[], excludeDotFiles: boolean, includeWellKnown: boolean, diff --git a/src/models/config/static-publish-rc.ts b/src/models/config/static-publish-rc.ts index 1e4b122..2cfa001 100644 --- a/src/models/config/static-publish-rc.ts +++ b/src/models/config/static-publish-rc.ts @@ -13,4 +13,9 @@ export type StaticPublishRc = { // Default collection name. defaultCollectionName: string, -} + + // A directory that holds the working files built by compute-js-static-publish, + // relative to the Compute application. + // These files should not be committed to source control. + staticPublisherWorkingDir: string, +}; diff --git a/src/models/time/index.ts b/src/models/time/index.ts index 8d54245..d3c9a76 100644 --- a/src/models/time/index.ts +++ b/src/models/time/index.ts @@ -35,11 +35,16 @@ function parseDuration(durationStr: string) { export type CalcExpirationTimeArg = { expiresIn?: string, expiresAt?: string, + expiresNever?: boolean, }; -export function calcExpirationTime({ expiresIn, expiresAt }: CalcExpirationTimeArg) { - if (expiresIn && expiresAt) { - throw new Error('Only one of expiresIn or expiresAt may be provided'); +export function calcExpirationTime({ expiresIn, expiresAt, expiresNever }: CalcExpirationTimeArg) { + if ([expiresIn, expiresAt, expiresNever].filter(x => x !== undefined).length > 1) { + throw new Error('Only one of expiresIn or expiresAt or expiresNever may be provided'); + } + + if (expiresNever) { + return null; } if (expiresIn) { From 077af2862d2e0b00e1810831a2da4ffafd7bce08 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 00:50:44 +0900 Subject: [PATCH 10/20] More v7 readme updates --- README.md | 176 +++++++++++++++++++++++++++++++++++++++++------- README.short.md | 14 +++- 2 files changed, 163 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d40e1fa..d4e338b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - [✨ Key Features](#-key-features) - [🏁 Quick Start](#-quick-start) - [⚙️ Configuring `static-publish.rc.js`](#️-configuring-static-publishrcjs) -- [🧾 Default Server Config: `publish-content.config.js`](#-default-server-config-publish-contentconfigjs) +- [🧾 Default Server Config: `publish-content.config.js`](#-publish-and-server-config-publish-contentconfigjs) - [📦 Collections (Publish, Preview, Promote)](#-collections-publish-preview-promote) - [🧹 Cleaning Up](#-cleaning-up) - [🔄 Content Compression](#-content-compression) @@ -116,6 +116,7 @@ const rc = { kvStoreName: 'site-content', defaultCollectionName: 'live', publishId: 'default', + staticPublisherWorkingDir: './static-publisher', }; export default rc; @@ -123,24 +124,24 @@ export default rc; ### Fields: -- `kvStoreName` - The name of the KV Store used for publishing (required). -- `defaultCollectionName` - Collection to serve when none is specified (required). -- `publishId` - Unique prefix for all keys in the KV Store (required). Override only for advanced setups (e.g., multiple apps sharing the same KV Store). +All fields are required. + +- `kvStoreName` - The name of the KV Store used for publishing. +- `defaultCollectionName` - Collection to serve when none is specified. +- `publishId` - Unique prefix for all keys in the KV Store. Override only for advanced setups (e.g., multiple apps sharing the same KV Store). +- `staticPublisherWorkingDir` - Directory to hold working files during publish. > [!NOTE] > Changes to this file require rebuilding the Compute app, since a copy of it is baked into the Wasm binary. -## 🧾 Default Server Config: `publish-content.config.js` +## 🧾 Publish and Server Config: `publish-content.config.js` This file is included as part of the scaffolding. Every time you publish content, the publish settings in this file are used for publishing the content, and the server settings are taken from this file and saved as the settings used by the server for that collection. -You can override this file for a single `publish-content` command by specifying an alternative using `--config` on the command line. - ```js const config = { // these paths are relative to compute-js dir rootDir: '../public', - staticPublisherWorkingDir: './static-publisher', // Include/exclude filters (optional): excludeDirs: ['node_modules'], @@ -175,13 +176,11 @@ const config = { export default config; ``` -> [!NOTE] -> Changes to this file apply when content is published. +You can override this file for a single `publish-content` command by specifying an alternative using `--config` on the command line. ### Fields: - `rootDir` - Directory to scan for content, relative to this file (required). -- `staticPublisherWorkingDir` - Directory to hold working files during publish (default: `'./static-publisher'`). - `excludeDirs` - Array of directory names or regex patterns to exclude (default: `['./node_modules']`). - `excludeDotFiles` - Exclude dotfiles and dot-named directories (default: true). - `includeWellKnown` - Always include `.well-known` even if dotfiles are excluded (default: true). @@ -225,18 +224,19 @@ You can overwrite or republish any collection at any time. Old file hashes will Collections can expire automatically: -- Expired collections return 404s -- They’re ignored by the server -- Their files are cleaned up by `clean` +- Expired collections are ignored by the server and return 404s +- Expiration limits can be modified (shortened, extended, reenstated) using `collections update-expiration` +- They are cleaned up by `clean --delete-expired-collections` ```sh --expires-in=3d # relative (e.g. 1h, 2d, 1w) --expires-at=2025-05-01T12:00Z # absolute (ISO 8601) +--expires-never ``` ### Switching the active collection -By default, the app serves from the collection named in `static-publish +By default, the server app serves assets from the "default collection", named in `static-publish .rc.js` under `defaultCollectionName`. To switch the active collection, you add custom code to your Compute app that calls `publisherServer.setActiveCollectionName(name)`: ```js @@ -286,6 +286,8 @@ collectionSelector.fromRequest(request, req => req.headers.get('x-collection') ? #### From a Cookie +See [fromCookie](#from-cookie) for details on this feature. + ```js const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); ``` @@ -303,9 +305,10 @@ If you're happy with a preview or staging collection and want to make it live, u ```sh npx @fastly/compute-js-static-publish collections promote \ --collection-name=staging + --to=live ``` -This copies all content and server settings from the `staging` collection to the collection named in your `defaultCollectionName`. To copy to a different name, add `--to=some-other-name`. +This copies all content and server settings from the `staging` collection to `live`. You can also specify a new expiration: @@ -321,12 +324,33 @@ npx @fastly/compute-js-static-publish collections promote \ ## 🛠 Development → Production Workflow -During development, starting the local preview server (`npm run start`) will run `publish-content --local-only` automatically via a `prestart hook`. This simulates publishing by writing to `kvstore.json` instead of uploading to the actual KV Store. You can preview your site at `http://127.0.0.1:7676` - no Fastly account or service required. +### Local development + +During development, the local preview server (`npm run dev:start`) will run against assets loaded into the simulated KV Store provided by the local development environment. + +Prior to starting the server, publish the content to the simulated KV Store: + +```sh +npm run dev:publish # 'publish' your files to the simulated local KV Store +npm run dev:start # preview locally +``` + +This simulates publishing by writing to `kvstore.json` instead of uploading to the actual KV Store. You can preview your site at `http://127.0.0.1:7676` - no Fastly account or service required. + +Note that for local development, you will have to stop and restart the local development server each time you publish updates to your content. + +To publish to an alternative collection name, use: + +```sh +npm run dev:publish -- --collection-name=preview-123 +``` + +### Production When you're ready for production: 1. [Create a free Fastly account](https://www.fastly.com/signup/?tier=free) if you haven't already. -2. Run `npm run publish-service` +2. Run `npm run fastly:deploy` - This builds your Compute app into a Wasm binary - Deploys it to a new or existing Fastly Compute service - If creating a new service: @@ -336,7 +360,7 @@ When you're ready for production: Once deployed, publish content like so: ```sh -npm run publish-content +npm run fastly:publish ``` This: @@ -348,7 +372,7 @@ This: > [!TIP] > Upload to a specific collection by specifying the collection name when publishing content: > ```sh -> npm run publish-content -- --collection-name=preview-42 +> npm run fastly:publish -- --collection-name=preview-42 > ``` **No Wasm redeploy needed** unless you: @@ -359,14 +383,25 @@ This: If you do need to rebuild and redeploy the Compute app, simply run: ```sh -npm run publish-service +npm run dev:deploy ``` ## 🧹 Cleaning Up -Every time you publish, old files are left behind for safety. **However, files with the same content will be re-used across collections and publishing events** - they are only stored once in the KV Store using their content hash as a key. This ensures that unchanged files aren't duplicated, keeping storage efficient and deduplicated. To avoid bloat, use: +Every time you publish, old files are left behind for safety. **However, files with the same content will be re-used across collections and publishing events.** They are only stored once in the KV Store using their content hash as a key. This ensures that unchanged files aren't duplicated, keeping storage efficient and deduplicated. + +Over time, however, collections may expire, old versions of files will be left behind, and some assets in the KV Store will no longer be referenced by any live collection. To avoid bloat, use: ```sh +npm run dev:clean +``` +and +```sh +npm run fastly:clean +``` + +These scripts run against the local and Fastly KV Stores respectively, and run the following command: +``` npx @fastly/compute-js-static-publish clean --delete-expired-collections ``` @@ -490,10 +525,10 @@ npx @fastly/compute-js-static-publish \ --root-dir=./public \ --kv-store-name=site-content \ [--output=./compute-js] \ + [--static-publisher-working-dir=/static-publisher] \ [--publish-id=] \ [--public-dir=./public] \ [--static-dir=./public/static] \ - [--static-publisher-working-dir=./compute-js/static-publisher] \ [--spa=./public/spa.html] \ [--not-found-page=./public/404.html] \ [--auto-index=index.html,index.htm] \ @@ -509,6 +544,7 @@ npx @fastly/compute-js-static-publish \ **Used to generate the Compute app:** - `--kv-store-name`: Required. Name of KV Store to use. - `--output`: Compute app destination (default: `./compute-js`) +- `--static-publisher-working-dir`: Directory to hold working files (default: `/static-publisher`). - `--name`: Application name to insert into `fastly.toml` - `--description`: Application description to insert into `fastly.toml` - `--author`: Author to insert into `fastly.toml` @@ -517,7 +553,6 @@ npx @fastly/compute-js-static-publish \ **Used in building config files:** - `--root-dir`: Required. Directory of static site content. - `--publish-id`: Optional key prefix for KV entries (default: `'default'`). -- `--static-publisher-working-dir`: Directory to hold working files (default: `/static-publisher`). - `--public-dir`: Public files base directory (default: same as `--root-dir`). - `--static-dir`: One or more directories to serve with long cache TTLs. - `--spa`: SPA fallback file path (e.g., `./public/spa.html`). @@ -637,6 +672,99 @@ Deletes a specific collection’s index and associated settings. --- +## Appendix + +### fromCookie + +`collectionSelector.fromCookie` is a utility function that enables the use of a browser +cookie to select the active collection. + +```js +import { collectionSelector } from '@fastly/compute-js-static-publish'; + +const { collectionName, redirectResponse } = + collectionSelector.fromCookie(request, { + // Everything here is optional – these are just examples + cookieName: 'publisher-collection', // default + activatePath: '/activate', // default + resetPath: '/reset', // default + cookieHttpOnly: true, // default + cookieMaxAge: 60 * 60 * 24 * 7, // 7-day preview + cookiePath: '/', // default + }); + +if (redirectResponse) { + // honor the redirect + return redirectResponse; +} + +// `collectionName` now holds the active collection (or `null` if none) +``` + +#### What it does, in plain English + +1. Reads a cookie + It looks for a cookie named cookieName (default `publisher-collection`) and hands you the value as `collectionName`. + +2. Handles two helper endpoints for you + +| Endpoint | Purpose | Query params | Result | +|------------------------------------|------------------|------------------------------------------------------------------------------------------------|-------------------------------------------| +| activatePath (default `/activate`) | Set the cookie | `collection` (required) - name of the collection
`redirectTo` (optional, `/` by default) | `302` redirect with a `Set-Cookie` header | +| resetPath (default `/reset`) | Clear the cookie | `redirectTo` (optional, `/` by default) | `302` redirect that expires the cookie | + +If a visitor accesses `/activate?collection=blue&redirectTo=/preview`, the helper will issue a redirect and drop `publisher-collection=blue` into their cookie jar. + - If someone forgets `?collection=?`? Then `/activate` replies with HTTP `400`. + +When the visitor hits `/reset`, the cookie is deleted. + +3. Safety flags are baked in + + - `HttpOnly` is on by default (configurable), to help avoid XSS issues. + - `Secure` is automatically added on HTTPS requests. + - `SameSite=Lax` is always set – reasonable default for previews. + +#### Option reference + +| Option | Type | Default | Notes | +|------------------|-------------------|--------------------------|------------------------------------------------| +| `cookieName` | string | `'publisher-collection'` | Name of the cookie to read/write | +| `activatePath` | string | `'/activate'` | Path that sets the cookie | +| `resetPath` | string | `'/reset'` | Path that clears the cookie | +| `cookieHttpOnly` | boolean | `true` | Turn off if the client-side needs to read it | +| `cookieMaxAge` | number\|undefined | `undefined` | Seconds until expiry; omit for session cookies | +| `cookiePath` | string | `'/'` | Path attribute in the cookie | + +#### Example + +`fromCookie` can be dropped into a Fastly Compute app: + +```js +async function handleRequest(event) { + const request = event.request; + + // --- Cookie handling starts here + const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); + if (redirectResponse) { + // obey redirect first + return redirectResponse; + } + if (collectionName != null) { + publisherServer.setActiveCollectionName(collectionName); + } + // --- Cookie handling ends here + + // Regular routing follows... + const response = await publisherServer.serveRequest(request); + if (response != null) { + return response; + } + return new Response('Not found', { status: 404 }); +} +``` + +--- + ## 📚 Next Steps - View CLI command help: `npx @fastly/compute-js-static-publish --help` diff --git a/README.short.md b/README.short.md index 4dbd7b8..50f0c1a 100644 --- a/README.short.md +++ b/README.short.md @@ -25,7 +25,8 @@ npx @fastly/compute-js-static-publish --root-dir=./public --kv-store-name=site-c ```sh cd compute-js npm install -npm run start # preview locally +npm run dev:publish # 'publish' your files to the simulated local KV Store +npm run dev:start # preview locally ``` Serves your app at `http://127.0.0.1:7676`, powered by a simulated KV Store. @@ -36,8 +37,15 @@ When you're ready to go live, [create a free Fastly account](https://www.fastly. ```sh cd compute-js -npm run publish-service # deploy the app (publish the "service") -npm run publish-content # upload your static files (publish the "content") +npm run fastly:deploy # deploy the app +npm run fastly:publish # upload your static files +``` + +In the future, unless you have further changes to make to your app itself, you can +upload further updates to your static files: +```sh +cd compute-js +npm run fastly:publish # upload your static files ``` ## Features From 954ccecf0cb65dd9308fff8f9330caee6646a96c Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 14:29:59 +0900 Subject: [PATCH 11/20] v7.0.0-beta.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6e1b5d5..ae36fc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.2", + "version": "7.0.0-beta.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.2", + "version": "7.0.0-beta.3", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index 15daf73..5aa051d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "7.0.0-beta.2", + "version": "7.0.0-beta.3", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From 2a112c5b842492d2ae82535ff3d1003cb27111e5 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 19:16:35 +0900 Subject: [PATCH 12/20] v7 updates --- src/cli/commands/manage/clean.ts | 2 +- src/cli/commands/manage/collections/delete.ts | 4 ++-- src/cli/commands/manage/collections/update-expiration.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts index ad31bb7..d722231 100644 --- a/src/cli/commands/manage/clean.ts +++ b/src/cli/commands/manage/clean.ts @@ -31,7 +31,7 @@ Usage: Description: Cleans up expired or unreferenced items in the Fastly KV Store. - This includes expired collection indexes and orphaned content assets. + This can include expired collection indexes and orphaned content assets. Options: --delete-expired-collections If set, expired collection index files will be deleted. diff --git a/src/cli/commands/manage/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts index 3ce3a15..688cb37 100644 --- a/src/cli/commands/manage/collections/delete.ts +++ b/src/cli/commands/manage/collections/delete.ts @@ -23,8 +23,8 @@ Usage: npx @fastly/compute-js-static-publish collections delete --collection-name= [options] Description: - Deletes a collection index from the KV Store. The content files will remain but will no - longer be referenced. + Deletes a collection index from the KV Store. The content files will remain as they may still + be referenced by other collection indexes. Use the 'npx @fastly/compute-js-static-publish clean' command afterward to remove content files that are no longer referenced by any collection. diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts index 0009bbf..f267e34 100644 --- a/src/cli/commands/manage/collections/update-expiration.ts +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -29,7 +29,7 @@ Usage: [options] Description: - Updates the expiration time of an existing collection. + Sets or updates the expiration time of an existing collection. Required: --collection-name= The name of the collection to modify From 5b0f59327333e1cc24cf80e439811209a13c311b Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 18:24:22 +0900 Subject: [PATCH 13/20] More v7 readme updates --- README.md | 168 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 115 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index d4e338b..7d31a5b 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,14 @@ - [✨ Key Features](#-key-features) - [🏁 Quick Start](#-quick-start) - [⚙️ Configuring `static-publish.rc.js`](#️-configuring-static-publishrcjs) -- [🧾 Default Server Config: `publish-content.config.js`](#-publish-and-server-config-publish-contentconfigjs) +- [🧾 Config for Publishing and Server: `publish-content.config.js`](#-config-for-publishing-and-server-publish-contentconfigjs) - [📦 Collections (Publish, Preview, Promote)](#-collections-publish-preview-promote) - [🧹 Cleaning Up](#-cleaning-up) - [🔄 Content Compression](#-content-compression) - [🧩 Using PublisherServer in Custom Apps](#-using-publisherserver-in-custom-apps) - [📥 Using Published Assets in Your Code](#-using-published-assets-in-your-code) +- [📚 CLI Reference](#-cli-reference) +- [📕 Appendix](#-appendix) - [📚 Next Steps](#-next-steps) Serve static websites and web apps at the edge — no backends and no CDN invalidation delays. @@ -55,7 +57,8 @@ Type the following — no Fastly account or service required yet! ```sh cd compute-js npm install -npm run start +npm run dev:publish +npm run dev:start ``` Fastly's [local development environment](https://www.fastly.com/documentation/guides/compute/testing/#running-a-local-testing-server) serves your static website at `http://127.0.0.1:7676`, powered by a simulated KV Store. @@ -65,7 +68,7 @@ Fastly's [local development environment](https://www.fastly.com/documentation/gu Ready to go live? All you need is a [free Fastly account](https://www.fastly.com/signup/?tier=free)! ```sh -npm run publish-service +npm run fastly:deploy ``` The command publishes your Compute app and creates the KV Store. (No content uploaded yet!) @@ -73,7 +76,7 @@ The command publishes your Compute app and creates the KV Store. (No content upl ### 4. Publish Content ```sh -npm run publish-content +npm run fastly:publish ``` Upload static files to the KV Store and applies the server config. Your website is now up and live! @@ -134,7 +137,7 @@ All fields are required. > [!NOTE] > Changes to this file require rebuilding the Compute app, since a copy of it is baked into the Wasm binary. -## 🧾 Publish and Server Config: `publish-content.config.js` +## 🧾 Config for Publishing and Server: `publish-content.config.js` This file is included as part of the scaffolding. Every time you publish content, the publish settings in this file are used for publishing the content, and the server settings are taken from this file and saved as the settings used by the server for that collection. @@ -225,19 +228,21 @@ You can overwrite or republish any collection at any time. Old file hashes will Collections can expire automatically: - Expired collections are ignored by the server and return 404s +- The default collection never expires - Expiration limits can be modified (shortened, extended, reenstated) using `collections update-expiration` - They are cleaned up by `clean --delete-expired-collections` ```sh ---expires-in=3d # relative (e.g. 1h, 2d, 1w) +--expires-in=3d # relative (e.g. 1h, 2d, 1w) --expires-at=2025-05-01T12:00Z # absolute (ISO 8601) ---expires-never +--expires-never # the collection never expires ``` +*(Only one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified)* + ### Switching the active collection -By default, the server app serves assets from the "default collection", named in `static-publish -.rc.js` under `defaultCollectionName`. To switch the active collection, you add custom code to your Compute app that calls `publisherServer.setActiveCollectionName(name)`: +By default, the server app serves assets from the "default collection", named in `static-publish.rc.js` under `defaultCollectionName`. To switch the active collection, you add custom code to your Compute app that calls `publisherServer.setActiveCollectionName(name)`: ```js publisherServer.setActiveCollectionName("preview-42"); @@ -286,7 +291,7 @@ collectionSelector.fromRequest(request, req => req.headers.get('x-collection') ? #### From a Cookie -See [fromCookie](#from-cookie) for details on this feature. +See [fromCookie](#fromcookie) for details on this feature. ```js const { collectionName, redirectResponse } = collectionSelector.fromCookie(request); @@ -490,7 +495,7 @@ if (asset != null) { ```js const kvAssetVariant = await publisherServer.loadKvAssetVariant(asset, null); // pass 'gzip' or 'br' for compressed -kvAssetVariant.kvStoreEntry; // KV Store entry +kvAssetVariant.kvStoreEntry; // KV Store entry (type KVStoreEntry defined in 'fastly:kv-store') kvAssetVariant.size; // Size of the variant kvAssetVariant.hash; // SHA256 of the variant kvAssetVariant.contentEncoding; // 'gzip', 'br', or null @@ -571,37 +576,37 @@ npx @fastly/compute-js-static-publish publish-content \ [--root-dir=./public] \ [--collection-name=preview-42] \ [--config=./publish-content.config.js] \ - [--expires-in=7d | --expires-at=2025-05-01T12:00Z] \ - [--local-only | --no-local] \ + [--expires-in=7d | --expires-at=2025-05-01T12:00Z | --expires-never] \ + [--local] \ [--fastly-api-token=...] ``` -Publishes your static files and server config for a given collection. +Publishes static files from your local root directory into a named collection, either in the Fastly KV Store (default) or to a local dev directory (`--local`). Files that already exist with the same hash are skipped automatically. -#### Options: +After this process is complete, the PublisherServer object in the Compute application will see the updated index of files and updated server settings from the `publish-content.config.js` file. -**Configuration:** +##### Options: +- `--collection-name`: Name of the collection to create/update (default: value in `static-publish.rc.js`) - `--config`: Path to a config file to configure server behavior for this collection (default: `./publish-content.config.js`) +- `--root-dir`: Source directory to read files from (overrides value in `publish-content.config.js`) +- `--kv-overwrite`: Cannot be used with `--local`. When using Fastly KV Store, always overwrites existing entries, even if unchanged. -**Content and Collection:** +**Expiration:** -- `--root-dir`: Source directory to read files from (overrides value in `publish-content.config.js`) -- `--collection-name`: Name of the collection to create/update (default: value in `static-publish.rc.js`) - `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) -- `--expires-at`: Absolute expiration time (ISO format: `2025-05-01T12:00Z`) *(Only one of **`--expires-in`** or **`--expires-at`** may be specified)* +- `--expires-at`: Absolute expiration time (ISO format: `2025-05-01T12:00Z`) +- `--expires-never`: Collection never expires -**Mode:** +*At most one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified* -- `--local-only`: Do not upload files to Fastly KV Store; only simulate KV Store locally -- `--no-local`: Do not prepare files for local simulated KV Store; upload to real KV Store only +**Global Options:** -**Auth:** +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. -- `--fastly-api-token`: API token to use when publishing\ - *(Overrides **`FASTLY_API_TOKEN`** environment variable and **`fastly profile token`**)* -- Stores server config per collection -- Supports expiration settings +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile #### `clean` @@ -611,19 +616,38 @@ npx @fastly/compute-js-static-publish clean \ [--dry-run] ``` -Removes unreferenced content from the KV Store. +Cleans up expired or unreferenced items in the Fastly KV Store. +This can include expired collection indexes and orphaned content assets. -#### Options: +##### Options: + +- `--delete-expired-collections`: If set, expired collection index files will be deleted. +- `--dry-run`: Show what would be deleted without performing any deletions. + +**Global Options:** -- `--delete-expired-collections`: Also delete collection index files that have expired -- `--dry-run`: Show what would be deleted without actually removing anything +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile #### `collections list` ```sh npx @fastly/compute-js-static-publish collections list ``` -Lists all known collection names and metadata. +Lists all collections currently published in the KV Store. + +##### Options: + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile #### `collections promote` @@ -631,33 +655,58 @@ Lists all known collection names and metadata. npx @fastly/compute-js-static-publish collections promote \ --collection-name=preview-42 \ --to=live \ - [--expires-in=7d | --expires-at=2025-06-01T00:00Z] + [--expires-in=7d | --expires-at=2025-06-01T00:00Z | --expires-never] ``` Copies an existing collection (content + config) to a new collection name. -#### Options: +##### Options: + - `--collection-name`: The name of the source collection to promote (required) - `--to`: The name of the new (target) collection to create or overwrite (required) + +**Expiration:** + - `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) - `--expires-at`: Absolute expiration time (ISO format) +- `--expires-never`: Collection never expires -*Exactly one of **`--expires-in`** or **`--expires-at`** may be provided.* +*At most one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** may be specified*. If not provided, then the existing expiration rule of the collection being promoted is used. + +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile #### `collections update-expiration` ```sh npx @fastly/compute-js-static-publish collections update-expiration \ --collection-name=preview-42 \ - --expires-in=3d | --expires-at=2025-06-01T00:00Z + --expires-in=3d | --expires-at=2025-06-01T00:00Z | --expires-never ``` -Sets or updates the expiration time of a collection. +Sets or updates the expiration time of an existing collection. -#### Options: +##### Options: - `--collection-name`: The name of the collection to update (required) + +**Expiration:** + - `--expires-in`: Time-to-live from now (e.g. `1h`, `2d`, `1w`) - `--expires-at`: Absolute expiration time (ISO format) +- `--expires-never`: Collection never expires + +*Exactly one of **`--expires-in`**, **`--expires-at`**, or **`--expires-never`** must be specified* + +**Global Options:** -*Exactly one of **`--expires-in`** or **`--expires-at`** must be provided.* +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile #### `collections delete` @@ -665,14 +714,26 @@ Sets or updates the expiration time of a collection. npx @fastly/compute-js-static-publish collections delete \ --collection-name=preview-42 ``` -Deletes a specific collection’s index and associated settings. -#### Options: +Deletes a collection index from the KV Store. The content files will remain as they may still be referenced by other collection indexes. + +Use the `npx @fastly/compute-js-static-publish clean` command afterward to remove content files that are no longer referenced by any collection. + +##### Options: + - `--collection-name`: The name of the collection to delete (required) +**Global Options:** + +- `--local`: Instead of working with the Fastly KV Store, operate on local files that will be used to simulate the KV Store with the local development environment. + +- `--fastly-api-token`: API token to use when publishing. If not set, the tool will check: + - **`FASTLY_API_TOKEN` environment variable** + - Logged-in Fastly CLI profile + --- -## Appendix +## 📕 Appendix ### fromCookie @@ -689,7 +750,7 @@ const { collectionName, redirectResponse } = activatePath: '/activate', // default resetPath: '/reset', // default cookieHttpOnly: true, // default - cookieMaxAge: 60 * 60 * 24 * 7, // 7-day preview + cookieMaxAge: 60 * 60 * 24 * 7, // 7-days. default is `null`, never expires. cookiePath: '/', // default }); @@ -704,19 +765,20 @@ if (redirectResponse) { #### What it does, in plain English 1. Reads a cookie + It looks for a cookie named cookieName (default `publisher-collection`) and hands you the value as `collectionName`. 2. Handles two helper endpoints for you -| Endpoint | Purpose | Query params | Result | -|------------------------------------|------------------|------------------------------------------------------------------------------------------------|-------------------------------------------| -| activatePath (default `/activate`) | Set the cookie | `collection` (required) - name of the collection
`redirectTo` (optional, `/` by default) | `302` redirect with a `Set-Cookie` header | -| resetPath (default `/reset`) | Clear the cookie | `redirectTo` (optional, `/` by default) | `302` redirect that expires the cookie | - -If a visitor accesses `/activate?collection=blue&redirectTo=/preview`, the helper will issue a redirect and drop `publisher-collection=blue` into their cookie jar. - - If someone forgets `?collection=?`? Then `/activate` replies with HTTP `400`. - -When the visitor hits `/reset`, the cookie is deleted. + | Endpoint | Purpose | Query params | Result | + |------------------------------------|------------------|------------------------------------------------------------------------------------------------|-------------------------------------------| + | activatePath (default `/activate`) | Set the cookie | `collection` (required) - name of the collection
`redirectTo` (optional, `/` by default) | `302` redirect with a `Set-Cookie` header | + | resetPath (default `/reset`) | Clear the cookie | `redirectTo` (optional, `/` by default) | `302` redirect that expires the cookie | + + If a visitor accesses `/activate?collection=blue&redirectTo=/preview`, the helper will issue a redirect and drop `publisher-collection=blue` into their cookie jar. + - If someone forgets `?collection=?`? Then `/activate` replies with HTTP `400`. + + When the visitor hits `/reset`, the cookie is deleted. 3. Safety flags are baked in From a98d8edc868cf7d0ff41a3f6c5d2543f45917e82 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Wed, 23 Apr 2025 22:01:02 +0900 Subject: [PATCH 14/20] v7.0.0-beta.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index ae36fc4..568a01d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.3", + "version": "7.0.0-beta.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.3", + "version": "7.0.0-beta.4", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index 5aa051d..9751da2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "7.0.0-beta.3", + "version": "7.0.0-beta.4", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From 216fc9e2c99ccab5ace166298b219eff5afe5c09 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 13:06:31 +0900 Subject: [PATCH 15/20] More v7 updates --- src/cli/commands/manage/clean.ts | 16 ++- src/cli/commands/manage/collections/index.ts | 6 +- .../commands/manage/collections/promote.ts | 3 +- .../manage/collections/update-expiration.ts | 18 ++- src/cli/commands/manage/index.ts | 6 +- src/cli/commands/manage/publish-content.ts | 121 +++++++++--------- src/cli/commands/scaffold/index.ts | 3 +- src/cli/util/api-token.ts | 13 +- src/cli/util/args.ts | 13 +- 9 files changed, 115 insertions(+), 84 deletions(-) diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts index d722231..e7e37c8 100644 --- a/src/cli/commands/manage/clean.ts +++ b/src/cli/commands/manage/clean.ts @@ -10,18 +10,22 @@ import { type OptionDefinition } from 'command-line-args'; import { type KVAssetEntryMap } from '../../../models/assets/kvstore-assets.js'; import { type IndexMetadata } from '../../../models/server/index.js'; import { isExpired } from '../../../models/time/index.js'; -import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; -import { getKvStoreEntry, getKVStoreKeys, kvStoreDeleteEntry } from '../../util/kv-store.js'; import { type FastlyApiContext, loadApiToken } from '../../util/api-token.js'; import { parseCommandLine } from '../../util/args.js'; +import { LoadConfigError, loadStaticPublisherRcFile } from '../../util/config.js'; import { readServiceId } from '../../util/fastly-toml.js'; -import { doKvStoreItemsOperation } from "../../util/kv-store-items.js"; -import { isNodeError } from '../../util/node.js'; +import { + getKvStoreEntry, + getKVStoreKeys, + kvStoreDeleteEntry, +} from '../../util/kv-store.js'; +import { doKvStoreItemsOperation } from '../../util/kv-store-items.js'; import { getLocalKvStoreEntry, getLocalKVStoreKeys, - localKvStoreDeleteEntry -} from "../../util/kv-store-local-server.js"; + localKvStoreDeleteEntry, +} from '../../util/kv-store-local-server.js'; +import { isNodeError } from '../../util/node.js'; function help() { console.log(`\ diff --git a/src/cli/commands/manage/collections/index.ts b/src/cli/commands/manage/collections/index.ts index c34f438..0e7a09f 100644 --- a/src/cli/commands/manage/collections/index.ts +++ b/src/cli/commands/manage/collections/index.ts @@ -55,9 +55,9 @@ export async function action(actionArgs: string[]) { const commandAndArgs = getCommandAndArgs(actionArgs, modes); if (commandAndArgs.needHelp) { - if (commandAndArgs.command != null) { - console.error(`Unknown subcommand: ${commandAndArgs.command}`); - console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + if (commandAndArgs.error != null) { + console.error(commandAndArgs.error); + console.error(`Specify one of the following sub-commands: ${Object.keys(modes).join(', ')}`); console.error(); process.exitCode = 1; } diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts index 120ec57..93e5319 100644 --- a/src/cli/commands/manage/collections/promote.ts +++ b/src/cli/commands/manage/collections/promote.ts @@ -3,6 +3,8 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import fs from 'node:fs'; + import { type OptionDefinition } from 'command-line-args'; import { type IndexMetadata } from '../../../../models/server/index.js'; @@ -18,7 +20,6 @@ import path from "node:path"; import { readServiceId } from "../../../util/fastly-toml.js"; import { isNodeError } from "../../../util/node.js"; import { getLocalKvStoreEntry, localKvStoreSubmitEntry } from "../../../util/kv-store-local-server.js"; -import fs from "node:fs"; function help() { console.log(`\ diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts index f267e34..10f155c 100644 --- a/src/cli/commands/manage/collections/update-expiration.ts +++ b/src/cli/commands/manage/collections/update-expiration.ts @@ -3,22 +3,26 @@ * Licensed under the MIT license. See LICENSE file for details. */ +import fs from 'node:fs'; +import path from 'node:path'; + import { type OptionDefinition } from 'command-line-args'; import { type IndexMetadata } from '../../../../models/server/index.js'; import { calcExpirationTime } from '../../../../models/time/index.js'; +import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; import { LoadConfigError, loadStaticPublisherRcFile } from '../../../util/config.js'; +import { parseCommandLine } from '../../../util/args.js'; +import { readServiceId } from '../../../util/fastly-toml.js'; import { getKvStoreEntry, kvStoreSubmitEntry, } from '../../../util/kv-store.js'; -import { type FastlyApiContext, loadApiToken } from '../../../util/api-token.js'; -import { parseCommandLine } from '../../../util/args.js'; -import { readServiceId } from "../../../util/fastly-toml.js"; -import { isNodeError } from "../../../util/node.js"; -import path from "node:path"; -import fs from "node:fs"; -import { getLocalKvStoreEntry, localKvStoreSubmitEntry } from "../../../util/kv-store-local-server.js"; +import { + getLocalKvStoreEntry, + localKvStoreSubmitEntry, +} from '../../../util/kv-store-local-server.js'; +import { isNodeError } from '../../../util/node.js'; function help() { console.log(`\ diff --git a/src/cli/commands/manage/index.ts b/src/cli/commands/manage/index.ts index 19ff824..34f5b6b 100644 --- a/src/cli/commands/manage/index.ts +++ b/src/cli/commands/manage/index.ts @@ -61,9 +61,9 @@ export async function action(actionArgs: string[]) { const commandAndArgs = getCommandAndArgs(actionArgs, modes); if (commandAndArgs.needHelp) { - if (commandAndArgs.command != null) { - console.error(`Unknown command: ${commandAndArgs.command}`); - console.error(`Specify one of: ${Object.keys(modes).join(', ')}`); + if (commandAndArgs.error != null) { + console.error(commandAndArgs.error); + console.error(`Specify one of the following commands: ${Object.keys(modes).join(', ')}`); console.error(); process.exitCode = 1; } diff --git a/src/cli/commands/manage/publish-content.ts b/src/cli/commands/manage/publish-content.ts index 214ffed..9034fe3 100644 --- a/src/cli/commands/manage/publish-content.ts +++ b/src/cli/commands/manage/publish-content.ts @@ -21,13 +21,16 @@ import { mergeContentTypes, testFileContentType } from '../../util/content-types import { LoadConfigError, loadPublishContentConfigFile, loadStaticPublisherRcFile } from '../../util/config.js'; import { applyDefaults } from '../../util/data.js'; import { readServiceId } from '../../util/fastly-toml.js'; -import { calculateFileSizeAndHash, enumerateFiles, getFileSize, rootRelative } from '../../util/files.js'; +import { calculateFileSizeAndHash, enumerateFiles, rootRelative } from '../../util/files.js'; import { applyKVStoreEntriesChunks, doKvStoreItemsOperation, type KVStoreItemDesc, } from '../../util/kv-store-items.js'; -import { writeKVStoreEntriesForLocal } from '../../util/kv-store-local-server.js'; +import { + localKvStoreSubmitEntry, + writeKVStoreEntriesForLocal, +} from '../../util/kv-store-local-server.js'; import { isNodeError } from '../../util/node.js'; import { ensureVariantFileExists, type Variants } from '../../util/variants.js'; @@ -285,11 +288,6 @@ export async function action(actionArgs: string[]) { const collectionName = collectionNameValue ?? process.env.PUBLISHER_COLLECTION_NAME ?? defaultCollectionName; console.log(`✔️ Collection Name: ${collectionName}`); - if (expirationTime != null) { - console.log(`✔️ Publishing with expiration timestamp: ${new Date(expirationTime * 1000).toISOString()}`); - } else { - console.log(`✔️ Publishing with no expiration timestamp.`); - } if (expirationTime == null) { // includes null and undefined console.log(`✔️ Publishing with no expiration timestamp.`); @@ -555,34 +553,56 @@ export async function action(actionArgs: string[]) { } console.log(`✅ Scan complete.`) - // TODO: fix this bug: - // Technically it's a bug to WRITE the index and settings json files to the - // staticPublisherKvStoreContent dir when target === 'fastly'. - // For these files we should not be creating the files using - // kvStoreItemDescriptions, but rather creating them directly. - - // #### INDEX FILE - console.log(`🗂️ Creating Index...`); - const indexFileName = `index_${collectionName}.json`; - const indexFileKey = `${publishId}_index_${collectionName}`; + console.log(`🍪 Chunking large files...`); + await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); + console.log(`✅ Large files have been chunked.`); - const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); - fs.writeFileSync(indexFilePath, JSON.stringify(kvAssetsIndex)); + if (localMode) { + console.log(`📝 Writing local server KV Store entries.`); + writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); + console.log(`✅ Wrote KV Store entries for local server.`); + } else { + console.log(`📤 Uploading entries to KV Store.`); + // fastlyApiContext is non-null if useKvStore is true + await doKvStoreItemsOperation( + kvStoreItemDescriptions.filter(x => x.write), + async ({filePath, metadataJson}, key) => { + const fileBytes = fs.readFileSync(filePath); + await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); + console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) + } + ); + console.log(`✅ Uploaded entries to KV Store.`); + } - const indexFileSize = getFileSize(indexFilePath); + // #### INDEX FILE + console.log(`🗂️ Saving Index...`); + const indexFileKey = `${publishId}_index_${collectionName}`; const indexMetadata: IndexMetadata = { publishedTime: Math.floor(Date.now() / 1000), expirationTime: expirationTime ?? undefined, }; - kvStoreItemDescriptions.push({ - write: true, - size: indexFileSize, - key: indexFileKey, - filePath: indexFilePath, - metadataJson: indexMetadata, - }); + if (localMode) { + const indexFileName = `index_${collectionName}.json`; + const indexFilePath = path.resolve(staticPublisherKvStoreContent, indexFileName); + fs.writeFileSync(indexFilePath, JSON.stringify(kvAssetsIndex)); + await localKvStoreSubmitEntry( + storeFile, + indexFileKey, + path.relative(computeAppDir, indexFilePath), + JSON.stringify(indexMetadata), + ); + } else { + await kvStoreSubmitEntry( + fastlyApiContext!, + kvStoreName, + indexFileKey, + JSON.stringify(kvAssetsIndex), + JSON.stringify(indexMetadata), + ); + } console.log(`✅ Index has been saved.`) // #### SERVER SETTINGS @@ -652,42 +672,29 @@ export async function action(actionArgs: string[]) { autoIndex, }; - const settingsFileName = `settings_${collectionName}.json`; const settingsFileKey = `${publishId}_settings_${collectionName}`; - const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); - fs.writeFileSync(settingsFilePath, JSON.stringify(serverSettings)); - const settingsFileSize = getFileSize(settingsFilePath); - - kvStoreItemDescriptions.push({ - write: true, - size: settingsFileSize, - key: settingsFileKey, - filePath: settingsFilePath, - }); - console.log(`✅ Settings have been saved.`); - - console.log(`🍪 Chunking large files...`); - await applyKVStoreEntriesChunks(kvStoreItemDescriptions, KV_STORE_CHUNK_SIZE); - console.log(`✅ Large files have been chunked.`); - if (localMode) { - console.log(`📝 Writing local server KV Store entries.`); - writeKVStoreEntriesForLocal(storeFile, computeAppDir, kvStoreItemDescriptions); - console.log(`✅ Wrote KV Store entries for local server.`); + const settingsFileName = `settings_${collectionName}.json`; + const settingsFilePath = path.resolve(staticPublisherKvStoreContent, settingsFileName); + fs.writeFileSync(settingsFilePath, JSON.stringify(serverSettings)); + await localKvStoreSubmitEntry( + storeFile, + settingsFileKey, + path.relative(computeAppDir, settingsFilePath), + undefined, + ); + } else { - console.log(`📤 Uploading entries to KV Store.`); - // fastlyApiContext is non-null if useKvStore is true - await doKvStoreItemsOperation( - kvStoreItemDescriptions.filter(x => x.write), - async ({filePath, metadataJson}, key) => { - const fileBytes = fs.readFileSync(filePath); - await kvStoreSubmitEntry(fastlyApiContext!, kvStoreName, key, fileBytes, metadataJson != null ? JSON.stringify(metadataJson) : undefined); - console.log(` 🌐 Submitted asset "${rootRelative(filePath)}" to KV Store with key "${key}".`) - } + await kvStoreSubmitEntry( + fastlyApiContext!, + kvStoreName, + settingsFileKey, + JSON.stringify(serverSettings), + undefined, ); - console.log(`✅ Uploaded entries to KV Store.`); } + console.log(`✅ Settings have been saved.`); console.log(`🎉 Completed.`); diff --git a/src/cli/commands/scaffold/index.ts b/src/cli/commands/scaffold/index.ts index 3bdbb8e..d704956 100644 --- a/src/cli/commands/scaffold/index.ts +++ b/src/cli/commands/scaffold/index.ts @@ -7,15 +7,16 @@ // in a subfolder named compute-js. // This project can be served using fastly compute serve // or deployed to a Compute service using fastly compute publish. + import * as child_process from 'node:child_process'; import * as path from 'node:path'; import * as fs from 'node:fs'; import { type CommandLineOptions, type OptionDefinition } from 'command-line-args'; +import { parseCommandLine } from '../../util/args.js'; import { dotRelative, rootRelative } from '../../util/files.js'; import { findComputeJsStaticPublisherVersion, type PackageJson } from '../../util/package.js'; -import { parseCommandLine } from "../../util/args.js"; function help() { console.log(`\ diff --git a/src/cli/util/api-token.ts b/src/cli/util/api-token.ts index 9f9353b..7c0be5a 100644 --- a/src/cli/util/api-token.ts +++ b/src/cli/util/api-token.ts @@ -43,10 +43,19 @@ export function loadApiToken(params: LoadApiParams): LoadApiTokenResult | null { // Try fastly cli if (apiToken == null) { try { - const { stdout, error } = spawnSync(cli, ['profile', 'token', '--quiet'], { + const { stdout, error, status } = spawnSync(cli, ['profile', 'token', '--quiet'], { encoding: 'utf-8', }); - apiToken = error ? null : stdout.trim(); + if (status != null && status !== 0) { + console.warn(`⚠️ Warning: 'fastly profile token' returned a non-zero status code.`); + apiToken = null; + } else if (error) { + console.warn(`⚠️ Warning: 'fastly profile token' returned an error:`); + console.warn(String(error)); + apiToken = null; + } else { + apiToken = stdout.trim(); + } } catch { apiToken = null; } diff --git a/src/cli/util/args.ts b/src/cli/util/args.ts index 15cddea..9d62999 100644 --- a/src/cli/util/args.ts +++ b/src/cli/util/args.ts @@ -45,7 +45,7 @@ export type CommandAndArgs = { export type CommandAndArgsHelp = { needHelp: true, - command: string | null, + error: string | null, } export type CommandAndArgsResult = CommandAndArgs | CommandAndArgsHelp; @@ -57,7 +57,7 @@ export function getCommandAndArgs( if (isHelpArgs(argv)) { return { needHelp: true, - command: null, + error: null, }; } @@ -74,17 +74,22 @@ export function getCommandAndArgs( }; } } + return { + needHelp: true, + error: `Unknown command: ${command}`, + }; } + let error = null; try { commandLineArgs([], { argv: actionArgv }); } catch(err) { - console.log(String(err)); + error = String(err); } return { needHelp: true, - command, + error, }; } From 5f05905e88261c21b22fd4cf0ad675f96a0f9608 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 13:06:42 +0900 Subject: [PATCH 16/20] More v7 readme updates --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7d31a5b..f0eaaaf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # Static Publisher for JavaScript on Fastly Compute +> [!NOTE] +> These docs are for v7, a major rewrite that adds powerful new features such as named collections. +> If you're looking for v6, please check out the [v6 branch](https://github.com/fastly/compute-js-static-publish/tree/v6). + +> [!WARNING] +> Version 7 no longer supports module assets. If you require this feature, consider using [v6](https://github.com/fastly/compute-js-static-publish/tree/v6). + ## 📖 Table of Contents - [✨ Key Features](#-key-features) @@ -252,6 +259,8 @@ This only affects the current request (in Compute, requests do not share state). #### Example: Subdomain-based Routing +In the following example, assume that the Compute application is hosted using a wildcard domain `*.example.com`. A request for `preview-pr-123.example.com` would activate the collection `'pr-123'`. + ```js import { PublisherServer, collectionSelector } from '@fastly/compute-js-static-publish'; import rc from '../static-publish.rc.js'; @@ -260,7 +269,7 @@ const publisherServer = PublisherServer.fromStaticPublishRc(rc); addEventListener("fetch", event => { const request = event.request; - const collectionName = collectionSelector.fromHostDomain(request, /^preview-(.*)\./); + const collectionName = collectionSelector.fromHostDomain(request, /^preview-([^\.]*)\./); if (collectionName != null) { publisherServer.setActiveCollectionName(collectionName); } @@ -274,7 +283,7 @@ addEventListener("fetch", event => { The `collectionSelector` module provides helpers to extract a collection name from different parts of a request: ```js -collectionSelector.fromHostDomain(request, /^preview-(.*)\./); +collectionSelector.fromHostDomain(request, /^preview-([^\.]*)\./); ``` #### From the Request URL From 0845c5d6a529345dd1540b7e1ef16c046ce8a105 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 14:28:51 +0900 Subject: [PATCH 17/20] Changelog --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0228ed8..a583fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] +> [!NOTE] +> Breaking release. Every project will need a re-scaffold. + +### Added + +- Named Collections + - Support for named collections of assets that are managed together and can be individually previewed. + - Expiration trio: --expires-in, --expires-at, --expires-never. + - Collection selector utilities that can be run at the edge code to activate a collection. + - Default collection never expires. + +- git-style subcommands + - Actions separated into subcommands, such as `clean`, `publish-content`, and `collections`. + - Dry-run mode (--dry-run) for commands that mutates KV or disk. + +- KV Store + - Bytes of assets are stored in the Fastly KV Store. + - Items in the KV Store are keyed by the hash of the file, keeping storage efficient and deduplicated, even across collections. + - Metadata (file sizes, encodings, compression) for static assets is stored in KV Store Item metadata. + - Large-object chunking: files > 20 MB are split into segments behind the scenes and reassembled at read time. + - Upload process has been optimized - files are uploaded in parallel, and are only compressed and uploaded when necessary. + - Automatic retry with exponential back-off when the Fastly KV API rate-limits a burst of uploads. + - Fully supported in the local development environment during development. + - `--local` flag for all management commands. Passing this flag makes the command operate on the local KV Store instead of the Fastly KV Store. + +### Changed + +- Separate config files: + - `static-publisher.rc.js` now owns behavior that is common to the scaffolded Compute app + - publish-time settings are in `publish-content.config.js`. + +- Script names in the scaffold are now grouped by environment: `dev:publish`, `dev:start`, `fastly:publish`, `fastly:deploy`. + +- Asset inclusion test renamed to `kvStoreAssetInclusionTest` and now expects a boolean return value. + +### Removed / Deprecated + +- This tool drops `wasm-inline`, and no longer inlines bytes of assets into the Wasm binary. + +- Static files metadata is no longer stored in the Wasm binary, + +- This tool drops support for `moduleAssets`. + ## [6.3.0] - 2025-03-19 ### Added From c27d1dc9f90b0bdcddcecb9015bd7ee0617239c5 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 14:36:08 +0900 Subject: [PATCH 18/20] v7.0.0-beta.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 568a01d..e354e78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.4", + "version": "7.0.0-beta.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fastly/compute-js-static-publish", - "version": "7.0.0-beta.4", + "version": "7.0.0-beta.5", "license": "MIT", "dependencies": { "@fastly/cli": "^11.2.0", diff --git a/package.json b/package.json index 9751da2..d71e839 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@fastly/compute-js-static-publish", "type": "module", - "version": "7.0.0-beta.4", + "version": "7.0.0-beta.5", "description": "Static Publisher for Fastly Compute JavaScript", "main": "build/index.js", "exports": { From 085de0a29464304868eaf22a4dc61237aae59b77 Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 14:46:56 +0900 Subject: [PATCH 19/20] Changelog wording fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a583fc3..326db28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Static files metadata is no longer stored in the Wasm binary, -- This tool drops support for `moduleAssets`. +- This tool drops `moduleAssets`. ## [6.3.0] - 2025-03-19 From 4728a22bd0609f57c70ec9f4d705d491c45505ff Mon Sep 17 00:00:00 2001 From: Katsuyuki Omuro Date: Thu, 24 Apr 2025 15:00:27 +0900 Subject: [PATCH 20/20] Starting point of migration guide --- MIGRATING.md | 260 +-------------------------------------------------- 1 file changed, 3 insertions(+), 257 deletions(-) diff --git a/MIGRATING.md b/MIGRATING.md index 700d342..3b919b3 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -5,260 +5,6 @@ are generated during scaffolding. For this reason, it is recommended that you re This is straightforward if you're using `compute-js-static-publisher` out-of-the-box. Otherwise, read on. -## KV Store - -KV Store is no longer optional as of v7 - -Starting with `v5.0.0`, this tool refers to the KV Store using its finalized product name, "KV Store". References in -code and in configuration that used the previous "Object Store" name have been changed to the new name. If you have -been using the feature, you can take the following steps: - -* In your `static-publish.rc.js` file: - * Rename the `objectStore` key to `kvStoreName`. For example, if your KV Store is named `'my-store'`, then change: - ``` - objectStore: 'my-store' - ``` - to - ``` - kvStoreName: 'my-store' - ``` - -* In your `fastly.toml` file, find all lines that pertain to object store entries. - * Such lines may look like this and there may be many: - ```toml - [[local_server.object_store.my-store]] - key = "4QEM5nIpyLiNF3wwsbnda5:/404.html_aeed29478691e636b4ded20c17e9eae437614617067a8751882368b965a21bcb" - path = "output/404.html" - ``` - * Change the `object_store` of these lines to `kv_stores`. For example: - ```toml - [[local_server.kv_stores.my-store]] - key = "4QEM5nIpyLiNF3wwsbnda5:/404.html_aeed29478691e636b4ded20c17e9eae437614617067a8751882368b965a21bcb" - path = "output/404.html" - ``` - -## Webpack - -Webpack is no longer enabled for new projects - -Starting with `v4.0.0` of this tool, webpack is no longer required and is disabled by default for new applications. This can simplify development and result in shorter build times. - -You may still wish to use webpack if you need some of the features it provides, e.g., the ability to use loaders, asset modules, module replacement, dynamic imports, etc. - -To migrate away from using webpack, make the following changes in your `./compute-js` directory: - -* First, check your `webpack.config.js` file to make sure you aren't actually depending on any custom webpack features. When you're ready, continue to the next step. -* Delete `webpack.config.js`. -* Modify `static-publish.rc.js`: - * Change the line `module.exports = {` to `const config = {` - * At the end of the file, add `export default config;` -* In your `package.json` file: - * At the top level, add a `"type"` key if one doesn't already exist, with the value `"module"`. - * Under `devDependencies`, remove the `webpack` and `webpack-cli` entries. - * Under `scripts`, modify the `prebuild` script by removing the `&& webpack` at the end - of it. - * Under `scripts`, modify the `build` script by replacing the parameter `./bin/index.js` - with `./src/index.js`. - * In the end, the two scripts should look like this (along with any other scripts you may have): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static", - "build": "js-compute-runtime ./src/index.js ./bin/main.wasm" - } - ``` - -If you aren't moving away from webpack just yet, check that your `webpack.config.js` is up-to-date. Refer -to the [default `webpack.config.js` in this package](./resources/webpack.config.js) and add your changes, -or modify your configuration file using the following steps: - -* To make the resulting bundle easier to debug, it is recommended to set the `devtool` value to `false`. - -* The JavaScript SDK automatically adds the named condition `fastly` when resolving dependency packages. - To match the behavior when bundling with webpack, set `resolve.conditionsNames` to the following: - ``` - resolve: { - conditionNames: [ - 'fastly', - '...', - ], - ], - ``` - -* Starting `v3.0.0`, we depend on `v1.0.0` of the `js-compute` library, which provides namespaced exports for Fastly - features. To use them, you'll need to make the following changes to `webpack.config.js`: - - * Set the `target` value to `false`. - - * The `output` section should look like this: - ``` - output: { - filename: "index.js", - path: path.resolve(__dirname, "bin"), - chunkFormat: 'commonjs', - library: { - type: 'commonjs', - }, - }, - ``` - - * Add a new `externals` array to the bottom if it doesn't exist already. Add the following entry: - - ```javascript - module.exports = { - /* ... other config ... */ - externals: [ - /^fastly:.*$/, - ], - } - ``` - -* Starting `v3.0.0`, we no longer use webpack static assets to include the contents of static files, and instead [use the - `includeBytes` function](https://js-compute-reference-docs.edgecompute.app/docs/fastly:experimental/includeBytes) - to enable more performant loading, as well as a more size-efficient Wasm binary. As a result, the following code can - safely be removed from the `module.rules` array. - - ```javascript - { - // asset/source exports the source code of the asset. - resourceQuery: /staticText/, - type: "asset/source", - }, - { - // asset/inline exports the raw bytes of the asset. - // We base64 encode them here - resourceQuery: /staticBinary/, - type: "asset/inline", - generator: { - /** - * @param {Buffer} content - * @returns {string} - */ - dataUrl: content => { - return content.toString('base64'); - }, - } - }, - ``` - -If you need webpack for a new project you are scaffolding with this site, specify the `--webpack` command-line option -when you scaffold your application, e.g.: - -``` -npx @fastly/compute-js-static-publish@latest --webpack --root-dir=./public -``` - -## Removal of Expressly - -Previous versions of `@fastly/compute-js-static-publish` used [Expressly](https://expressly.edgecompute.app) to serve -assets. `v4` does away with this dependency and implements its own server in the `PublisherServer` -class. - -When using `v4`, you can remove the dependency on Expressly by deleting the `@fastly/expressly` entry from -`dependencies` or `devDependencies`, in your `package.json` file. - -If your application depended on Expressly for things like middleware, you will need to make further -changes. - -### The entry point `src/index.js` - -As of `v4`, the `src/index.js` entry point no longer uses Expressly, and looks like this: - -```js -/// -import { getServer } from './statics.js'; -const staticContentServer = getServer(); - -// eslint-disable-next-line no-restricted-globals -addEventListener("fetch", (event) => event.respondWith(handleRequest(event))); -async function handleRequest(event) { - - const response = await staticContentServer.serveRequest(event.request); - if (response != null) { - return response; - } - - // Do custom things here! - // Handle API requests, serve non-static responses, etc. - - return new Response('Not found', { status: 404 }); -} -``` - -If you've previously made changes to `src/index.js`, you will need to make the equivalent changes in this new format. - -## `static-publish.rc.js` - -This configuration file has changed in v4, and you may find that some features have stopped working after -upgrading from v3. - -* In v3, the configuration object was typed `Config`. In v4, it is now typed with a more descriptive name, `StaticPublisherConfig`. - -```js -/** @type {import('@fastly/compute-js-static-publish').StaticPublisherConfig} */ -const config = { - rootDir: './public', - // ... and so on -}; -``` - -* A new key, `server`, was added to group configurations that pertain to Publisher Server. - -To migrate this file, you'll need to make the following changes: - -* `publicDir` - rename this to `rootDir`. All files under this root directory will be included by default in the publishing, - except for those that are excluded using some of the following features. -* `excludeDirs`, `includeDirs`, `excludeTest`, `moduleTest` - In v3, these were used in combination to determine whether - each file would be included in the publishing, and whether files would be included as modules. The interaction between - these four tests was not clearly defined, often having one option exclude files, only to have other options add them - back. In addition, in v3 it was not possible to have a module asset that was not also already a content asset. - In v4, these are more clearly defined. These four options should be rewritten in terms of - `excludeDirs`, `excludeDotFiles`, `includeWellKnown`, `contentAssetInclusionTest`, and `moduleAssetInclusionTest`. -* `staticDirs` - in v4, this was renamed to `staticItems` and moved under the new `server` key. -* `spa` - in v4, this was renamed to `spaFile` and moved under the new `server` key. -* `notFoundPage` - in v4, this was renamed to `notFoundPageFile` and moved under the new `server` key. -* `autoExt` - in v4, this was moved under the new `server` key. -* `autoIndex` - in v4, this was moved under the new `server` key. -* `contentTypes` - This is unchanged. - -See [static-publish.rc.js config file](./README.md#static-publish-rc) for a detailed explanation of each of these new values. - -* `.gitignore` - - Depending on the version of `compute-js-static-publisher` used to scaffold your application, your `.gitignore` file - may have been generated with different entries. Add any of the following entries that may be missing from your - `.gitignore` file: - - ```gitignore - /src/statics.js - /src/statics.d.ts - /src/statics-metadata.js - /src/statics-metadata.d.ts - /src/static-content - ``` - -* Build scripts - * Various versions of `@fastly/compute-js-static-publish` have specified different build scripts. We recommend the following setup, regardless of the version of `@fastly/compute-js-static-publish` or Fastly CLI. - - * The build script listed in `fastly.toml` of your `compute-js` directory should look like this: - ```toml - [scripts] - build = "npm run build" - ``` - - * If you're using webpack, then the `scripts` section of `package.json` of your `compute-js` directory should contain - the following items (along with any other scripts): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static && webpack", - "build": "js-compute-runtime ./bin/index.js ./bin/main.wasm" - } - ``` - - * If you're not using webpack, then the `scripts` section of `package.json` of your `compute-js` directory should - contain the following items (along with any other scripts): - ```json - { - "prebuild": "npx @fastly/compute-js-static-publish --build-static", - "build": "js-compute-runtime ./src/index.js ./bin/main.wasm" - } - ``` +> [!NOTE] +> This document is under construction. +>