fix(aws-arch): pricing manifest caching and automation

To support deterministic caching of aws-arch genoration of
aws pricing, we need to store the manifest locally since.

Creating github workflow to support weekly updates along with
snapshot updates and build verification - currently requires
manually approval.

.github/workflows/aws-arch-pricing.yml

@@ -0,0 +1,84 @@
+name: aws-arch-pricing
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: 0 0 * * 1
+jobs:
+  update:
+    name: Update Pricing
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      patch_created: ${{ steps.create_patch.outputs.patch_created }}
+    env:
+      CI: "false"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: mainline
+      - name: Install dependencies
+        run: yarn install --check-files --frozen-lockfile
+      - name: Fetch Pricing Manifest
+        working-directory: packages/aws-arch
+        run: npx projen fetch-pricing-manifest
+      - name: Build
+        working-directory: packages/aws-arch
+        run: npx nx run build
+      - id: create_patch
+        name: Find mutations
+        run: |-
+          git add .
+          git diff --staged --patch --exit-code > .repo.patch || echo "::set-output name=patch_created::true"
+      - if: steps.create_patch.outputs.patch_created
+        name: Upload patch
+        uses: actions/upload-artifact@v2
+        with:
+          name: .repo.patch
+          path: .repo.patch
+  pr:
+    name: Create Pull Request
+    needs: update
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    if: ${{ needs.upgrade.outputs.patch_created }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PROJEN_GITHUB_TOKEN }}
+          ref: mainline
+      - name: Download patch
+        uses: actions/download-artifact@v2
+        with:
+          name: .repo.patch
+          path: ${{ runner.temp }}
+      - name: Apply patch
+        run: '[ -s ${{ runner.temp }}/.repo.patch ] && git apply ${{ runner.temp }}/.repo.patch || echo "Empty patch. Skipping."'
+      - name: Set git identity
+        run: |-
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+      - name: Create Pull Request
+        id: create-pr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.PROJEN_GITHUB_TOKEN }}
+          commit-message: |-
+            chore(aws-arch): update pricing manifest
+
+            Update aws pricing manifest and snapshots for aws-arch package. See details in [workflow run].
+
+            [Workflow Run]: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          branch: github-actions/aws-arch-pricing
+          title: "chore(aws-arch): update pricing manifest"
+          body: |-
+            Update aws pricing manifest and snapshots for aws-arch package. See details in [workflow run].
+
+            [Workflow Run]: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          author: github-actions <github-actions@github.com>
+          committer: github-actions <github-actions@github.com>
+          signoff: false

packages/aws-arch/ASSETS.md

@@ -223,6 +223,7 @@ The _default theme_ is **light**.
 |  └── step_functions/
 |     ├── service_icon.png
 |     └── service_icon.svg
+├── aws-pricing-manifest.json
 ├── blockchain/
 |  ├── category_icon.png
 |  ├── category_icon.svg
@@ -2882,7 +2883,7 @@ The _default theme_ is **light**.
       ├── service_icon.png
       └── service_icon.svg
 
-directory: 329 file: 2529
+directory: 329 file: 2530
 
 
 ```

packages/aws-arch/package.json

@@ -15,6 +15,7 @@
     "eject": "npx projen eject",
     "eslint": "npx projen eslint",
     "eslint-staged": "npx projen eslint-staged",
+    "fetch-pricing-manifest": "npx projen fetch-pricing-manifest",
     "generate": "npx projen generate",
     "generate:assets": "npx projen generate:assets",
     "generate:cfnspec-types": "npx projen generate:cfnspec-types",

packages/aws-arch/scripts/fetch-pricing-manifest.ts

@@ -0,0 +1,36 @@
+#!/usr/bin/env ts-node
+/*********************************************************************************************************************
+ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License").
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ ******************************************************************************************************************** */
+import * as path from 'path';
+import * as util from 'util';
+import * as stream from 'stream';
+import * as fs from 'fs-extra';
+import fetch from 'node-fetch';
+
+const URL = 'https://d1qsjq9pzbk1k6.cloudfront.net/manifest/en_US.json';
+
+const FILEPATH = path.join(__dirname, '..', 'static', 'aws-pricing-manifest.json');
+
+(async () => {
+  console.info('Fetching pricing manifest...');
+  const response = await fetch(URL);
+  if (!response.ok) {
+    console.debug(response);
+    throw new Error(`Failed to download pricing manifest: ${response.statusText} - ${response.statusText}`);
+  }
+  await util.promisify(stream.pipeline)(response.body, fs.createWriteStream(FILEPATH));
+  console.info('Done - pricing manifest fetched')
+})();

packages/aws-arch/scripts/generate-pricing-manifest.ts

@@ -14,29 +14,18 @@
  See the License for the specific language governing permissions and
  limitations under the License.
  ******************************************************************************************************************** */
-import * as path from 'path';
-import * as util from 'util';
-import * as stream from 'stream';
+import * as path from 'node:path';
 import * as fs from 'fs-extra';
-import fetch from 'node-fetch';
-import { TMP_DIR, GENERATED_DIR, logCount } from './common';
+import { GENERATED_DIR, logCount } from './common';
 import { normalizeComparisonString, parseAwsUrl } from "../src/utils";
 import { sortedUniq } from "lodash";
 
-const PRICING_MANIFEST_URL = 'https://d1qsjq9pzbk1k6.cloudfront.net/manifest/en_US.json';
+const PRICING_MANIFEST_JSON = path.join(__dirname, '..', 'static', 'aws-pricing-manifest.json');
 
 const PRICING_MANIFEST_TS = path.join(GENERATED_DIR, 'pricing-manifest.ts');
 
 /** Generates `generated/pricing-manifest.ts` file. */
 export async function generate () {
-  const manifestJsonPath = path.join(TMP_DIR, 'pricing-manifest.json');
-
-  if (!await fs.pathExists(manifestJsonPath)) {
-    const response = await fetch(PRICING_MANIFEST_URL);
-    if (!response.ok) throw new Error(`unexpected response ${response.statusText}`)
-    await util.promisify(stream.pipeline)(response.body, fs.createWriteStream(manifestJsonPath))
-  }
-
   const slugs = new Set<string>();
   const missingSlugs = new Set<string>();
   const missingLinkUrl = new Set<string>();
@@ -50,7 +39,7 @@ export async function generate () {
     set.add(value);
   }
 
-  const manifest = (require(manifestJsonPath).awsServices as any[]).reduce((dict, service) => {
+  const manifest = (require(PRICING_MANIFEST_JSON).awsServices as any[]).reduce((dict, service) => {
     service = transformServiceForJsii(service);
     const serviceCode: string = service.serviceCode;
     const isSubFor = isSubServiceForDefinition(service);

private/projects/aws-arch.ts

@@ -53,6 +53,10 @@ export class AwsArchProject extends PDKProject {
       "jsii",
     ]);
 
+    this.addTask("fetch-pricing-manifest", {
+      exec: "ts-node ./scripts/fetch-pricing-manifest.ts",
+    });
+
     const generateTasks = [
       this.addTask("clean", {
         exec: "rm -rf assets src/generated",