OpenTTD · TrueBrain · Feb 2, 2020 · Jan 26, 2020
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,8 @@
+[flake8]
+max-line-length = 120
+inline-quotes = double
+
+# We use 'black' for coding style; these next two warnings are not PEP-8
+# compliant.
+# E231 is a bug in 'black', see https://github.com/psf/black/issues/1202
+ignore = E203, E231, W503
diff --git a/.github/workflows/new_openttd_release.yml b/.github/workflows/new_openttd_release.yml
@@ -7,31 +7,42 @@ on:
     - new_master
     - new_pullrequest
 
+# client_payload should contain:
+#  - version: the version of the new release, e.g.: 20200101-master-g12345
+#  - folder: the folder in which the release is located openttd-nightlies/2020
+
 jobs:
   new_release:
     name: New OpenTTD release
     runs-on: ubuntu-latest
 
     steps:
-    - name: Trigger website (staging)
-      uses: peter-evans/repository-dispatch@v1
-      with:
-        token: ${{ secrets.DEPLOYMENT_TOKEN }}
-        repository: OpenTTD/website
-        event-type: publish_master
+    - name: Sanity check
+      run: |
+        set -x
+
+        if [ -z "${{ github.event.client_payload.version }}" ]; then
+          echo "::error::version in client_payload is not set"
+          exit 1
+        fi
+        if [ -z "${{ github.event.client_payload.folder }}" ]; then
+          echo "::error::folder in client_payload is not set"
+          exit 1
+        fi
 
-    - name: Trigger website (Production)
+    - name: Trigger 'update CDN'
       uses: peter-evans/repository-dispatch@v1
       with:
         token: ${{ secrets.DEPLOYMENT_TOKEN }}
-        repository: OpenTTD/website
-        event-type: publish_latest_tag
+        repository: OpenTTD/workflows
+        event-type: update_cdn
+        client-payload: '{"version": "${{ github.event.client_payload.version }}", "folder": "${{ github.event.client_payload.folder }}"}'
 
     - if: github.event.action == 'new_master'
-      name: Trigger docs
+      name: Trigger 'publish docs'
       uses: peter-evans/repository-dispatch@v1
       with:
         token: ${{ secrets.DEPLOYMENT_TOKEN }}
         repository: OpenTTD/workflows
         event-type: publish_docs
-        client-payload: '{"version": "${{ github.event.client_payload.version }}"}'
+        client-payload: '{"version": "${{ github.event.client_payload.version }}", "folder": "${{ github.event.client_payload.folder }}"}'
diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml
@@ -5,59 +5,72 @@ on:
     types:
     - publish_docs
 
+# client_payload should contain:
+#  - version: the version of the new release, e.g.: 20200101-master-g12345
+#  - folder: the folder in which the release is located openttd-nightlies/2020
+
 jobs:
   publish_docs:
     name: Publish docs
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Publish docs
+    - name: Sanity check
       run: |
         set -x
 
-        VERSION=${{ github.event.client_payload.version }}
-
-        if [ -z "${VERSION}" ]; then
+        if [ -z "${{ github.event.client_payload.version }}" ]; then
           echo "::error::version in client_payload is not set"
           exit 1
         fi
-        if [ -z "${{ secrets.S3_BUCKET }}" ]; then
-          echo "::error::secret S3_BUCKET is not set"
+        if [ -z "${{ github.event.client_payload.folder }}" ]; then
+          echo "::error::folder in client_payload is not set"
+          exit 1
+        fi
+        if [ -z "${{ secrets.DOCS_S3_BUCKET }}" ]; then
+          echo "::error::secret DOCS_S3_BUCKET is not set"
           exit 1
         fi
-        if [ -z "${{ secrets.CF_DISTRIBUTION_ID }}" ]; then
-          echo "::error::secret CF_DISTRIBUTION_ID is not set"
+        if [ -z "${{ secrets.DOCS_CF_DISTRIBUTION_ID }}" ]; then
+          echo "::error::secret DOCS_CF_DISTRIBUTION_ID is not set"
           exit 1
         fi
 
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Publish docs
+      run: |
+        set -x
+
+        VERSION=${{ github.event.client_payload.version }}
+        FOLDER=${{ github.event.client_payload.folder }}
+
         # Fetch and extract the docs
-        curl --fail -s -L -o docs-ai.tar.xz https://proxy.binaries.openttd.org/openttd-nightlies/${VERSION}/openttd-${VERSION}-docs-ai.tar.xz
-        curl --fail -s -L -o docs-gs.tar.xz https://proxy.binaries.openttd.org/openttd-nightlies/${VERSION}/openttd-${VERSION}-docs-gs.tar.xz
-        curl --fail -s -L -o docs.tar.xz https://proxy.binaries.openttd.org/openttd-nightlies/${VERSION}/openttd-${VERSION}-docs.tar.xz
+        curl --fail -s -L -o docs-ai.tar.xz https://cdn.openttd.org/${FOLDER}/${VERSION}/openttd-${VERSION}-docs-ai.tar.xz
+        curl --fail -s -L -o docs-gs.tar.xz https://cdn.openttd.org/${FOLDER}/${VERSION}/openttd-${VERSION}-docs-gs.tar.xz
+        curl --fail -s -L -o docs.tar.xz https://cdn.openttd.org/${FOLDER}/${VERSION}/openttd-${VERSION}-docs.tar.xz
         tar xf docs-ai.tar.xz
         tar xf docs-gs.tar.xz
         tar xf docs.tar.xz
 
         # Sync all the new/modified files
-        aws s3 sync --only-show-errors docs/root/ s3://${{ secrets.S3_BUCKET }}/
-        aws s3 sync --only-show-errors docs/errors/ s3://${{ secrets.S3_BUCKET }}/errors/
-        aws s3 sync --only-show-errors openttd-${VERSION}-docs-ai/html/ s3://${{ secrets.S3_BUCKET }}/ai-api/
-        aws s3 sync --only-show-errors openttd-${VERSION}-docs-gs/html/ s3://${{ secrets.S3_BUCKET }}/gs-api/
-        aws s3 sync --only-show-errors openttd-${VERSION}-docs/html/ s3://${{ secrets.S3_BUCKET }}/source/
+        aws s3 sync --only-show-errors docs/root/ s3://${{ secrets.DOCS_S3_BUCKET }}/
+        aws s3 sync --only-show-errors docs/errors/ s3://${{ secrets.DOCS_S3_BUCKET }}/errors/
+        aws s3 sync --only-show-errors openttd-${VERSION}-docs-ai/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/ai-api/
+        aws s3 sync --only-show-errors openttd-${VERSION}-docs-gs/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/gs-api/
+        aws s3 sync --only-show-errors openttd-${VERSION}-docs/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/source/
 
         # Delete any unknown file; this is a separate step, as 'aws s3 sync'
         # can delete before uploading, which could mean files temporary link
         # to invalid files.
-        aws s3 sync --only-show-errors --delete docs/errors/ s3://${{ secrets.S3_BUCKET }}/errors/
-        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs-ai/html/ s3://${{ secrets.S3_BUCKET }}/ai-api/
-        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs-gs/html/ s3://${{ secrets.S3_BUCKET }}/gs-api/
-        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs/html/ s3://${{ secrets.S3_BUCKET }}/source/
+        aws s3 sync --only-show-errors --delete docs/errors/ s3://${{ secrets.DOCS_S3_BUCKET }}/errors/
+        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs-ai/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/ai-api/
+        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs-gs/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/gs-api/
+        aws s3 sync --only-show-errors --delete openttd-${VERSION}-docs/html/ s3://${{ secrets.DOCS_S3_BUCKET }}/source/
 
         # Invalidate the cache of the CloudFront distribution
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.CF_DISTRIBUTION_ID }} --paths "/*"
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.DOCS_CF_DISTRIBUTION_ID }} --paths "/*"
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -0,0 +1,35 @@
+name: Testing
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  flake8:
+    name: Flake8
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Flake8
+      uses: TrueBrain/actions-flake8@master
+      with:
+        path: cdn/cdn_generator
+
+  black:
+    name: Black
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Black
+      run: |
+        python -m pip install --upgrade pip
+        pip install black
+        black -l 120 --check cdn/cdn_generator
diff --git a/.github/workflows/update_cdn.yml b/.github/workflows/update_cdn.yml
@@ -0,0 +1,86 @@
+name: Update CDN
+
+on:
+  repository_dispatch:
+    types:
+    - update_cdn
+
+# client_payload should contain:
+#  - version: the version of the new release, e.g.: 20200101-master-g12345
+#  - folder: the folder in which the release is located openttd-nightlies/2020
+
+jobs:
+  update_cdn:
+    name: Update CDN
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Sanity check
+      run: |
+        set -x
+
+        if [ -z "${{ github.event.client_payload.version }}" ]; then
+          echo "::error::version in client_payload is not set"
+          exit 1
+        fi
+        if [ -z "${{ github.event.client_payload.folder }}" ]; then
+          echo "::error::folder in client_payload is not set"
+          exit 1
+        fi
+        if [ -z "${{ secrets.CDN_S3_BUCKET }}" ]; then
+          echo "::error::secret CDN_S3_BUCKET is not set"
+          exit 1
+        fi
+        if [ -z "${{ secrets.CDN_CF_DISTRIBUTION_ID }}" ]; then
+          echo "::error::secret CDN_CF_DISTRIBUTION_ID is not set"
+          exit 1
+        fi
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r cdn/requirements.txt
+
+    # By default, mime.types doesn't include one for YAML files. This results
+    # in "binary/octet-stream" as content-type on the CDN, which means
+    # browsers want to download the file instead of showing it inline. By
+    # adding this entry to mime.types, we fix that problem.
+    - name: Fix mimetypes
+      run: |
+        echo "text/vnd.yaml  yaml" | sudo tee -a /etc/mime.types
+
+    - name: Publish CDN files
+      run: |
+        set -x
+
+        cd cdn
+
+        python -m cdn_generator --bucket-id ${{ secrets.CDN_S3_BUCKET }} --new-release ${{ github.event.client_payload.folder }}/${{ github.event.client_payload.version }}
+
+        aws s3 cp --recursive --only-show-errors generated/ s3://${{ secrets.CDN_S3_BUCKET }}/
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.CDN_CF_DISTRIBUTION_ID }} --paths "/*"
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: Trigger 'publish website (staging)'
+      uses: peter-evans/repository-dispatch@v1
+      with:
+        token: ${{ secrets.DEPLOYMENT_TOKEN }}
+        repository: OpenTTD/website
+        event-type: publish_master
+
+    - name: Trigger 'publish website (Production)'
+      uses: peter-evans/repository-dispatch@v1
+      with:
+        token: ${{ secrets.DEPLOYMENT_TOKEN }}
+        repository: OpenTTD/website
+        event-type: publish_latest_tag
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+cdn/generated
diff --git a/README.md b/README.md
@@ -7,3 +7,4 @@ This repository runs a few workflows that don't belong to a repository we have.
 
 - [New OpenTTD Release](.github/workflows/new_openttd_release.yml): workflow that is triggered when a new OpenTTD binary is produced
 - [Publish Docs](.github/workflows/publish_docs.yml): publishes the HTML docs to https://docs.openttd.org/ based on an openttd-nightly version.
+- [Update CDN](.github/workflows/update_cdn.yml): updates the [CDN](https://cdn.openttd.org) with required files like index.html, manifest.yaml, etc.
diff --git a/cdn/README.md b/cdn/README.md
@@ -0,0 +1,58 @@
+# Content Delivery Network
+
+OpenTTD publishes all the binaries produced to a CDN.
+This CDN is hosted by AWS CloudFront, which fetches the files from a private AWS S3.
+There are a various amount of files on the CDN to help with automation and human readability.
+
+## index.html
+
+AWS S3 has no ability to do directory listing.
+From experience, OpenTTD has found out that a CDN without being able to browse files, often leads to frustration by its end users.
+As such, part of the `cdn-generator` is to create `index.html` files in every folder.
+It does a few tricks, like sorting by version and like putting the newest version on top, to make it more useful.
+By publishing this `index.html` on the CDN, it feels like there is directory listing enabled.
+It really isn't.
+
+## folders.yaml
+
+Similar to `index.html`, but with only the folders, and in a format that can easily be automated.
+
+## manifest.yaml
+
+Every release has a `manifest.yaml`, which contains all the important parts of the release.
+Automation can use this file to know what file to download, or what size to expect.
+There are also checksum values there to validate a download.
+
+## latest.yaml
+
+For automation, you often want to know: what is the latest release.
+In the root folder and every subfolder there-of, there is a `latest.yaml` indicating exactly this.
+Per category, there are two choices: either it is "releases" or something else.
+In case of the first, there can be a "stable" and "testing" version.
+Otherwise there is always one.
+There are several keys per entry:
+
+- `version`: indicates the latest version.
+- `name`: indicates if it is stable/testing (in case of "releases") or what type this is ("master", "trunk", "nightly", ..).
+- `category`: the category this entry belongs to.
+- `date`: the date when this version was released.
+- `folder`: in which folder thie version can be found, relative to `latest.yaml`.
+
+In the root, all `latest.yaml` from the children are combined together.
+
+## config.yaml
+
+In the root there is a `config.yaml`.
+This tells the `cdn-generator` which folders to index, and how to do this exactly.
+There are several keys per entry:
+
+- `name`: name of the folder to configure.
+- `subfolders`: optional (default: none).
+  - `per-name`: the name of the subfolder is just a name, and the versions are a subfolder of those folders.
+  - `per-year`: the subfolder is split per year.
+- `override-name`: optional (default: use `name`).
+  - `nightly`: force the name to be "nightly".
+  - `in-folder-name`: folder should be named `XXX-NAME-XXX`, and the name is the `NAME` part.
+- `sort`: optional (default: `version`).
+  - `normal`: use "abc" sorting.
+  - `version`: use version sorting: "1.9-alpha1" < "1.9-beta2" < "1.9-RC3" < "1.9" < "1.10".