From d4ea14c032bb5a5e48b8399ff82e611a602e31a2 Mon Sep 17 00:00:00 2001
From: Steven Silvester <stslve@amazon.com>
Date: Sat, 2 May 2020 05:28:47 -0500
Subject: [PATCH] Add link caching

request the cache version of pytest-check-links

clean up packages

Revert "clean up packages"

This reverts commit dc193e77abfacecfa99e33131597fe278a26bea1.

Fix ci script

build api docs after link checks

cache action must target directory

break link check into two build and add hashing

Ensure the cache dir exists

Install docs requirements

consolidate jobs again

break changelog into its own job

cleanup cache paths and increase test timeout

fix timeout-minutes specifier

try changing cache name and adding restore-keys

try changing cache name and adding restore-keys

Clean up cache handling

Use matrix.group

bump version requirement

We weren't actually using a cache...

Print the contents of the cache dir

change the cache name

consolidate to one build

remove changelog build

update restore-keys
---
 .github/workflows/linuxtests.yml | 20 ++++++++++++++------
 docs/requirements.txt            |  2 +-
 scripts/ci_script.sh             | 30 ++++++++++++------------------
 3 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/linuxtests.yml b/.github/workflows/linuxtests.yml
index cd3594ac80ad..ed165cbc0dab 100644
--- a/.github/workflows/linuxtests.yml
+++ b/.github/workflows/linuxtests.yml
@@ -7,7 +7,7 @@ jobs:
     name: Linux
     strategy:
       matrix:
-        group: [integrity, integrity2, docs, usage, docs2, python,  nonode]
+        group: [integrity, integrity2, docs, usage, python,  nonode]
         python: [3.5, 3.8]
         exclude:
           - group: integrity
@@ -16,11 +16,8 @@ jobs:
             python: 3.5
           - group: docs
             python: 3.5
-          - group: docs2
-            python: 3.5
-          - group: changelog
-            python: 3.5
       fail-fast: false
+    timeout-minutes: 90
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -28,10 +25,12 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python }}
+
       - name: Set up Node
         uses: actions/setup-node@v1
         with:
           node-version: '12.x'
+
       - name: Cache pip on Linux
         uses: actions/cache@v1
         if: startsWith(runner.os, 'Linux')
@@ -41,7 +40,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-${{ matrix.python }}
 
-      # Cache yarn
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
@@ -54,11 +52,21 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
+      - name: Cache checked links build
+        uses: actions/cache@v1
+        if: ${{ matrix.group == 'docs' }}
+        with:
+          path: ~/.cache/pytest-link-check
+          key: ${{ runner.os }}-link-check-${{ hashFiles('**/*.rst') }}
+          restore-keys: |
+            ${{ runner.os }}-link-check-
+
       - name: Install dependencies
         env:
           GROUP: ${{ matrix.group }}
         run: |
           bash ./scripts/ci_install.sh
+
       - name: Run test ${{ matrix.group }}
         env:
           GROUP: ${{ matrix.group }}
diff --git a/docs/requirements.txt b/docs/requirements.txt
index df76c6091e7a..680705227b35 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -3,5 +3,5 @@ sphinx-copybutton
 sphinx_rtd_theme
 recommonmark
 pytest
-pytest-check-links
+pytest-check-links[cache]>=0.4.1
 jsx-lexer
diff --git a/scripts/ci_script.sh b/scripts/ci_script.sh
index 39f5282ed637..08890155c1d7 100644
--- a/scripts/ci_script.sh
+++ b/scripts/ci_script.sh
@@ -37,34 +37,28 @@ fi
 
 
 if [[ $GROUP == docs ]]; then
-    # Verify tutorial docs build
+    # Build the tutorial docs
     pushd docs
     pip install -r ./requirements.txt
     make html
-
-    # Remove internal sphinx files and use pytest-check-links on the generated html
-    rm build/html/genindex.html
-    rm build/html/search.html
-
-    # Changelog has a lot of links and is covered in a separate job.
-    changelog_html=./build/html/getting_started/changelog.html
-    py.test --check-links --links-ext .html -k .html --ignore $changelog_html build/html || py.test --check-links --links-ext .html -k .html --ignore $changelog_html --lf build/html
-
     popd
-fi
 
-
-if [[ $GROUP == docs2 ]]; then
-    # Run the link check on md files - allow for a link to fail once (--lf means only run last failed)
-    py.test --check-links --links-ext .md -k .md . || py.test --check-links --links-ext .md -k .md --lf .
+    # Run the link check on the built html files
+    CACHE_DIR="${HOME}/.cache/pytest-link-check"
+    mkdir -p ${CACHE_DIR}
+    echo "Existing cache:"
+    ls -ltr ${CACHE_DIR}
+    args="--check-links --check-links-cache --check-links-cache-expire-after 86400 --check-links-cache-name ${CACHE_DIR}/cache"
+    args="--ignore docs/build/html/genindex.html --ignore docs/build/html/search.html ${args}"
+    py.test $args --links-ext .html -k .html docs/build/html || py.test $args --links-ext .html -k .html --lf docs/build/html
 
     # Build the API docs
     jlpm build:packages
     jlpm docs
 
-    # Run the link check on the changelog - allow for a link to fail once (--lf means only run last failed)
-    changelog=./docs/source/getting_started/changelog.rst
-    py.test --check-links $changelog || py.test --check-links --lf $changelog
+    # Run the link check on md files - allow for a link to fail once (--lf means only run last failed)
+    args="--check-links --check-links-cache --check-links-cache-expire-after 86400 --check-links-cache-name ${CACHE_DIR}/cache"
+    py.test $args --links-ext .md -k .md . || py.test $args --links-ext .md -k .md --lf .
 fi