Merge pull request actions#448 from actions/users/aiyan/cache-package

Initial commit to create @actions/cache package

Author: aiqiaoy

.github/workflows/artifact-tests.yml

@@ -72,8 +72,8 @@ jobs:
     - name: Verify downloadArtifact()
       shell: bash
       run: |
-        scripts/test-artifact-file.sh "artifact-1-directory/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
-        scripts/test-artifact-file.sh "artifact-2-directory/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
+        packages/artifact/__tests__/test-artifact-file.sh "artifact-1-directory/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
+        packages/artifact/__tests__/test-artifact-file.sh "artifact-2-directory/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
 
     - name: Download artifacts using downloadAllArtifacts()
       run: |
@@ -83,5 +83,5 @@ jobs:
     - name: Verify downloadAllArtifacts()
       shell: bash
       run: |
-        scripts/test-artifact-file.sh "multi-artifact-directory/my-artifact-1/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
-        scripts/test-artifact-file.sh "multi-artifact-directory/my-artifact-2/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"
+        packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-1/artifact-path/world.txt" "${{ env.non-gzip-artifact-content }}"
+        packages/artifact/__tests__/test-artifact-file.sh "multi-artifact-directory/my-artifact-2/artifact-path/gzip.txt" "${{ env.gzip-artifact-content }}"

.github/workflows/cache-tests.yml

@@ -0,0 +1,69 @@
+name: cache-unit-tests
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+
+jobs:
+  build:
+    name: Build
+
+    strategy:
+      matrix:
+        runs-on: [ubuntu-latest, windows-latest, macOS-latest]
+      fail-fast: false
+
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set Node.js 12.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12.x
+
+    # In order to save & restore cache from a shell script, certain env variables need to be set that are only available in the
+    # node context. This runs a local action that gets and sets the necessary env variables that are needed
+    - name: Set env variables
+      uses: ./packages/cache/__tests__/__fixtures__/
+
+    # Need root node_modules because certain npm packages like jest are configured for the entire repository and it won't be possible
+    # without these to just compile the cache package
+    - name: Install root npm packages
+      run: npm ci
+
+    - name: Compile cache package
+      run: |
+        npm ci
+        npm run tsc
+      working-directory: packages/cache
+    
+    - name: Generate files in working directory
+      shell: bash
+      run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} test-cache
+
+    - name: Generate files outside working directory
+      shell: bash
+      run: packages/cache/__tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache
+
+    # We're using node -e to call the functions directly available in the @actions/cache package
+    - name: Save cache using saveCache()
+      run: |
+        node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
+
+    - name: Restore cache using restoreCache()
+      run: |
+        node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))"
+
+    - name: Verify cache 
+      shell: bash
+      run: |
+        packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} test-cache
+        packages/cache/__tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache

README.md

@@ -59,6 +59,8 @@ $ npm install @actions/io
 
 Provides functions for downloading and caching tools.  e.g. setup-* actions. Read more [here](packages/tool-cache)
 
+See @actions/cache for caching workflow dependencies. 
+
 ```bash
 $ npm install @actions/tool-cache
 ```
@@ -82,6 +84,15 @@ $ npm install @actions/artifact
 ```
 <br/>
 
+:dart: [@actions/cache](packages/cache) 
+
+Provides functions to cache dependencies and build outputs to improve workflow execution time. Read more [here](packages/cache)
+
+```bash
+$ npm install @actions/cache
+```
+<br/>
+
 ## Creating an Action with the Toolkit
 
 :question: [Choosing an action type](docs/action-types.md)

scripts/test-artifact-file.sh renamed to packages/artifact/__tests__/test-artifact-file.sh

@@ -0,0 +1,41 @@
+# `@actions/cache`
+
+> Functions necessary for caching dependencies and build outputs to improve workflow execution time.
+
+See ["Caching dependencies to speed up workflows"](https://help.github.com/github/automating-your-workflow-with-github-actions/caching-dependencies-to-speed-up-workflows) for how caching works.
+
+Note that GitHub will remove any cache entries that have not been accessed in over 7 days. There is no limit on the number of caches you can store, but the total size of all caches in a repository is limited to 5 GB. If you exceed this limit, GitHub will save your cache but will begin evicting caches until the total size is less than 5 GB.
+
+## Usage
+
+#### Restore Cache
+
+Restores a cache based on `key` and `restoreKeys` to the `paths` provided. Function returns the cache key for cache hit and returns undefined if cache not found. 
+
+```js
+const cache = require('@actions/cache');
+const paths = [
+    'node_modules',
+    'packages/*/node_modules/'
+]
+const key = 'npm-foobar-d5ea0750'
+const restoreKeys = [
+    'npm-foobar-',
+    'npm-'
+]
+const cacheKey = await cache.restoreCache(paths, key, restoreKeys)
+```
+
+#### Save Cache
+
+Saves a cache containing the files in `paths` using the `key` provided. The files would be compressed using zstandard compression algorithm if zstd is installed, otherwise gzip is used. Function returns the cache id if the cache was saved succesfully and throws an error if cache upload fails. 
+
+```js
+const cache = require('@actions/cache');
+const paths = [
+    'node_modules',
+    'packages/*/node_modules/'
+]
+const key = 'npm-foobar-d5ea0750'
+const cacheId = await cache.saveCache(paths, key)
+```

packages/cache/README.md

@@ -0,0 +1,5 @@
+# @actions/cache Releases
+
+### 0.1.0
+
+- Initial release

packages/cache/RELEASES.md

@@ -0,0 +1,5 @@
+name: 'Set env variables'
+description: 'Sets certain env variables so that e2e restore and save cache can be tested in a shell'
+runs:
+  using: 'node12'
+  main: 'index.js'

packages/cache/__tests__/__fixtures__/action.yml

@@ -0,0 +1 @@
+hello world

packages/cache/__tests__/__fixtures__/helloWorld.txt

@@ -0,0 +1,5 @@
+// Certain env variables are not set by default in a shell context and are only available in a node context from a running action
+// In order to be able to restore and save cache e2e in a shell when running CI tests, we need these env variables set
+console.log(`::set-env name=ACTIONS_RUNTIME_URL::${process.env.ACTIONS_RUNTIME_URL}`)
+console.log(`::set-env name=ACTIONS_RUNTIME_TOKEN::${process.env.ACTIONS_RUNTIME_TOKEN}`)
+console.log(`::set-env name=GITHUB_RUN_ID::${process.env.GITHUB_RUN_ID}`)

packages/cache/__tests__/__fixtures__/index.js

@@ -0,0 +1,175 @@
+import {getCacheVersion, retry} from '../src/internal/cacheHttpClient'
+import {CompressionMethod} from '../src/internal/constants'
+
+test('getCacheVersion with one path returns version', async () => {
+  const paths = ['node_modules']
+  const result = getCacheVersion(paths)
+  expect(result).toEqual(
+    'b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985'
+  )
+})
+
+test('getCacheVersion with multiple paths returns version', async () => {
+  const paths = ['node_modules', 'dist']
+  const result = getCacheVersion(paths)
+  expect(result).toEqual(
+    '165c3053bc646bf0d4fac17b1f5731caca6fe38e0e464715c0c3c6b6318bf436'
+  )
+})
+
+test('getCacheVersion with zstd compression returns version', async () => {
+  const paths = ['node_modules']
+  const result = getCacheVersion(paths, CompressionMethod.Zstd)
+
+  expect(result).toEqual(
+    '273877e14fd65d270b87a198edbfa2db5a43de567c9a548d2a2505b408befe24'
+  )
+})
+
+test('getCacheVersion with gzip compression does not change vesion', async () => {
+  const paths = ['node_modules']
+  const result = getCacheVersion(paths, CompressionMethod.Gzip)
+
+  expect(result).toEqual(
+    'b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985'
+  )
+})
+
+interface TestResponse {
+  statusCode: number
+  result: string | null
+}
+
+async function handleResponse(
+  response: TestResponse | undefined
+): Promise<TestResponse> {
+  if (!response) {
+    // eslint-disable-next-line no-undef
+    fail('Retry method called too many times')
+  }
+
+  if (response.statusCode === 999) {
+    throw Error('Test Error')
+  } else {
+    return Promise.resolve(response)
+  }
+}
+
+async function testRetryExpectingResult(
+  responses: TestResponse[],
+  expectedResult: string | null
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  const actualResult = await retry(
+    'test',
+    async () => handleResponse(responses.pop()),
+    (response: TestResponse) => response.statusCode
+  )
+
+  expect(actualResult.result).toEqual(expectedResult)
+}
+
+async function testRetryExpectingError(
+  responses: TestResponse[]
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  expect(
+    retry(
+      'test',
+      async () => handleResponse(responses.pop()),
+      (response: TestResponse) => response.statusCode
+    )
+  ).rejects.toBeInstanceOf(Error)
+}
+
+test('retry works on successful response', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry works after retryable status code', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 503,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry fails after exhausting retries', async () => {
+  await testRetryExpectingError([
+    {
+      statusCode: 503,
+      result: null
+    },
+    {
+      statusCode: 503,
+      result: null
+    },
+    {
+      statusCode: 200,
+      result: 'Ok'
+    }
+  ])
+})
+
+test('retry fails after non-retryable status code', async () => {
+  await testRetryExpectingError([
+    {
+      statusCode: 500,
+      result: null
+    },
+    {
+      statusCode: 200,
+      result: 'Ok'
+    }
+  ])
+})
+
+test('retry works after error', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 999,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry returns after client error', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 400,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    null
+  )
+})

packages/cache/__tests__/cacheHttpClient.test.ts

@@ -0,0 +1,26 @@
+import {promises as fs} from 'fs'
+import * as path from 'path'
+import * as cacheUtils from '../src/internal/cacheUtils'
+
+test('getArchiveFileSizeIsBytes returns file size', () => {
+  const filePath = path.join(__dirname, '__fixtures__', 'helloWorld.txt')
+
+  const size = cacheUtils.getArchiveFileSizeIsBytes(filePath)
+
+  expect(size).toBe(11)
+})
+
+test('unlinkFile unlinks file', async () => {
+  const testDirectory = await fs.mkdtemp('unlinkFileTest')
+  const testFile = path.join(testDirectory, 'test.txt')
+  await fs.writeFile(testFile, 'hello world')
+
+  await expect(fs.stat(testFile)).resolves.not.toThrow()
+
+  await cacheUtils.unlinkFile(testFile)
+
+  // This should throw as testFile should not exist
+  await expect(fs.stat(testFile)).rejects.toThrow()
+
+  await fs.rmdir(testDirectory)
+})