From b5feb401d6cd7ef3b2cd9d21d44c2987a0fba008 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Thu, 3 Aug 2023 15:28:10 -0700
Subject: [PATCH 1/8] chore: add canary tests

---
 .github/canary-config/canary-all.yml          | 301 ++++++++++++++++++
 .../workflows/callable-canary-e2e-runner.yml  |  41 +++
 .../workflows/callable-canary-e2e-tests.yml   | 135 ++++++++
 .github/workflows/callable-canary-e2e.yml     |  44 +++
 .../callable-canary-sampleapp-tests.yml       | 228 +++++++++++++
 .github/workflows/cron-canary-test.yml        |   9 +
 6 files changed, 758 insertions(+)
 create mode 100644 .github/canary-config/canary-all.yml
 create mode 100644 .github/workflows/callable-canary-e2e-runner.yml
 create mode 100644 .github/workflows/callable-canary-e2e-tests.yml
 create mode 100644 .github/workflows/callable-canary-e2e.yml
 create mode 100644 .github/workflows/callable-canary-sampleapp-tests.yml
 create mode 100644 .github/workflows/cron-canary-test.yml

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
new file mode 100644
index 00000000000..9598fa51bac
--- /dev/null
+++ b/.github/canary-config/canary-all.yml
@@ -0,0 +1,301 @@
+minimal_browser_list: &minimal_browser_list
+  - chrome
+  - firefox
+extended_browser_list: &extended_browser_list
+  - chrome
+  - firefox
+  - edge
+tests:
+  - test_name: 'DataStore Auth - Chrome'
+    framework: react
+    category: datastore
+    sample_name: owner-based-default
+    spec: owner-based-default
+    browser: *minimal_browser_list
+  - test_name: 'React GraphQL API'
+    framework: react
+    category: api
+    sample_name: [graphql]
+    spec: graphql
+    browser: *minimal_browser_list
+  - test_name: 'React Authenticator'
+    framework: react
+    category: auth
+    sample_name: [amplify-authenticator]
+    spec: new-ui-authenticator
+    browser: *minimal_browser_list
+  - test_name: 'React Credentials Different Region'
+    framework: react
+    category: auth
+    sample_name: [credentials-auth]
+    spec: credentials-auth
+    browser: *minimal_browser_list
+  - test_name: 'amazon-cognito-identity-js-cookie-storage'
+    framework: react
+    category: auth
+    sample_name: [amazon-cognito-identity-js-cookie-storage]
+    spec: amazon-cognito-identity-js-cookie-storage
+    browser: *minimal_browser_list
+  - test_name: 'amazon-cognito-identity-js'
+    framework: react
+    category: auth
+    sample_name: [amazon-cognito-identity-js]
+    spec: amazon-cognito-identity-js
+    browser: *minimal_browser_list
+  - test_name: 'cognito-device-tracking'
+    framework: react
+    category: auth
+    sample_name: [device-tracking]
+    spec: device-tracking
+    browser: *minimal_browser_list
+  - test_name: 'delete-user'
+    framework: react
+    category: auth
+    sample_name: [delete-user]
+    spec: delete-user
+    browser: *minimal_browser_list
+  - test_name: 'Angular Authenticator'
+    framework: angular
+    category: auth
+    sample_name: [amplify-authenticator]
+    spec: ui-amplify-authenticator
+    browser: *minimal_browser_list
+  - test_name: 'JavaScript Auth CDN'
+    framework: javascript
+    category: auth
+    sample_name: [auth-cdn]
+    spec: auth-cdn
+    browser: *minimal_browser_list
+    amplifyjs_dir: true
+  - test_name: 'Legacy Vue Authenticator'
+    framework: vue
+    category: auth
+    sample_name: [amplify-authenticator-legacy]
+    spec: authenticator
+    browser: *minimal_browser_list
+  - test_name: 'Authenticator and SSR page'
+    framework: next
+    category: auth
+    sample_name: [auth-ssr]
+    spec: auth-ssr
+    browser: *minimal_browser_list
+  - test_name: 'amazon-cognito-identity-js'
+    framework: node
+    category: auth
+    sample_name: [amazon-cognito-identity-js]
+    build_type: dev
+  - test_name: 'React DataStore'
+    framework: react
+    category: datastore
+    sample_name: [many-to-many]
+    spec: many-to-many
+    browser: *minimal_browser_list
+  - test_name: 'React DataStore Multi-Auth - One Rule'
+    framework: react
+    category: datastore
+    sample_name: [multi-auth]
+    spec: multi-auth-one-rule
+    browser: *minimal_browser_list
+  - test_name: 'React DataStore Multi-Auth - Two Rules'
+    framework: react
+    category: datastore
+    sample_name: [multi-auth]
+    spec: multi-auth-two-rules
+    browser: *minimal_browser_list
+  - test_name: 'React DataStore Multi-Auth - Three Plus Rules'
+    framework: react
+    category: datastore
+    sample_name: [multi-auth]
+    spec: multi-auth-three-plus-rules
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Subs Disabled'
+    framework: react
+    category: datastore
+    sample_name: [subs-disabled]
+    spec: subs-disabled
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Subs Disabled'
+    framework: react
+    category: datastore
+    sample_name: [consecutive-saves]
+    spec: consecutive-saves
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Schema Drift'
+    framework: react
+    category: datastore
+    sample_name: [schema-drift]
+    spec: schema-drift
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Custom Primary Key (Has One)'
+    framework: react
+    category: datastore
+    sample_name: [v2/custom-pk-has-one]
+    spec: custom-pk-has-one
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Custom Primary Key (Has Many / Many to Many)'
+    framework: react
+    category: datastore
+    sample_name: [v2/custom-pk-has-many-many-to-many]
+    spec: custom-pk-has-many-many-to-many
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Background Process Manager'
+    framework: react
+    category: datastore
+    sample_name: [v2/background-process-manager]
+    spec: background-process-manager
+    browser: *extended_browser_list
+  - test_name: 'DataStore - Related Models'
+    framework: react
+    category: datastore
+    sample_name: [related-models]
+    spec: related-models
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Selective Sync'
+    framework: react
+    category: datastore
+    sample_name: [selective-sync-v5]
+    spec: selective-sync-v5
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Nested Predicate'
+    framework: react
+    category: datastore
+    sample_name: [nested-predicate]
+    spec: nested-predicate
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Docs Examples'
+    framework: react
+    category: datastore
+    sample_name: [v2/amplify-docs-examples]
+    spec: amplify-docs-examples
+    browser: *minimal_browser_list
+    timeout_minutes: 45
+    retry_count: 10
+  - test_name: 'DataStore - WebSocket Disruption'
+    framework: react
+    category: datastore
+    sample_name: [websocket-disruption]
+    spec: websocket-disruption
+    browser: *minimal_browser_list
+  - test_name: 'Vanilla JS + Webpack 4 + DataStore - Basic CRUD'
+    framework: javascript
+    category: datastore
+    sample_name: [basic-crud]
+    browser: *minimal_browser_list
+    spec: vanilla-js-basic-crud
+    amplifyjs_dir: true
+    timeout_minutes: 45
+    retry_count: 10
+  - test_name: 'next owner auth'
+    framework: next
+    category: datastore
+    sample_name: [owner-based-default]
+    spec: next-owner-based-default
+    browser: *minimal_browser_list
+  - test_name: 'DataStore - Nextjs 13 build with SWC - JS app'
+    framework: next
+    category: datastore
+    sample_name: [next-13-js]
+    spec: nextjs-13
+    browser: *minimal_browser_list
+  - test_name: 'Vite + DataStore - Basic CRUD'
+    framework: vite
+    category: datastore
+    sample_name: [v2/basic-crud]
+    spec: vite-basic-crud
+  - test_name: 'Rollup + DataStore - Basic CRUD'
+    framework: rollup
+    category: datastore
+    sample_name: [rollup-basic-crud]
+    spec: rollup-basic-crud
+  - test_name: 'Display Map'
+    framework: react
+    category: geo
+    sample_name: [display-map]
+    spec: display-map
+    # Temp fix:
+    browser: [chrome]
+  - test_name: 'Search Outside Map'
+    framework: react
+    category: geo
+    sample_name: [search-outside-map]
+    spec: search-outside-map
+    # Temp fix:
+    browser: [chrome]
+  - test_name: 'React Interactions'
+    framework: react
+    category: interactions
+    sample_name: [chatbot-component]
+    spec: chatbot-component
+    browser: *minimal_browser_list
+  - test_name: 'Chatbot V1'
+    framework: react
+    category: interactions
+    sample_name: [lex-test-component]
+    spec: chatbot-v1
+    browser: *minimal_browser_list
+  - test_name: 'Angular Interactions'
+    framework: angular
+    category: interactions
+    sample_name: [chatbot-component]
+    spec: chatbot-component
+    browser: *minimal_browser_list
+  - test_name: 'Vue 2 Interactions'
+    framework: vue
+    category: interactions
+    sample_name: [chatbot-component]
+    spec: chatbot-component
+    browser: [chrome]
+  - test_name: 'React Predictions'
+    framework: react
+    category: predictions
+    sample_name: [multi-user-translation]
+    spec: multiuser-translation
+    browser: *minimal_browser_list
+  - test_name: 'PubSub - Reconnection for IoT'
+    framework: react
+    category: pubsub
+    sample_name: [reconnection-iot]
+    spec: reconnection
+    # Firefox doesn't support network state management in cypress
+    browser: [chrome]
+  - test_name: 'PubSub - Reconnection for API'
+    framework: react
+    category: pubsub
+    sample_name: [reconnection-api]
+    spec: reconnection
+    # Firefox doesn't support network state management in cypress
+    browser: [chrome]
+  - test_name: 'React Storage'
+    framework: react
+    category: storage
+    sample_name: [storageApp]
+    spec: storage
+    browser: *minimal_browser_list
+  - test_name: 'React Storage Multi-Part Upload with Progress'
+    framework: react
+    category: storage
+    sample_name: [multi-part-upload-with-progress]
+    spec: multi-part-upload-with-progress
+    browser: *minimal_browser_list
+  - test_name: 'React Storage Copy'
+    framework: react
+    category: storage
+    sample_name: [multi-part-copy-with-progress]
+    spec: multi-part-copy-with-progress
+    browser: *minimal_browser_list
+  - test_name: 'Rollup + DataStore - Basic CRUD'
+    framework: rollup
+    category: datastore
+    sample_name: [rollup-basic-crud]
+    spec: rollup-basic-crud
+    browser: [chrome]
+    timeout_minutes: 45
+    retry_count: 10
+  - test_name: 'Vite + DataStore - Basic CRUD'
+    framework: vite
+    category: datastore
+    sample_name: [v2/basic-crud]
+    spec: vite-basic-crud
+    browser: [chrome]
+    timeout_minutes: 45
+    retry_count: 10
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-e2e-runner.yml b/.github/workflows/callable-canary-e2e-runner.yml
new file mode 100644
index 00000000000..9f1cce9fd85
--- /dev/null
+++ b/.github/workflows/callable-canary-e2e-runner.yml
@@ -0,0 +1,41 @@
+name: 'Canary E2E Test Runner'
+
+on:
+  workflow_call:
+
+jobs:
+  e2e-prep:
+    name: Get required configs to run canary e2e tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout AmplifyJs
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f
+        with:
+          path: amplify-js
+      - name: read canary config files
+        id: load_config
+        run: |
+          echo "CANARY_CONFIG=$(cat .github/canary-config/canary-all.yml | yq '.tests' -o=json | jq -c .)" >> $GITHUB_OUTPUT
+        working-directory: ./amplify-js
+    outputs:
+      canary-config: ${{steps.load_config.outputs.CANARY_CONFIG}}
+  canary-e2e-test-runner:
+    name: canary e2e test runner
+    needs: e2e-prep
+    secrets: inherit
+    strategy:
+      matrix:
+        canary-config: ${{ fromJson(needs.e2e-prep.outputs.canary-config) }}
+      fail-fast: false
+    uses: ./.github/workflows/callable-canary-e2e-tests.yml
+    with:
+      test_name: ${{ matrix.canary-config.test_name }}
+      framework: ${{ matrix.canary-config.framework }}
+      category: ${{ matrix.canary-config.category }}
+      spec: ${{ matrix.canary-config.spec || '' }}
+      amplifyjs_dir: ${{ matrix.canary-config.amplifyjs_dir || false }}
+      sample_name: ${{ toJSON(matrix.canary-config.sample_name) || '[""]' }}
+      browser: ${{ toJSON(matrix.canary-config.browser) || '[""]' }}
+      timeout_minutes: ${{ matrix.canary-config.timeout_minutes || 35 }}
+      retry_count: ${{ matrix.canary-config.retry_count || 3 }}
+
diff --git a/.github/workflows/callable-canary-e2e-tests.yml b/.github/workflows/callable-canary-e2e-tests.yml
new file mode 100644
index 00000000000..e52e070aa8b
--- /dev/null
+++ b/.github/workflows/callable-canary-e2e-tests.yml
@@ -0,0 +1,135 @@
+name: 'Canary E2E Tests'
+
+on:
+  workflow_call:
+    inputs:
+      test_name:
+        required: true
+        type: string
+      framework:
+        required: true
+        type: string
+      category:
+        required: true
+        type: string
+      sample_name:
+        required: true
+        type: string
+      spec:
+        required: false
+        type: string
+        default: ''
+      browser:
+        required: false
+        type: string
+        default: ''
+      amplifyjs_dir:
+        required: false
+        type: boolean
+        default: false
+      timeout_minutes:
+        required: true
+        type: number
+      retry_count:
+        required: true
+        type: number
+
+env:
+  NPM_REGISTRY: http://localhost:4873/
+  NPM_USER: circleci
+  NPM_PASS: circleci
+  NPM_EMAIL: circleci@amplify.js
+  AMPLIFY_DIR: /home/runner/work/amplify-js/amplify-js/amplify-js
+  CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }}
+  CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }}
+  CYPRESS_GOOGLE_REFRESH_TOKEN: ${{ secrets.CYPRESS_GOOGLE_REFRESH_TOKEN }}
+
+jobs:
+  canary-e2e-test:
+    name: 'E2E: ${{ inputs.test_name }}'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        browser:
+          - ${{ fromJson(inputs.browser) }}
+        sample_name:
+          - ${{ fromJson(inputs.sample_name) }}
+      fail-fast: false
+    timeout-minutes: ${{ inputs.timeout_minutes }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 https://github.com/actions/checkout/commit/24cb9080177205b6e8c946b17badbe402adc938f
+        with:
+          # Minimal depth 2 so we can checkout the commit before possible merge commit.
+          fetch-depth: 2
+          path: amplify-js          
+      - name: Setup Node.js 16
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 https://github.com/actions/setup-node/commit/64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: 16
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
+
+      - name: Install
+        run: yarn
+        shell: bash
+        working-directory: ./amplify-js
+      - name: Bootstrap
+        run: yarn bootstrap
+        shell: bash
+        working-directory: ./amplify-js
+      - name: Build packages
+        run: yarn build
+        shell: bash
+        working-directory: ./amplify-js
+        
+      - name: Setup samples staging repository
+        uses: ./amplify-js/.github/actions/setup-samples-staging
+        with:
+          GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
+
+      - name: Modify package.json to run against aws-amplify latest 
+        run: |
+          tmpfile=$(mktemp)
+          cp samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" &&
+          jq 'select(.dependencies) |= with_entries(select(.key | startswith("aws-amplify")).value = "latest")' samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" > samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json && rm -f -- "$tmpfile"
+        working-directory: amplify-js-samples-staging
+        
+      - name: Run cypress tests for ${{ inputs.test_name }} dev
+        run: |
+          ../amplify-js/.circleci/retry-yarn-script.sh -s \
+          "ci:test \
+          ${{ inputs.framework }} \
+          ${{ inputs.category }} \
+          ${{ matrix.sample_name }} \
+          ${{ inputs.spec }} \
+          ${{ matrix.browser }} \
+          dev \
+          ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}" \
+          -n ${{ inputs.retry_count }}
+        working-directory: amplify-js-samples-staging
+        shell: bash
+      - name: Run cypress tests for ${{ inputs.test_name }} prod
+        run: |
+          ../amplify-js/.circleci/retry-yarn-script.sh -s \
+          "ci:test \
+          ${{ inputs.framework }} \
+          ${{ inputs.category }} \
+          ${{ matrix.sample_name }} \
+          ${{ inputs.spec }} \
+          ${{ matrix.browser }} \
+          prod \
+          ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}" \
+          -n ${{ inputs.retry_count }}
+        working-directory: amplify-js-samples-staging
+        shell: bash
+      - name: Upload artifact
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 https://github.com/actions/upload-artifact/commit/0b7f8abb1508181956e8e162db84b466c27e18ce
+        if: always()
+        with:
+          name: canary-e2e-test-artifacts
+          if-no-files-found: ignore
+          path: |
+            amplify-js-samples-staging/cypress/videos
+            amplify-js-samples-staging/cypress/screenshots
diff --git a/.github/workflows/callable-canary-e2e.yml b/.github/workflows/callable-canary-e2e.yml
new file mode 100644
index 00000000000..ff84d74abac
--- /dev/null
+++ b/.github/workflows/callable-canary-e2e.yml
@@ -0,0 +1,44 @@
+name: 'Canary E2E Tests'
+
+on:
+  workflow_call:
+
+env:
+  WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+jobs:
+  prebuild-ubuntu:
+    uses: ./.github/workflows/callable-prebuild-amplify-js.yml
+    with:
+      runs_on: ubuntu-latest
+  prebuild-macos:
+    uses: ./.github/workflows/callable-prebuild-amplify-js.yml
+    with:
+      runs_on: macos-latest
+  prebuild-samples-staging:
+    secrets: inherit
+    uses: ./.github/workflows/callable-prebuild-samples-staging.yml
+  e2e:
+    needs:
+      - prebuild-macos
+      - prebuild-ubuntu
+      - prebuild-samples-staging
+    secrets: inherit
+    uses: ./.github/workflows/callable-canary-e2e-runner.yml
+  sample-app-tests:
+    secrets: inherit
+    uses: ./.github/workflows/callable-canary-sampleapp-tests.yml
+  slack-alert:
+    needs:
+      - e2e
+      - sample-app-tests
+    runs-on: ubuntu-latest
+    if: failure()
+    steps:
+      - name: Send slack message
+        run: |
+          curl -X POST -H "Content-Type: application/json" \
+          --data '{"URL":"${{ env.WORKFLOW_URL }}"}' \
+          https://hooks.slack.com/workflows/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
+         
+        
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-sampleapp-tests.yml b/.github/workflows/callable-canary-sampleapp-tests.yml
new file mode 100644
index 00000000000..9b21b93860c
--- /dev/null
+++ b/.github/workflows/callable-canary-sampleapp-tests.yml
@@ -0,0 +1,228 @@
+name: 'Canary Sample App Tests'
+
+on:
+  workflow_call:
+    inputs:
+      amplifyjs_dir:
+        required: false
+        type: boolean
+        default: true
+
+env:      
+  NPM_REGISTRY: http://localhost:4873/
+  NPM_USER: circleci
+  NPM_PASS: circleci
+  NPM_EMAIL: circleci@amplify.js
+  AMPLIFY_DIR: /home/runner/work/amplify-js/amplify-js/amplify-js
+  CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }}
+  CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }}
+  CYPRESS_GOOGLE_REFRESH_TOKEN: ${{ secrets.CYPRESS_GOOGLE_REFRESH_TOKEN }}
+
+jobs:
+  build_apps:
+    name: Build Sample App Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        BROWSER: [chrome]
+
+    steps: 
+      - name: Checkout repository
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 https://github.com/actions/checkout/commit/24cb9080177205b6e8c946b17badbe402adc938f
+        with:
+          # Minimal depth 2 so we can checkout the commit before possible merge commit.
+          fetch-depth: 2
+          path: amplify-js  
+      - name: Setup samples staging repository
+        uses: ./amplify-js/.github/actions/setup-samples-staging
+        with:
+          GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
+      - name: Setup node and build the repository
+        uses: stocaaro/amplify-js/.github/actions/node-and-build@main
+
+        #React
+      - name: Create react application
+        run: npx create-react-app new-react-app
+        working-directory: amplify-js-samples-staging/samples/react/auth
+      - name: Remove existing src folder
+        run: rm -rf src
+        working-directory: amplify-js-samples-staging/samples/react/auth/new-react-app
+      - name: Copy files from samples staging repo
+        run: |
+          cp -R ./samples/react/auth/credentials-auth/src ./samples/react/auth/new-react-app
+          cp ./samples/react/auth/credentials-auth/package.json ./samples/react/auth/new-react-app/package.json
+        working-directory: amplify-js-samples-staging
+      - name: Copy test file from samples staging repo
+        run: |
+          cp credentials-auth.spec.js new-react-app.spec.js
+        working-directory: amplify-js-samples-staging/cypress/integration/auth
+      - name: Install amplify
+        run: npm install amplify
+        working-directory: amplify-js-samples-staging/samples/react/auth/new-react-app
+      - name: Start new application and run test
+        run: |
+            ../amplify-js/.circleci/retry-yarn-script.sh -s \
+            "ci:test \
+            react \
+            auth \
+            new-react-app \
+            new-react-app \
+            ${{ matrix.BROWSER }} \
+            dev" \
+            -n 3
+        working-directory: amplify-js-samples-staging
+        shell: bash
+      
+      #Angular
+      - name: Install angular CLI
+        run: npm install -g @angular/cli
+      - name: Create angular application
+        run: npx -p @angular/cli ng new new-angular-app
+        working-directory: amplify-js-samples-staging/samples/angular/interactions
+      - name: Remove existing src folder
+        run: rm -rf src
+        working-directory: amplify-js-samples-staging/samples/angular/interactions/new-angular-app
+      - name: Copy files from samples staging repo
+        run: |
+          rm -r ./samples/angular/interactions/new-angular-app  
+          cp -r ./samples/angular/interactions/chatbot-component ./samples/angular/interactions/new-angular-app  
+        working-directory: amplify-js-samples-staging
+      - name: Copy test file from samples staging repo
+        run: |
+          cp chatbot-component.spec.js new-angular-app.spec.js
+        working-directory: amplify-js-samples-staging/cypress/integration/interactions
+      - name: Install dependencies
+        run: npm install --save --legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/angular/interactions/new-angular-app
+      - name: Install amplify
+        run: |
+          npm install -g npm@latest
+          npm install aws-amplify -legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/angular/interactions/new-angular-app
+      - name: Start application and run test
+        run: |
+            ../amplify-js/.circleci/retry-yarn-script.sh -s \
+            "ci:test \
+            angular \
+            interactions \
+            new-angular-app \
+            new-angular-app \
+            ${{ matrix.BROWSER }} \
+            dev" \
+            -n 3
+        working-directory: amplify-js-samples-staging
+        shell: bash
+
+      #Next
+      - name: Create next application
+        run: npx create-next-app new-next-app --no-app
+        working-directory: amplify-js-samples-staging/samples/next/auth
+      - name: Remove existing src folder
+        run: rm -rf src
+        working-directory: amplify-js-samples-staging/samples/next/auth/new-next-app
+      - name: Copy files from samples staging repo
+        run: |
+          rm -r ./samples/next/auth/new-next-app 
+          cp -r ./samples/next/auth/auth-rsc ./samples/next/auth/new-next-app 
+        working-directory: amplify-js-samples-staging
+      - name: Copy test file from samples staging repo
+        run: |
+          cp auth-rsc.spec.js new-next-app.spec.js
+        working-directory: amplify-js-samples-staging/cypress/integration/auth
+      - name: Install dependencies
+        run: npm install
+        working-directory: amplify-js-samples-staging/samples/next/auth/new-next-app
+      - name: Install amplify
+        run: |
+          npm install -g npm@latest
+          npm install aws-amplify -legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/next/auth/new-next-app
+      - name: Start application and run test
+        run: |
+            ../amplify-js/.circleci/retry-yarn-script.sh -s \
+            "ci:test \
+            next \
+            auth \
+            new-next-app \
+            new-next-app \
+            ${{ matrix.BROWSER }} \
+            dev" \
+            -n 3
+        working-directory: amplify-js-samples-staging
+        shell: bash
+      
+        #Vue
+      - name: Create Vue application
+        run: | 
+          npm init vue@3
+        working-directory: amplify-js-samples-staging/samples/vue/auth
+      - name: Copy files from samples staging repo
+        run: |
+          cp -r ./samples/vue/auth/authenticator-vue3 ./samples/vue/auth/new-vue-app
+        working-directory: amplify-js-samples-staging
+      - name: Copy test file from samples staging repo
+        run: |
+          cp new-ui-authenticator.spec.js new-vue-app.spec.js
+        working-directory: amplify-js-samples-staging/cypress/integration/auth
+      - name: Install amplify
+        run: |
+          npm install -g npm@latest
+          npm install aws-amplify -legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/vue/auth/new-vue-app
+      - name: Install dependencies
+        run: npm install --save --legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/vue/auth/new-vue-app
+      - name: Start application and run test
+        run: |
+            ../amplify-js/.circleci/retry-yarn-script.sh -s \
+            "ci:test \
+            vue \
+            auth \
+            new-vue-app \
+            new-vue-app \
+            ${{ matrix.BROWSER }} \
+            dev" \
+            -n 3
+        working-directory: amplify-js-samples-staging
+        shell: bash
+
+        #Javascript
+      - name: Create javascript application
+        run: |
+          mkdir -p new-javascript-app/src && cd new-javascript-app
+          touch index.html src/app.js webpack.config.js
+        working-directory: amplify-js-samples-staging/samples/javascript/auth
+      - name: Install amplify
+        run: |
+          npm install -g npm@latest
+          npm install aws-amplify -legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/javascript/auth/new-javascript-app
+      - name: Remove existing src folder
+        run: rm -rf src
+        working-directory: amplify-js-samples-staging/samples/javascript/auth/new-javascript-app
+      - name: Copy files from samples staging repo
+        run: |
+          rm -r ./samples/javascript/auth/new-javascript-app 
+          cp -r ./samples/javascript/auth/auth-cdn ./samples/javascript/auth/new-javascript-app 
+        working-directory: amplify-js-samples-staging
+      - name: Copy test file from samples staging repo
+        run: |
+          cp auth-cdn.spec.js new-javascript-app.spec.js
+        working-directory: amplify-js-samples-staging/cypress/integration/auth
+      - name: Install dependencies
+        run: npm install --save --legacy-peer-deps
+        working-directory: amplify-js-samples-staging/samples/javascript/auth/new-javascript-app
+      - name: Start application and run test
+        run: |
+            ../amplify-js/.circleci/retry-yarn-script.sh -s \
+            "ci:test \
+            javascript \
+            auth \
+            new-javascript-app \
+            new-javascript-app \
+            ${{ matrix.BROWSER }} \
+            dev \
+            ${{ env.AMPLIFY_DIR }}" \
+            -n 3
+        working-directory: amplify-js-samples-staging
+        shell: bash
\ No newline at end of file
diff --git a/.github/workflows/cron-canary-test.yml b/.github/workflows/cron-canary-test.yml
new file mode 100644
index 00000000000..49cc80113f5
--- /dev/null
+++ b/.github/workflows/cron-canary-test.yml
@@ -0,0 +1,9 @@
+on:
+  push:
+  schedule:
+  - cron: '0 */1 * * *'
+
+jobs:
+  canaries:
+    secrets: inherit
+    uses: ./.github/workflows/callable-canary-e2e.yml

From b9b6cb25695844a3c5dd754e80976b8e3066e928 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Thu, 3 Aug 2023 15:29:59 -0700
Subject: [PATCH 2/8] chore: add canary tests

---
 .github/workflows/cron-canary-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cron-canary-test.yml b/.github/workflows/cron-canary-test.yml
index 49cc80113f5..0d9650720db 100644
--- a/.github/workflows/cron-canary-test.yml
+++ b/.github/workflows/cron-canary-test.yml
@@ -1,5 +1,5 @@
 on:
-  push:
+  pull_request:
   schedule:
   - cron: '0 */1 * * *'
 

From 8b8b024a6ce15b81ea1d628d35e48b8b7a32b928 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Thu, 3 Aug 2023 16:33:36 -0700
Subject: [PATCH 3/8] chore: add canary testing

---
 .github/canary-config/canary-all.yml          | 302 ++++++++++--------
 .../workflows/callable-canary-e2e-runner.yml  |   3 +-
 .../workflows/callable-canary-e2e-tests.yml   |   3 -
 3 files changed, 178 insertions(+), 130 deletions(-)

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
index 9598fa51bac..2318b79c9ad 100644
--- a/.github/canary-config/canary-all.yml
+++ b/.github/canary-config/canary-all.yml
@@ -1,168 +1,114 @@
 minimal_browser_list: &minimal_browser_list
   - chrome
   - firefox
+
 extended_browser_list: &extended_browser_list
   - chrome
   - firefox
   - edge
+
 tests:
-  - test_name: 'DataStore Auth - Chrome'
+    #DATASTORE
+  - test_name: integ_datastore_auth-owner-based-default
+    desc: 'DataStore Auth - Chrome'
     framework: react
     category: datastore
     sample_name: owner-based-default
     spec: owner-based-default
     browser: *minimal_browser_list
-  - test_name: 'React GraphQL API'
-    framework: react
-    category: api
-    sample_name: [graphql]
-    spec: graphql
-    browser: *minimal_browser_list
-  - test_name: 'React Authenticator'
-    framework: react
-    category: auth
-    sample_name: [amplify-authenticator]
-    spec: new-ui-authenticator
-    browser: *minimal_browser_list
-  - test_name: 'React Credentials Different Region'
-    framework: react
-    category: auth
-    sample_name: [credentials-auth]
-    spec: credentials-auth
-    browser: *minimal_browser_list
-  - test_name: 'amazon-cognito-identity-js-cookie-storage'
-    framework: react
-    category: auth
-    sample_name: [amazon-cognito-identity-js-cookie-storage]
-    spec: amazon-cognito-identity-js-cookie-storage
-    browser: *minimal_browser_list
-  - test_name: 'amazon-cognito-identity-js'
-    framework: react
-    category: auth
-    sample_name: [amazon-cognito-identity-js]
-    spec: amazon-cognito-identity-js
-    browser: *minimal_browser_list
-  - test_name: 'cognito-device-tracking'
-    framework: react
-    category: auth
-    sample_name: [device-tracking]
-    spec: device-tracking
-    browser: *minimal_browser_list
-  - test_name: 'delete-user'
-    framework: react
-    category: auth
-    sample_name: [delete-user]
-    spec: delete-user
-    browser: *minimal_browser_list
-  - test_name: 'Angular Authenticator'
-    framework: angular
-    category: auth
-    sample_name: [amplify-authenticator]
-    spec: ui-amplify-authenticator
-    browser: *minimal_browser_list
-  - test_name: 'JavaScript Auth CDN'
-    framework: javascript
-    category: auth
-    sample_name: [auth-cdn]
-    spec: auth-cdn
-    browser: *minimal_browser_list
-    amplifyjs_dir: true
-  - test_name: 'Legacy Vue Authenticator'
-    framework: vue
-    category: auth
-    sample_name: [amplify-authenticator-legacy]
-    spec: authenticator
-    browser: *minimal_browser_list
-  - test_name: 'Authenticator and SSR page'
-    framework: next
-    category: auth
-    sample_name: [auth-ssr]
-    spec: auth-ssr
-    browser: *minimal_browser_list
-  - test_name: 'amazon-cognito-identity-js'
-    framework: node
-    category: auth
-    sample_name: [amazon-cognito-identity-js]
-    build_type: dev
-  - test_name: 'React DataStore'
+  - test_name: integ_react_datastore
+    desc: 'React DataStore'
     framework: react
     category: datastore
     sample_name: [many-to-many]
     spec: many-to-many
     browser: *minimal_browser_list
-  - test_name: 'React DataStore Multi-Auth - One Rule'
+  - test_name: integ_react_datastore_multi_auth_one_rule
+    desc: 'React DataStore Multi-Auth - One Rule'
     framework: react
     category: datastore
     sample_name: [multi-auth]
     spec: multi-auth-one-rule
     browser: *minimal_browser_list
-  - test_name: 'React DataStore Multi-Auth - Two Rules'
+  - test_name: integ_react_datastore_multi_auth_two_rules
+    desc: 'React DataStore Multi-Auth - Two Rules'
     framework: react
     category: datastore
     sample_name: [multi-auth]
     spec: multi-auth-two-rules
     browser: *minimal_browser_list
-  - test_name: 'React DataStore Multi-Auth - Three Plus Rules'
+  - test_name: integ_react_datastore_multi_auth_three_plus_rules
+    desc: 'React DataStore Multi-Auth - Three Plus Rules'
     framework: react
     category: datastore
     sample_name: [multi-auth]
     spec: multi-auth-three-plus-rules
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Subs Disabled'
+  - test_name: integ_react_datastore_subs_disabled
+    desc: 'DataStore - Subs Disabled'
     framework: react
     category: datastore
     sample_name: [subs-disabled]
     spec: subs-disabled
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Subs Disabled'
+  - test_name: integ_react_datastore_consecutive_saves
+    desc: 'DataStore - Subs Disabled'
     framework: react
     category: datastore
     sample_name: [consecutive-saves]
     spec: consecutive-saves
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Schema Drift'
+  - test_name: integ_react_datastore_schema_drift
+    desc: 'DataStore - Schema Drift'
     framework: react
     category: datastore
     sample_name: [schema-drift]
     spec: schema-drift
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Custom Primary Key (Has One)'
+  - test_name: integ_react_datastore_custom_pk_has_one
+    desc: 'DataStore - Custom Primary Key (Has One)'
     framework: react
     category: datastore
     sample_name: [v2/custom-pk-has-one]
     spec: custom-pk-has-one
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Custom Primary Key (Has Many / Many to Many)'
+  - test_name: integ_react_datastore_custom_pk_has_many_many_to_many
+    desc: 'DataStore - Custom Primary Key (Has Many / Many to Many)'
     framework: react
     category: datastore
     sample_name: [v2/custom-pk-has-many-many-to-many]
     spec: custom-pk-has-many-many-to-many
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Background Process Manager'
+  - test_name: integ_react_datastore_background_process_manager
+    desc: 'DataStore - Background Process Manager'
     framework: react
     category: datastore
     sample_name: [v2/background-process-manager]
     spec: background-process-manager
     browser: *extended_browser_list
-  - test_name: 'DataStore - Related Models'
+  - test_name: integ_react_datastore_related_models
+    desc: 'DataStore - Related Models'
     framework: react
     category: datastore
     sample_name: [related-models]
     spec: related-models
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Selective Sync'
+  - test_name: integ_react_datastore_selective_sync
+    desc: 'DataStore - Selective Sync'
     framework: react
     category: datastore
     sample_name: [selective-sync-v5]
     spec: selective-sync-v5
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Nested Predicate'
+  - test_name: integ_react_datastore_nested_predicate
+    desc: 'DataStore - Nested Predicate'
     framework: react
     category: datastore
     sample_name: [nested-predicate]
     spec: nested-predicate
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Docs Examples'
+  - test_name: integ_react_datastore_docs_examples
+    desc: 'DataStore - Docs Examples'
     framework: react
     category: datastore
     sample_name: [v2/amplify-docs-examples]
@@ -170,13 +116,15 @@ tests:
     browser: *minimal_browser_list
     timeout_minutes: 45
     retry_count: 10
-  - test_name: 'DataStore - WebSocket Disruption'
+  - test_name: integ_react_datastore_websocket_disruption
+    desc: 'DataStore - WebSocket Disruption'
     framework: react
     category: datastore
     sample_name: [websocket-disruption]
     spec: websocket-disruption
     browser: *minimal_browser_list
-  - test_name: 'Vanilla JS + Webpack 4 + DataStore - Basic CRUD'
+  - test_name: integ_vanilla_js_datastore_basic_crud
+    desc: 'Vanilla JS + Webpack 4 + DataStore - Basic CRUD'
     framework: javascript
     category: datastore
     sample_name: [basic-crud]
@@ -185,117 +133,221 @@ tests:
     amplifyjs_dir: true
     timeout_minutes: 45
     retry_count: 10
-  - test_name: 'next owner auth'
+  - test_name: integ_next_datastore_owner_auth
+    desc: 'next owner auth'
     framework: next
     category: datastore
     sample_name: [owner-based-default]
     spec: next-owner-based-default
     browser: *minimal_browser_list
-  - test_name: 'DataStore - Nextjs 13 build with SWC - JS app'
+  - test_name: integ_next_datastore_13_js
+    desc: 'DataStore - Nextjs 13 build with SWC - JS app'
     framework: next
     category: datastore
     sample_name: [next-13-js]
     spec: nextjs-13
     browser: *minimal_browser_list
-  - test_name: 'Vite + DataStore - Basic CRUD'
-    framework: vite
-    category: datastore
-    sample_name: [v2/basic-crud]
-    spec: vite-basic-crud
-  - test_name: 'Rollup + DataStore - Basic CRUD'
+  - test_name: integ_rollup_datastore_basic_crud
+    desc: 'Rollup + DataStore - Basic CRUD'
     framework: rollup
     category: datastore
     sample_name: [rollup-basic-crud]
     spec: rollup-basic-crud
-  - test_name: 'Display Map'
+    browser: [chrome]
+    timeout_minutes: 45
+    retry_count: 10
+  - test_name: integ_vite_datastore_basic_crud
+    desc: 'Vite + DataStore - Basic CRUD'
+    framework: vite
+    category: datastore
+    sample_name: [v2/basic-crud]
+    spec: vite-basic-crud
+    browser: [chrome]
+    timeout_minutes: 45
+    retry_count: 10
+
+    #GEO
+  - test_name: integ_react_geo_display_map
+    desc: 'Display Map'
     framework: react
     category: geo
     sample_name: [display-map]
     spec: display-map
     # Temp fix:
     browser: [chrome]
-  - test_name: 'Search Outside Map'
+  - test_name: integ_react_geo_search_outside_map
+    desc: 'Search Outside Map'
     framework: react
     category: geo
     sample_name: [search-outside-map]
     spec: search-outside-map
     # Temp fix:
     browser: [chrome]
-  - test_name: 'React Interactions'
+
+    #INTERACTIONS
+  - test_name: integ_react_interactions_react_interactions
+    desc: 'React Interactions'
     framework: react
     category: interactions
     sample_name: [chatbot-component]
     spec: chatbot-component
     browser: *minimal_browser_list
-  - test_name: 'Chatbot V1'
+  - test_name: integ_react_interactions_chatbot_v1
+    desc: 'Chatbot V1'
     framework: react
     category: interactions
     sample_name: [lex-test-component]
     spec: chatbot-v1
     browser: *minimal_browser_list
-  - test_name: 'Angular Interactions'
+  - test_name: integ_angular_interactions
+    desc: 'Angular Interactions'
     framework: angular
     category: interactions
     sample_name: [chatbot-component]
     spec: chatbot-component
     browser: *minimal_browser_list
-  - test_name: 'Vue 2 Interactions'
+  - test_name: integ_vue_interactions_vue_2_interactions
+    desc: 'Vue 2 Interactions'
     framework: vue
     category: interactions
     sample_name: [chatbot-component]
     spec: chatbot-component
     browser: [chrome]
-  - test_name: 'React Predictions'
+
+    #PREDICTIONS
+  - test_name: integ_react_predictions
+    desc: 'React Predictions'
     framework: react
     category: predictions
     sample_name: [multi-user-translation]
     spec: multiuser-translation
     browser: *minimal_browser_list
-  - test_name: 'PubSub - Reconnection for IoT'
+
+    #PUBSUB
+  - test_name: integ_react_iot_reconnect
+    desc: 'PubSub - Reconnection for IoT'
     framework: react
     category: pubsub
     sample_name: [reconnection-iot]
     spec: reconnection
     # Firefox doesn't support network state management in cypress
     browser: [chrome]
-  - test_name: 'PubSub - Reconnection for API'
+  - test_name: integ_react_api_reconnect
+    desc: 'PubSub - Reconnection for API'
     framework: react
     category: pubsub
     sample_name: [reconnection-api]
     spec: reconnection
     # Firefox doesn't support network state management in cypress
     browser: [chrome]
-  - test_name: 'React Storage'
+
+    #STORAGE
+  - test_name: integ_react_storage
+    desc: 'React Storage'
     framework: react
     category: storage
     sample_name: [storageApp]
     spec: storage
     browser: *minimal_browser_list
-  - test_name: 'React Storage Multi-Part Upload with Progress'
+  - test_name: integ_react_storage_multipart_progress
+    desc: 'React Storage Multi-Part Upload with Progress'
     framework: react
     category: storage
     sample_name: [multi-part-upload-with-progress]
     spec: multi-part-upload-with-progress
     browser: *minimal_browser_list
-  - test_name: 'React Storage Copy'
+  - test_name: integ_react_storage_copy
+    desc: 'React Storage Copy'
     framework: react
     category: storage
     sample_name: [multi-part-copy-with-progress]
     spec: multi-part-copy-with-progress
     browser: *minimal_browser_list
-  - test_name: 'Rollup + DataStore - Basic CRUD'
-    framework: rollup
-    category: datastore
-    sample_name: [rollup-basic-crud]
-    spec: rollup-basic-crud
-    browser: [chrome]
-    timeout_minutes: 45
-    retry_count: 10
-  - test_name: 'Vite + DataStore - Basic CRUD'
-    framework: vite
-    category: datastore
-    sample_name: [v2/basic-crud]
-    spec: vite-basic-crud
-    browser: [chrome]
-    timeout_minutes: 45
-    retry_count: 10
\ No newline at end of file
+ 
+    #API
+  - test_name: integ_react_graphql_api
+    desc: 'React GraphQL API'
+    framework: react
+    category: api
+    sample_name: [graphql]
+    spec: graphql
+    browser: *minimal_browser_list
+
+    #AUTH
+  - test_name: integ_react_auth_1_react_authenticator
+    desc: 'React Authenticator'
+    framework: react
+    category: auth
+    sample_name: [amplify-authenticator]
+    spec: new-ui-authenticator
+    browser: *minimal_browser_list
+  - test_name: integ_react_auth_2_react_credentials_different_region
+    desc: 'React Credentials Different Region'
+    framework: react
+    category: auth
+    sample_name: [credentials-auth]
+    spec: credentials-auth
+    browser: *minimal_browser_list
+  - test_name: integ_react_amazon_cognito_identity_js_cookie_storage
+    desc: 'amazon-cognito-identity-js-cookie-storage'
+    framework: react
+    category: auth
+    sample_name: [amazon-cognito-identity-js-cookie-storage]
+    spec: amazon-cognito-identity-js-cookie-storage
+    browser: *minimal_browser_list
+  - test_name: integ_react_amazon_cognito_identity_js
+    desc: 'amazon-cognito-identity-js'
+    framework: react
+    category: auth
+    sample_name: [amazon-cognito-identity-js]
+    spec: amazon-cognito-identity-js
+    browser: *minimal_browser_list
+  - test_name: integ_react_device_tracking
+    desc: 'cognito-device-tracking'
+    framework: react
+    category: auth
+    sample_name: [device-tracking]
+    spec: device-tracking
+    browser: *minimal_browser_list
+  - test_name: integ_react_delete_user
+    desc: 'delete-user'
+    framework: react
+    category: auth
+    sample_name: [delete-user]
+    spec: delete-user
+    browser: *minimal_browser_list
+  - test_name: integ_angular_auth_angular_authenticator
+    desc: 'Angular Authenticator'
+    framework: angular
+    category: auth
+    sample_name: [amplify-authenticator]
+    spec: ui-amplify-authenticator
+    browser: *minimal_browser_list
+  - test_name: integ_javascript_auth
+    desc: 'JavaScript Auth CDN'
+    framework: javascript
+    category: auth
+    sample_name: [auth-cdn]
+    spec: auth-cdn
+    browser: *minimal_browser_list
+    amplifyjs_dir: true
+  - test_name: integ_vue_auth_legacy_vue_authenticator
+    desc: 'Legacy Vue Authenticator'
+    framework: vue
+    category: auth
+    sample_name: [amplify-authenticator-legacy]
+    spec: authenticator
+    browser: *minimal_browser_list
+  - test_name: integ_next_auth_authenticator_and_ssr_page
+    desc: 'Authenticator and SSR page'
+    framework: next
+    category: auth
+    sample_name: [auth-ssr]
+    spec: auth-ssr
+    browser: *minimal_browser_list
+  - test_name: integ_node_amazon_cognito_identity_js
+    desc: 'amazon-cognito-identity-js'
+    framework: node
+    category: auth
+    sample_name: [amazon-cognito-identity-js]
+    build_type: dev
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-e2e-runner.yml b/.github/workflows/callable-canary-e2e-runner.yml
index 9f1cce9fd85..ecde3811698 100644
--- a/.github/workflows/callable-canary-e2e-runner.yml
+++ b/.github/workflows/callable-canary-e2e-runner.yml
@@ -37,5 +37,4 @@ jobs:
       sample_name: ${{ toJSON(matrix.canary-config.sample_name) || '[""]' }}
       browser: ${{ toJSON(matrix.canary-config.browser) || '[""]' }}
       timeout_minutes: ${{ matrix.canary-config.timeout_minutes || 35 }}
-      retry_count: ${{ matrix.canary-config.retry_count || 3 }}
-
+      retry_count: ${{ matrix.canary-config.retry_count || 3 }}
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-e2e-tests.yml b/.github/workflows/callable-canary-e2e-tests.yml
index e52e070aa8b..44e04fc9bcb 100644
--- a/.github/workflows/callable-canary-e2e-tests.yml
+++ b/.github/workflows/callable-canary-e2e-tests.yml
@@ -83,19 +83,16 @@ jobs:
         run: yarn build
         shell: bash
         working-directory: ./amplify-js
-        
       - name: Setup samples staging repository
         uses: ./amplify-js/.github/actions/setup-samples-staging
         with:
           GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
-
       - name: Modify package.json to run against aws-amplify latest 
         run: |
           tmpfile=$(mktemp)
           cp samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" &&
           jq 'select(.dependencies) |= with_entries(select(.key | startswith("aws-amplify")).value = "latest")' samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" > samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json && rm -f -- "$tmpfile"
         working-directory: amplify-js-samples-staging
-        
       - name: Run cypress tests for ${{ inputs.test_name }} dev
         run: |
           ../amplify-js/.circleci/retry-yarn-script.sh -s \

From fb355cf7f10704e74bcac3cff59b7b8e83459845 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Thu, 3 Aug 2023 16:40:35 -0700
Subject: [PATCH 4/8] chore: add canary testing

---
 .github/canary-config/canary-all.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
index 2318b79c9ad..cbeb7094a7e 100644
--- a/.github/canary-config/canary-all.yml
+++ b/.github/canary-config/canary-all.yml
@@ -100,13 +100,6 @@ tests:
     sample_name: [selective-sync-v5]
     spec: selective-sync-v5
     browser: *minimal_browser_list
-  - test_name: integ_react_datastore_nested_predicate
-    desc: 'DataStore - Nested Predicate'
-    framework: react
-    category: datastore
-    sample_name: [nested-predicate]
-    spec: nested-predicate
-    browser: *minimal_browser_list
   - test_name: integ_react_datastore_docs_examples
     desc: 'DataStore - Docs Examples'
     framework: react

From 9450b3ce19b47c04fb1ba39c3b84fab96bbda3b6 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Fri, 4 Aug 2023 11:04:06 -0700
Subject: [PATCH 5/8] chore: add canary tests

---
 .github/canary-config/canary-all.yml             | 16 ++++++++--------
 .github/workflows/callable-canary-e2e-tests.yml  | 12 +++---------
 .../callable-canary-sampleapp-tests.yml          | 14 +++++---------
 ...nary-test.yml => on-schedule-canary-test.yml} |  1 +
 4 files changed, 17 insertions(+), 26 deletions(-)
 rename .github/workflows/{cron-canary-test.yml => on-schedule-canary-test.yml} (80%)

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
index cbeb7094a7e..44f677c1ed5 100644
--- a/.github/canary-config/canary-all.yml
+++ b/.github/canary-config/canary-all.yml
@@ -8,7 +8,7 @@ extended_browser_list: &extended_browser_list
   - edge
 
 tests:
-    #DATASTORE
+    # DATASTORE
   - test_name: integ_datastore_auth-owner-based-default
     desc: 'DataStore Auth - Chrome'
     framework: react
@@ -159,7 +159,7 @@ tests:
     timeout_minutes: 45
     retry_count: 10
 
-    #GEO
+    # GEO
   - test_name: integ_react_geo_display_map
     desc: 'Display Map'
     framework: react
@@ -177,7 +177,7 @@ tests:
     # Temp fix:
     browser: [chrome]
 
-    #INTERACTIONS
+    # INTERACTIONS
   - test_name: integ_react_interactions_react_interactions
     desc: 'React Interactions'
     framework: react
@@ -207,7 +207,7 @@ tests:
     spec: chatbot-component
     browser: [chrome]
 
-    #PREDICTIONS
+    # PREDICTIONS
   - test_name: integ_react_predictions
     desc: 'React Predictions'
     framework: react
@@ -216,7 +216,7 @@ tests:
     spec: multiuser-translation
     browser: *minimal_browser_list
 
-    #PUBSUB
+    # PUBSUB
   - test_name: integ_react_iot_reconnect
     desc: 'PubSub - Reconnection for IoT'
     framework: react
@@ -234,7 +234,7 @@ tests:
     # Firefox doesn't support network state management in cypress
     browser: [chrome]
 
-    #STORAGE
+    # STORAGE
   - test_name: integ_react_storage
     desc: 'React Storage'
     framework: react
@@ -257,7 +257,7 @@ tests:
     spec: multi-part-copy-with-progress
     browser: *minimal_browser_list
  
-    #API
+    # API
   - test_name: integ_react_graphql_api
     desc: 'React GraphQL API'
     framework: react
@@ -266,7 +266,7 @@ tests:
     spec: graphql
     browser: *minimal_browser_list
 
-    #AUTH
+    # AUTH
   - test_name: integ_react_auth_1_react_authenticator
     desc: 'React Authenticator'
     framework: react
diff --git a/.github/workflows/callable-canary-e2e-tests.yml b/.github/workflows/callable-canary-e2e-tests.yml
index 44e04fc9bcb..e7f427b61af 100644
--- a/.github/workflows/callable-canary-e2e-tests.yml
+++ b/.github/workflows/callable-canary-e2e-tests.yml
@@ -18,11 +18,9 @@ on:
       spec:
         required: false
         type: string
-        default: ''
       browser:
         required: false
         type: string
-        default: ''
       amplifyjs_dir:
         required: false
         type: boolean
@@ -35,10 +33,6 @@ on:
         type: number
 
 env:
-  NPM_REGISTRY: http://localhost:4873/
-  NPM_USER: circleci
-  NPM_PASS: circleci
-  NPM_EMAIL: circleci@amplify.js
   AMPLIFY_DIR: /home/runner/work/amplify-js/amplify-js/amplify-js
   CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }}
   CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }}
@@ -123,10 +117,10 @@ jobs:
         shell: bash
       - name: Upload artifact
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 https://github.com/actions/upload-artifact/commit/0b7f8abb1508181956e8e162db84b466c27e18ce
-        if: always()
+        if: failure()
         with:
-          name: canary-e2e-test-artifacts
+          name: ${{ inputs.test_name }}
           if-no-files-found: ignore
           path: |
             amplify-js-samples-staging/cypress/videos
-            amplify-js-samples-staging/cypress/screenshots
+            amplify-js-samples-staging/cypress/screenshots
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-sampleapp-tests.yml b/.github/workflows/callable-canary-sampleapp-tests.yml
index 9b21b93860c..3f4837f407b 100644
--- a/.github/workflows/callable-canary-sampleapp-tests.yml
+++ b/.github/workflows/callable-canary-sampleapp-tests.yml
@@ -9,10 +9,6 @@ on:
         default: true
 
 env:      
-  NPM_REGISTRY: http://localhost:4873/
-  NPM_USER: circleci
-  NPM_PASS: circleci
-  NPM_EMAIL: circleci@amplify.js
   AMPLIFY_DIR: /home/runner/work/amplify-js/amplify-js/amplify-js
   CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }}
   CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }}
@@ -40,7 +36,7 @@ jobs:
       - name: Setup node and build the repository
         uses: stocaaro/amplify-js/.github/actions/node-and-build@main
 
-        #React
+        # React
       - name: Create react application
         run: npx create-react-app new-react-app
         working-directory: amplify-js-samples-staging/samples/react/auth
@@ -73,7 +69,7 @@ jobs:
         working-directory: amplify-js-samples-staging
         shell: bash
       
-      #Angular
+      # Angular
       - name: Install angular CLI
         run: npm install -g @angular/cli
       - name: Create angular application
@@ -113,7 +109,7 @@ jobs:
         working-directory: amplify-js-samples-staging
         shell: bash
 
-      #Next
+      # Next
       - name: Create next application
         run: npx create-next-app new-next-app --no-app
         working-directory: amplify-js-samples-staging/samples/next/auth
@@ -151,7 +147,7 @@ jobs:
         working-directory: amplify-js-samples-staging
         shell: bash
       
-        #Vue
+        # Vue
       - name: Create Vue application
         run: | 
           npm init vue@3
@@ -186,7 +182,7 @@ jobs:
         working-directory: amplify-js-samples-staging
         shell: bash
 
-        #Javascript
+        # Javascript
       - name: Create javascript application
         run: |
           mkdir -p new-javascript-app/src && cd new-javascript-app
diff --git a/.github/workflows/cron-canary-test.yml b/.github/workflows/on-schedule-canary-test.yml
similarity index 80%
rename from .github/workflows/cron-canary-test.yml
rename to .github/workflows/on-schedule-canary-test.yml
index 0d9650720db..9e9758281dd 100644
--- a/.github/workflows/cron-canary-test.yml
+++ b/.github/workflows/on-schedule-canary-test.yml
@@ -1,5 +1,6 @@
 on:
   pull_request:
+  # Tests scheduled for every 1 hour
   schedule:
   - cron: '0 */1 * * *'
 

From 2445bd733ca566fd5ed16275229b07c73f4ff0c3 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Fri, 4 Aug 2023 14:04:19 -0700
Subject: [PATCH 6/8] chore: add canary tests

---
 .github/canary-config/canary-all.yml          |  7 ---
 .../workflows/callable-canary-e2e-tests.yml   | 57 +++++++++++++------
 .github/workflows/callable-canary-e2e.yml     |  8 ++-
 .../callable-canary-sampleapp-tests.yml       | 20 +++----
 4 files changed, 55 insertions(+), 37 deletions(-)

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
index 44f677c1ed5..1879e75f514 100644
--- a/.github/canary-config/canary-all.yml
+++ b/.github/canary-config/canary-all.yml
@@ -30,13 +30,6 @@ tests:
     sample_name: [multi-auth]
     spec: multi-auth-one-rule
     browser: *minimal_browser_list
-  - test_name: integ_react_datastore_multi_auth_two_rules
-    desc: 'React DataStore Multi-Auth - Two Rules'
-    framework: react
-    category: datastore
-    sample_name: [multi-auth]
-    spec: multi-auth-two-rules
-    browser: *minimal_browser_list
   - test_name: integ_react_datastore_multi_auth_three_plus_rules
     desc: 'React DataStore Multi-Auth - Three Plus Rules'
     framework: react
diff --git a/.github/workflows/callable-canary-e2e-tests.yml b/.github/workflows/callable-canary-e2e-tests.yml
index e7f427b61af..2827adea201 100644
--- a/.github/workflows/callable-canary-e2e-tests.yml
+++ b/.github/workflows/callable-canary-e2e-tests.yml
@@ -82,37 +82,59 @@ jobs:
         with:
           GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
       - name: Modify package.json to run against aws-amplify latest 
+        env: 
+          E2E_FRAMEWORK: ${{ inputs.framework }}
+          E2E_CATEGORY: ${{ inputs.category }}
+          E2E_SAMPLE_NAME: ${{ matrix.sample_name }}
         run: |
           tmpfile=$(mktemp)
-          cp samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" &&
-          jq 'select(.dependencies) |= with_entries(select(.key | startswith("aws-amplify")).value = "latest")' samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json "$tmpfile" > samples/${{ inputs.framework }}/${{ inputs.category }}/${{ matrix.sample_name }}/package.json && rm -f -- "$tmpfile"
+          cp samples/$E2E_FRAMEWORK/$E2E_CATEGORY/$E2E_SAMPLE_NAME/package.json "$tmpfile" &&
+          jq 'select(.dependencies) |= with_entries(select(.key | startswith("aws-amplify")).value = "latest")' samples/$E2E_FRAMEWORK/$E2E_CATEGORY/$E2E_SAMPLE_NAME/package.json "$tmpfile" > samples/$E2E_FRAMEWORK/$E2E_CATEGORY/$E2E_SAMPLE_NAME/package.json && rm -f -- "$tmpfile"
         working-directory: amplify-js-samples-staging
       - name: Run cypress tests for ${{ inputs.test_name }} dev
+        env:
+          E2E_FRAMEWORK: ${{ inputs.framework }}
+          E2E_CATEGORY: ${{ inputs.category }}
+          E2E_SAMPLE_NAME: ${{ matrix.sample_name }}
+          E2E_SPEC: ${{ inputs.spec }}
+          E2E_BROWSER: ${{ matrix.browser }}
+          E2E_AMPLIFY_JS_DIR: ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}
+          E2E_RETRY_COUNT: ${{ inputs.retry_count }}
+          E2E_TEST_NAME: ${{ inputs.test_name }}
         run: |
           ../amplify-js/.circleci/retry-yarn-script.sh -s \
           "ci:test \
-          ${{ inputs.framework }} \
-          ${{ inputs.category }} \
-          ${{ matrix.sample_name }} \
-          ${{ inputs.spec }} \
-          ${{ matrix.browser }} \
+          $E2E_FRAMEWORK \
+          $E2E_CATEGORY \
+          $E2E_SAMPLE_NAME \
+          $E2E_SPEC \
+          $E2E_BROWSER \
           dev \
-          ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}" \
-          -n ${{ inputs.retry_count }}
+          $E2E_AMPLIFY_JS_DIR" \
+          -n $E2E_RETRY_COUNT
         working-directory: amplify-js-samples-staging
         shell: bash
       - name: Run cypress tests for ${{ inputs.test_name }} prod
+        env:
+          E2E_FRAMEWORK: ${{ inputs.framework }}
+          E2E_CATEGORY: ${{ inputs.category }}
+          E2E_SAMPLE_NAME: ${{ matrix.sample_name }}
+          E2E_SPEC: ${{ inputs.spec }}
+          E2E_BROWSER: ${{ matrix.browser }}
+          E2E_AMPLIFY_JS_DIR: ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}
+          E2E_RETRY_COUNT: ${{ inputs.retry_count }}
+          E2E_TEST_NAME: ${{ inputs.test_name }}
         run: |
           ../amplify-js/.circleci/retry-yarn-script.sh -s \
           "ci:test \
-          ${{ inputs.framework }} \
-          ${{ inputs.category }} \
-          ${{ matrix.sample_name }} \
-          ${{ inputs.spec }} \
-          ${{ matrix.browser }} \
+          $E2E_FRAMEWORK \
+          $E2E_CATEGORY \
+          $E2E_SAMPLE_NAME \
+          $E2E_SPEC \
+          $E2E_BROWSER \
           prod \
-          ${{ inputs.amplifyjs_dir == true && env.AMPLIFY_DIR || ''}}" \
-          -n ${{ inputs.retry_count }}
+          $E2E_AMPLIFY_JS_DIR" \
+          -n $E2E_RETRY_COUNT
         working-directory: amplify-js-samples-staging
         shell: bash
       - name: Upload artifact
@@ -123,4 +145,5 @@ jobs:
           if-no-files-found: ignore
           path: |
             amplify-js-samples-staging/cypress/videos
-            amplify-js-samples-staging/cypress/screenshots
\ No newline at end of file
+            amplify-js-samples-staging/cypress/screenshots
+          retention-days: 14
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-e2e.yml b/.github/workflows/callable-canary-e2e.yml
index ff84d74abac..2225b0c78ff 100644
--- a/.github/workflows/callable-canary-e2e.yml
+++ b/.github/workflows/callable-canary-e2e.yml
@@ -5,6 +5,7 @@ on:
 
 env:
   WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
 
 jobs:
   prebuild-ubuntu:
@@ -36,9 +37,12 @@ jobs:
     if: failure()
     steps:
       - name: Send slack message
+        env:
+          WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
+          WEBHOOK_URL: ${{ env.WEBHOOK_URL }}
         run: |
           curl -X POST -H "Content-Type: application/json" \
-          --data '{"URL":"${{ env.WORKFLOW_URL }}"}' \
-          https://hooks.slack.com/workflows/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
+          --data '{"URL":"$WORKFLOW_URL"}' \
+          $WEBHOOK_URL
          
         
\ No newline at end of file
diff --git a/.github/workflows/callable-canary-sampleapp-tests.yml b/.github/workflows/callable-canary-sampleapp-tests.yml
index 3f4837f407b..3315d7e9509 100644
--- a/.github/workflows/callable-canary-sampleapp-tests.yml
+++ b/.github/workflows/callable-canary-sampleapp-tests.yml
@@ -18,10 +18,6 @@ jobs:
   build_apps:
     name: Build Sample App Tests
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        BROWSER: [chrome]
-
     steps: 
       - name: Checkout repository
         uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 https://github.com/actions/checkout/commit/24cb9080177205b6e8c946b17badbe402adc938f
@@ -34,7 +30,7 @@ jobs:
         with:
           GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
       - name: Setup node and build the repository
-        uses: stocaaro/amplify-js/.github/actions/node-and-build@main
+        uses: ./amplify-js/.github/actions/node-and-build@main
 
         # React
       - name: Create react application
@@ -63,7 +59,7 @@ jobs:
             auth \
             new-react-app \
             new-react-app \
-            ${{ matrix.BROWSER }} \
+            chrome \
             dev" \
             -n 3
         working-directory: amplify-js-samples-staging
@@ -103,7 +99,7 @@ jobs:
             interactions \
             new-angular-app \
             new-angular-app \
-            ${{ matrix.BROWSER }} \
+            chrome \
             dev" \
             -n 3
         working-directory: amplify-js-samples-staging
@@ -141,7 +137,7 @@ jobs:
             auth \
             new-next-app \
             new-next-app \
-            ${{ matrix.BROWSER }} \
+            chrome \
             dev" \
             -n 3
         working-directory: amplify-js-samples-staging
@@ -176,7 +172,7 @@ jobs:
             auth \
             new-vue-app \
             new-vue-app \
-            ${{ matrix.BROWSER }} \
+            chrome \
             dev" \
             -n 3
         working-directory: amplify-js-samples-staging
@@ -209,6 +205,8 @@ jobs:
         run: npm install --save --legacy-peer-deps
         working-directory: amplify-js-samples-staging/samples/javascript/auth/new-javascript-app
       - name: Start application and run test
+        env:
+          AMPLIFY_DIR: ${{ env.AMPLIFY_DIR }}
         run: |
             ../amplify-js/.circleci/retry-yarn-script.sh -s \
             "ci:test \
@@ -216,9 +214,9 @@ jobs:
             auth \
             new-javascript-app \
             new-javascript-app \
-            ${{ matrix.BROWSER }} \
+            chrome \
             dev \
-            ${{ env.AMPLIFY_DIR }}" \
+            $AMPLIFY_DIR" \
             -n 3
         working-directory: amplify-js-samples-staging
         shell: bash
\ No newline at end of file

From 0587489eb4fde37f3e25aa4f33998970553b9690 Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Fri, 4 Aug 2023 14:12:24 -0700
Subject: [PATCH 7/8] chore: add canary tests

---
 .github/canary-config/canary-all.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/.github/canary-config/canary-all.yml b/.github/canary-config/canary-all.yml
index 1879e75f514..b9a433bb3d7 100644
--- a/.github/canary-config/canary-all.yml
+++ b/.github/canary-config/canary-all.yml
@@ -109,16 +109,6 @@ tests:
     sample_name: [websocket-disruption]
     spec: websocket-disruption
     browser: *minimal_browser_list
-  - test_name: integ_vanilla_js_datastore_basic_crud
-    desc: 'Vanilla JS + Webpack 4 + DataStore - Basic CRUD'
-    framework: javascript
-    category: datastore
-    sample_name: [basic-crud]
-    browser: *minimal_browser_list
-    spec: vanilla-js-basic-crud
-    amplifyjs_dir: true
-    timeout_minutes: 45
-    retry_count: 10
   - test_name: integ_next_datastore_owner_auth
     desc: 'next owner auth'
     framework: next

From 40f46c1ef6f791075911c0120cef549ac101bf1b Mon Sep 17 00:00:00 2001
From: yingku <yingku@umich.edu>
Date: Fri, 4 Aug 2023 14:18:45 -0700
Subject: [PATCH 8/8] chore: add canary tests

---
 .github/workflows/callable-canary-sampleapp-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/callable-canary-sampleapp-tests.yml b/.github/workflows/callable-canary-sampleapp-tests.yml
index 3315d7e9509..97478f0df89 100644
--- a/.github/workflows/callable-canary-sampleapp-tests.yml
+++ b/.github/workflows/callable-canary-sampleapp-tests.yml
@@ -30,7 +30,7 @@ jobs:
         with:
           GH_TOKEN_STAGING_READ: ${{ secrets.GH_TOKEN_STAGING_READ }}
       - name: Setup node and build the repository
-        uses: ./amplify-js/.github/actions/node-and-build@main
+        uses: ./amplify-js/.github/actions/node-and-build
 
         # React
       - name: Create react application