Feature/new test

.github/workflows/add-collection-test.yml

@@ -0,0 +1,156 @@
+# This is a test pipeline, that verifies and updates the associations in cmr
+name: Add Collection Test
+# Controls when the workflow will run
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  # First job in the workflow installs and verifies the software
+  build:
+    name: Test Execution
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Create Output Directory
+        run: mkdir test_output
+      - name: Set environment variables
+        run: |
+          echo "UAT_OUTPUT_FILE=test_output/output/uat_output_json.txt" >> $GITHUB_ENV
+          echo "OPS_OUTPUT_FILE=test_output/output/ops_output_json.txt" >> $GITHUB_ENV
+          echo "OUTPUT_DIR=test_output/output" >> $GITHUB_ENV
+          echo "UAT_SUCCESS_FILE=uat_success.txt" >> $GITHUB_ENV
+          echo "OPS_SUCCESS_FILE=ops_success.txt" >> $GITHUB_ENV
+          echo "UAT_FAILURE_FILE=uat_fail.txt" >> $GITHUB_ENV
+          echo "OPS_FAILURE_FILE=ops_fail.txt" >> $GITHUB_ENV
+          echo "SLACK_REPORT_CONTENT=" >> $GITHUB_ENV
+      - name: Set environment variables for notebook
+        run: |
+          echo "OPS_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
+          echo "OPS_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
+          echo "UAT_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
+          echo "UAT_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
+      - name: Create prerequisites
+        run: |
+          mkdir ${{ env.OUTPUT_DIR }}
+          realpath ${{ env.OUTPUT_DIR }}
+      - name: Install dependencies
+        run: |
+          pip3 install --upgrade pip
+          pip3 install black
+          pip3 install netCDF4
+          pip3 install git+https://github.com/nasa/harmony-py.git
+          pip3 install git+https://github.com/podaac/cmr-umm-updater.git
+          pip3 install git+https://github.com/podaac/cmr-association-diff.git@6193079a14e36f4c9526aa426015c2b6be41f0e2
+          pip3 install python-dateutil --upgrade
+      - name: Run CMR Association diff scripts
+        run: |
+          cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.UAT_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
+          cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.OPS_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
+      - name: Run Add Collection Test
+        run: |
+          python3 add_collection_test.py -e uat -i ${{ env.UAT_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
+          python3 add_collection_test.py -e ops -i ${{ env.OPS_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
+      - name: Check UAT files
+        id: check_UAT_output_files
+        run: |
+          echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo
+          echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+      - name: Check OPS files
+        id: check_OPS_output_files
+        run: |
+          echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo
+          echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+      - name: Slack Report update - Add info for UAT check - SUCCESS
+        if: |
+          steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for UAT check - FAILURE
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for OPS check - SUCCESS
+        if: |
+          steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for OPS check - FAILURE
+        if: |
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add action link on Failure
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' ||
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nExecution link:\ngithub.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV
+          echo ${{ env.SLACK_REPORT_CONTENT }}
+      - name: Send Report to Slack
+        uses: slackapi/slack-github-action@v1.21.0
+        if: |
+          env.SLACK_REPORT_CONTENT != ''
+        with:
+          payload: |
+            {
+              "status": "${{ env.SLACK_REPORT_CONTENT }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_WEBHOOK }}
+      - name: Remove UAT collections on FAIL
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e uat -i '${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
+      - name: Remove OPS collections on FAIL
+        if: |
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e ops -i '${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
+      - name: Update UAT collections on SUCCESS
+        if: |
+          steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
+        # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
+        run: |
+          vi -escwq ./cmr/uat_associations.txt
+          cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} >> ./cmr/uat_associations.txt
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git commit -am "Update uat_associations.txt with new collections"
+          git push
+      - name: Update OPS collections on SUCCESS
+        if: |
+          steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
+        # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
+        run: |
+          vi -escwq ./cmr/ops_associations.txt
+          cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} >> ./cmr/ops_associations.txt
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git commit -am "Update ops_associations.txt with new collections"
+          git push

.github/workflows/jupyter_test.yml

@@ -1,5 +1,5 @@
 # This is a test pipeline, that verifies and updates the associations in cmr
-name: Add Collection Test
+name: Add Collection Test OLD
 # Controls when the workflow will run
 on:
   # Allows you to run this workflow manually from the Actions tab