diff --git a/.github/workflows/add-collection-test.yml b/.github/workflows/add-collection-test.yml new file mode 100644 index 00000000..a156eb6b --- /dev/null +++ b/.github/workflows/add-collection-test.yml @@ -0,0 +1,156 @@ +# This is a test pipeline, that verifies and updates the associations in cmr +name: Add Collection Test +# Controls when the workflow will run +on: + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # First job in the workflow installs and verifies the software + build: + name: Test Execution + # The type of runner that the job will run on + runs-on: ubuntu-latest + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Create Output Directory + run: mkdir test_output + - name: Set environment variables + run: | + echo "UAT_OUTPUT_FILE=test_output/output/uat_output_json.txt" >> $GITHUB_ENV + echo "OPS_OUTPUT_FILE=test_output/output/ops_output_json.txt" >> $GITHUB_ENV + echo "OUTPUT_DIR=test_output/output" >> $GITHUB_ENV + echo "UAT_SUCCESS_FILE=uat_success.txt" >> $GITHUB_ENV + echo "OPS_SUCCESS_FILE=ops_success.txt" >> $GITHUB_ENV + echo "UAT_FAILURE_FILE=uat_fail.txt" >> $GITHUB_ENV + echo "OPS_FAILURE_FILE=ops_fail.txt" >> $GITHUB_ENV + echo "SLACK_REPORT_CONTENT=" >> $GITHUB_ENV + - name: Set environment variables for notebook + run: | + echo "OPS_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV + echo "OPS_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV + echo "UAT_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV + echo "UAT_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV + - name: Create prerequisites + run: | + mkdir ${{ env.OUTPUT_DIR }} + realpath ${{ env.OUTPUT_DIR }} + - name: Install dependencies + run: | + pip3 install --upgrade pip + pip3 install black + pip3 install netCDF4 + pip3 install git+https://github.com/nasa/harmony-py.git + pip3 install git+https://github.com/podaac/cmr-umm-updater.git + pip3 install git+https://github.com/podaac/cmr-association-diff.git@6193079a14e36f4c9526aa426015c2b6be41f0e2 + pip3 install python-dateutil --upgrade + - name: Run CMR Association diff scripts + run: | + cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.UAT_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }} + cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.OPS_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }} + - name: Run Add Collection Test + run: | + python3 add_collection_test.py -e uat -i ${{ env.UAT_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }} + python3 add_collection_test.py -e ops -i ${{ env.OPS_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }} + - name: Check UAT files + id: check_UAT_output_files + run: | + echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo + echo + echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo + echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + - name: Check OPS files + id: check_OPS_output_files + run: | + echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo + echo + echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)" + echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + echo + echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')" + - name: Slack Report update - Add info for UAT check - SUCCESS + if: | + steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true' + run: | + echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV + - name: Slack Report update - Add info for UAT check - FAILURE + if: | + steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' + run: | + echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV + - name: Slack Report update - Add info for OPS check - SUCCESS + if: | + steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true' + run: | + echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV + - name: Slack Report update - Add info for OPS check - FAILURE + if: | + steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' + run: | + echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV + - name: Slack Report update - Add action link on Failure + if: | + steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' || + steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' + run: | + echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nExecution link:\ngithub.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + echo ${{ env.SLACK_REPORT_CONTENT }} + - name: Send Report to Slack + uses: slackapi/slack-github-action@v1.21.0 + if: | + env.SLACK_REPORT_CONTENT != '' + with: + payload: | + { + "status": "${{ env.SLACK_REPORT_CONTENT }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_WEBHOOK }} + - name: Remove UAT collections on FAIL + if: | + steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' + run: | + python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e uat -i '${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }} + - name: Remove OPS collections on FAIL + if: | + steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true' + run: | + python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e ops -i '${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }} + - name: Update UAT collections on SUCCESS + if: | + steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true' + # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character + run: | + vi -escwq ./cmr/uat_associations.txt + cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} >> ./cmr/uat_associations.txt + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git commit -am "Update uat_associations.txt with new collections" + git push + - name: Update OPS collections on SUCCESS + if: | + steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true' + # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character + run: | + vi -escwq ./cmr/ops_associations.txt + cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} >> ./cmr/ops_associations.txt + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git commit -am "Update ops_associations.txt with new collections" + git push diff --git a/.github/workflows/jupyter_test.yml b/.github/workflows/jupyter_test.yml index 7eefcace..7ac1b70e 100644 --- a/.github/workflows/jupyter_test.yml +++ b/.github/workflows/jupyter_test.yml @@ -1,5 +1,5 @@ # This is a test pipeline, that verifies and updates the associations in cmr -name: Add Collection Test +name: Add Collection Test OLD # Controls when the workflow will run on: # Allows you to run this workflow manually from the Actions tab