Merge pull request #419 from grapl-security/staging

Merge staging --> master
grapl-security · Nov 11, 2020 · 348bc95 · 348bc95
2 parents 81ac08a + 61d38de
commit 348bc95
Show file tree

Hide file tree

Showing 264 changed files with 11,910 additions and 47,153 deletions.
diff --git a/.github/workflows/grapl-build.yml b/.github/workflows/grapl-build.yml
@@ -25,10 +25,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Build Rust services
         run: |
@@ -44,10 +44,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Build Python services
         run: |
@@ -63,10 +63,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Run dobi python-typecheck
         run: |
@@ -78,10 +78,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Build JS services
         run: |
@@ -97,10 +97,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Build Grapl
         run: |
@@ -110,23 +110,53 @@ jobs:
         run: |
           GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount integration-tests
 
+
   # In the future, this should probably be merged back into `dobi integration-tests`, but
   # since it's so timing-dependent I'm going to treat it separately until it stabilizes a bit.
   end-to-end-integration-tests:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        python-version: [3.7]
+
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Build Grapl
         run: |
           GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount build
 
+      # --no-bind-mount makes dobi screw up with volumes for some reason, and it's not really required here anyway.
       - name: Run end-to-end integration tests
         run: |
-          GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount run-e2e-integration-tests
+          GRAPL_RELEASE_TARGET=debug \
+          GRAPL_LOG_LEVEL=DEBUG \
+          DUMP_ARTIFACTS=True \
+          TAG=latest \
+          ./dobi-linux run-e2e-integration-tests
+       
+      # NOTE: This requires >= py37
+      - name: 'Collect e2e test artifacts'
+        if: ${{ always() }}
+        run: |
+          python3 ./etc/ci_scripts/dump_compose_artifacts.py --compose-project "grapl-integration-tests"
+           
+      - name: 'Upload e2e test artifacts'
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-artifacts
+          # this path is specified in dump_compose_artifacts.py
+          path: /tmp/compose_artifacts/
+          retention-days: 28
diff --git a/.github/workflows/grapl-lint.yml b/.github/workflows/grapl-lint.yml
@@ -44,13 +44,19 @@ jobs:
           python3 -mvenv .venv
           source .venv/bin/activate
           python -mpip install --upgrade pip
-          pip install black mypy pypi-simple
+          pip install black isort
 
       - name: Check python formatting
         run: |
           source .venv/bin/activate
           black --check .
 
+      - name: Check python isort
+        run: |
+          source .venv/bin/activate
+          cd src/python
+          isort --diff --check-only .
+
   check-pypi:
     runs-on: ubuntu-latest
 
@@ -72,48 +78,48 @@ jobs:
           python3 -mvenv venv && . venv/bin/activate
           pip install pypi-simple
           deactivate
-
-      - name: Check whether grapl_graph_descriptions version has been bumped
+      
+      - name: Export CHECK_PYPI_VERSION_ARGS as "true" if channel is not latest
+        # passing 'true' to check_pypi means "check test.pypi.org"
         run: |
-          . venv/bin/activate
           if [[ "$CHANNEL" == "latest" ]]; then
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_graph_descriptions \
-                $(cat src/rust/graph-descriptions/VERSION)
+            echo "CHECK_PYPI_VERSION_ARGS=false" >> $GITHUB_ENV
           else
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_graph_descriptions \
-                $(cat src/rust/graph-descriptions/VERSION) \
-                true
+            echo "CHECK_PYPI_VERSION_ARGS=true" >> $GITHUB_ENV
           fi
+
+      - name: Check whether grapl_graph_descriptions version has been bumped
+        run: |
+          . venv/bin/activate
+          python etc/build_scripts/check_pypi_version.py \
+            grapl_graph_descriptions \
+            $(cat src/rust/graph-descriptions/VERSION) \
+            $CHECK_PYPI_VERSION_ARGS
           deactivate
 
       - name: Check whether grapl_analyzerlib version has been bumped
         run: |
           . venv/bin/activate
-          if [[ "$CHANNEL" == "latest" ]]; then
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_analyzerlib \
-                $(cat src/python/grapl_analyzerlib/VERSION)
-          else
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_analyzerlib \
-                $(cat src/python/grapl_analyzerlib/VERSION) \
-                true
-          fi
+          python etc/build_scripts/check_pypi_version.py \
+            grapl_analyzerlib \
+            $(cat src/python/grapl_analyzerlib/VERSION) \
+            $CHECK_PYPI_VERSION_ARGS
           deactivate
 
       - name: Check whether grapl-common version has been bumped
         run: |
           . venv/bin/activate
-          if [[ "$CHANNEL" == "latest" ]]; then
-              python etc/build_scripts/check_pypi_version.py \
-                grapl-common \
-                $(cat src/python/grapl-common/VERSION)
-          else
-              python etc/build_scripts/check_pypi_version.py \
-                grapl-common \
-                $(cat src/python/grapl-common/VERSION) \
-                true
-          fi
+          python etc/build_scripts/check_pypi_version.py \
+            grapl-common \
+            $(cat src/python/grapl-common/VERSION) \
+            $CHECK_PYPI_VERSION_ARGS
+          deactivate
+
+      - name: Check whether grapl-tests-common version has been bumped
+        run: |
+          . venv/bin/activate
+          python etc/build_scripts/check_pypi_version.py \
+            grapl-tests-common \
+            $(cat src/python/grapl-tests-common/VERSION) \
+            $CHECK_PYPI_VERSION_ARGS
           deactivate
diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml
@@ -12,10 +12,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Run unit tests
         run: |
@@ -28,10 +28,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Run integration tests
         run: |
@@ -112,10 +112,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
       
       - name: Determine release channel
         run: |
@@ -232,10 +232,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Determine release channel
         run: |
@@ -394,7 +394,7 @@ jobs:
       - name: Prepare grapl-common dist
         run: |
           docker create -ti --name grapl-common grapl/grapl-common-python-build:latest
-          docker cp grapl-common:/home/grapl/grapl-common/dist .
+          docker cp grapl-common:/home/grapl/grapl_common/dist .
           docker rm -f grapl-common
 
       - name: Upload grapl-common to Test PyPI
@@ -416,6 +416,31 @@ jobs:
         run: |
           rm -rf dist
 
+      - name: Prepare grapl-tests-common dist
+        run: |
+          docker create -ti --name grapl-tests-common grapl/grapl-tests-common-python-build:latest
+          docker cp grapl-tests-common:/home/grapl/grapl-tests-common/dist .
+          docker rm -f grapl-tests-common
+
+      - name: Upload grapl-tests-common to Test PyPI
+        if: env.CHANNEL == 'beta'
+        uses: pypa/gh-action-pypi-publish@v1.1.0
+        with:
+          user: __token__
+          password: ${{ secrets.GRAPL_ANALYZERLIB_TEST_PYPI_TOKEN }}
+          repository_url: https://test.pypi.org/legacy/
+
+      - name: Upload grapl-tests-common to PyPI
+        if: env.CHANNEL == 'latest'
+        uses: pypa/gh-action-pypi-publish@v1.1.0
+        with:
+          user: __token__
+          password: ${{ secrets.GRAPL_ANALYZERLIB_PYPI_TOKEN }}
+
+      - name: Clean up grapl-tests-common dist
+        run: |
+          rm -rf dist
+
 
 
   release-js-services:
@@ -426,10 +451,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Determine release channel
         run: |

diff --git a/.github/workflows/grapl-staging-release.yml b/.github/workflows/grapl-staging-release.yml
@@ -14,10 +14,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Export tag
         run: |
@@ -45,10 +45,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
 
       - name: Export tag
         run: |
@@ -77,10 +77,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install dobi
+      - name: Prepare Github Actions CI
         run: |
-          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
-          chmod +x dobi-linux
+          ./etc/ci_scripts/clean_gh_actions_space.sh
+          ./etc/ci_scripts/install_requirements.sh
     
       - name: Export tag
         run: |