[resoto][chore] Use pyproject.toml instead of setup.py (#1613)

someengineering · May 30, 2023 · 368c1cc · 368c1cc
1 parent b709ed1
commit 368c1cc
Show file tree

Hide file tree

Showing 232 changed files with 2,436 additions and 1,798 deletions.
diff --git a/.github/workflows/check_pr_plugin_aws.yml b/.github/workflows/check_pr_plugin_aws.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/aws/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/aws/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_aws_k8s.yml b/.github/workflows/check_pr_plugin_aws_k8s.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/aws_k8s/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/aws_k8s/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_alarms.yml b/.github/workflows/check_pr_plugin_cleanup_aws_alarms.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_alarms/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_alarms/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml b/.github/workflows/check_pr_plugin_cleanup_aws_loadbalancers.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_loadbalancers/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_loadbalancers/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml b/.github/workflows/check_pr_plugin_cleanup_aws_vpcs.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_vpcs/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_aws_vpcs/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_expired.yml b/.github/workflows/check_pr_plugin_cleanup_expired.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_expired/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_expired/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_untagged.yml b/.github/workflows/check_pr_plugin_cleanup_untagged.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_untagged/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_untagged/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_cleanup_volumes.yml b/.github/workflows/check_pr_plugin_cleanup_volumes.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_volumes/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/cleanup_volumes/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_digitalocean.yml b/.github/workflows/check_pr_plugin_digitalocean.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/digitalocean/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/digitalocean/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_digitalocean_k8s.yml b/.github/workflows/check_pr_plugin_digitalocean_k8s.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/digitalocean_k8s/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/digitalocean_k8s/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_dockerhub.yml b/.github/workflows/check_pr_plugin_dockerhub.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/dockerhub/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/dockerhub/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_example_collector.yml b/.github/workflows/check_pr_plugin_example_collector.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/example_collector/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/example_collector/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_gcp.yml b/.github/workflows/check_pr_plugin_gcp.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/gcp/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/gcp/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_github.yml b/.github/workflows/check_pr_plugin_github.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/github/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/github/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_k8s.yml b/.github/workflows/check_pr_plugin_k8s.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/k8s/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/k8s/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_onelogin.yml b/.github/workflows/check_pr_plugin_onelogin.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/onelogin/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/onelogin/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_onprem.yml b/.github/workflows/check_pr_plugin_onprem.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/onprem/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/onprem/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_posthog.yml b/.github/workflows/check_pr_plugin_posthog.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/posthog/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/posthog/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_protector.yml b/.github/workflows/check_pr_plugin_protector.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/protector/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/protector/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_random.yml b/.github/workflows/check_pr_plugin_random.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/random/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/random/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_scarf.yml b/.github/workflows/check_pr_plugin_scarf.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/scarf/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/scarf/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_slack.yml b/.github/workflows/check_pr_plugin_slack.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/slack/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/slack/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_tagvalidator.yml b/.github/workflows/check_pr_plugin_tagvalidator.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/tagvalidator/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/tagvalidator/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_plugin_vsphere.yml b/.github/workflows/check_pr_plugin_vsphere.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./plugins/vsphere/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./plugins/vsphere/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/check_pr_resotolib.yml b/.github/workflows/check_pr_resotolib.yml
@@ -28,7 +28,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{hashFiles('./resotolib/requirements.txt')}}-${{hashFiles('./resotolib/requirements-test.txt')}}
+          key: ${{ runner.os }}-pip-${{hashFiles('./resotolib/pyproject.toml')}}-${{hashFiles('./resotolib/requirements-test.txt')}}
           restore-keys: |
             ${{ runner.os }}-pip-
 

diff --git a/.github/workflows/check_pr_resotometrics.yml b/.github/workflows/check_pr_resotometrics.yml
@@ -29,7 +29,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{hashFiles('./resotometrics/requirements.txt')}}
+          key: ${{ runner.os }}-pip-${{hashFiles('./resotometrics/pyproject.toml')}}
           restore-keys: |
             ${{ runner.os }}-pip-
 

diff --git a/.github/workflows/check_pr_resotoshell.yml b/.github/workflows/check_pr_resotoshell.yml
@@ -29,7 +29,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('./resotoshell/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./resotoshell/pyproject.toml')}}
           restore-keys: |
             ${{ runner.os }}-pip-
 

diff --git a/.github/workflows/check_pr_resotoworker.yml b/.github/workflows/check_pr_resotoworker.yml
@@ -29,7 +29,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{hashFiles('./resotoworker/requirements.txt')}}
+          key: ${{ runner.os }}-pip-${{hashFiles('./resotoworker/pyproject.toml')}}
           restore-keys: |
             ${{ runner.os }}-pip-
 

diff --git a/.github/workflows/create_plugin_workflows.py b/.github/workflows/create_plugin_workflows.py
@@ -38,7 +38,7 @@
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{runner.os}}-pip-${{hashFiles('@directory@/requirements.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('@directory@/pyproject.toml')}}
           restore-keys: |
             ${{runner.os}}-pip-
 

diff --git a/.github/workflows/model_check.yml b/.github/workflows/model_check.yml
@@ -38,30 +38,16 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{hashFiles('./resotolib/requirements.txt')}}-${{hashFiles('./resotolib/requirements-test.txt')}}-${{hashFiles('./plugins/**/requirements.txt')}}
+          key: ${{ runner.os }}-pip-${{hashFiles('./resotolib/pyproject.toml')}}-${{hashFiles('./resotolib/requirements-test.txt')}}-${{hashFiles('./plugins/**/pyproject.toml')}}
 
           restore-keys: |
             ${{ runner.os }}-pip-
 
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade --editable resotolib/
-          pip install --upgrade --editable plugins/aws/
-          pip install --upgrade --editable plugins/digitalocean/
-          pip install --upgrade --editable plugins/dockerhub/
-          pip install --upgrade --editable plugins/example_collector/
-          pip install --upgrade --editable plugins/gcp/
-          pip install --upgrade --editable plugins/github/
-          pip install --upgrade --editable plugins/k8s/
-          pip install --upgrade --editable plugins/onelogin/
-          pip install --upgrade --editable plugins/onprem/
-          pip install --upgrade --editable plugins/posthog/
-          pip install --upgrade --editable plugins/random/
-          pip install --upgrade --editable plugins/scarf/
-          pip install --upgrade --editable plugins/slack/
-          pip install --upgrade --editable plugins/vsphere/
-          pip install pytest
+          python -m pip install -r requirements-test.txt
+          pip install resotolib/ plugins/aws/ plugins/digitalocean/ plugins/dockerhub/ plugins/example_collector/ plugins/gcp/ plugins/github/ plugins/k8s/ plugins/onelogin/ plugins/onprem/ plugins/posthog/ plugins/random/ plugins/scarf/ plugins/slack/ plugins/vsphere/
 
       - name: Run tests
         working-directory: ./resotolib

diff --git a/.github/workflows/resotocore_coverage.yml b/.github/workflows/resotocore_coverage.yml
@@ -41,16 +41,16 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/requirements.txt')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/pyproject.toml')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade --editable ../resotolib
-          pip install -r requirements-test.txt
-          pip install -r requirements.txt
+          pip install -r ../requirements-test.txt
+          pip install -r ../requirements.txt
       - name: Run Tests
         env:
           RESOTOCORE_ANALYTICS_OPT_OUT: true

diff --git a/.github/workflows/resotocore_lint.yml b/.github/workflows/resotocore_lint.yml
@@ -29,7 +29,7 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/requirements.txt')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/pyproject.toml')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
       - name: Install Dependencies
         run: ./setup_venv.sh --dev --no-venv --no-plugins
       - name: Check Formatting

diff --git a/.github/workflows/resotocore_test_and_build.yml b/.github/workflows/resotocore_test_and_build.yml
@@ -47,17 +47,17 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/requirements.txt')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
+          key: ${{runner.os}}-pip-${{hashFiles('./resotocore/pyproject.toml')}}-${{hashFiles('./resotocore/requirements-dev.txt')}}-${{hashFiles('./resotocore/requirements-test.txt')}}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade --editable ../resotolib
-          pip install -r requirements-test.txt
-          pip install -r requirements-dev.txt
-          pip install -r requirements.txt
+          pip install -r ../requirements-test.txt
+          pip install -r ../requirements-dev.txt
+          pip install -r ../requirements.txt
       - name: Run Tests
         env:
           RESOTOCORE_ANALYTICS_OPT_OUT: true

diff --git a/Dockerfile.resotobase b/Dockerfile.resotobase
@@ -21,6 +21,7 @@ RUN echo "I am running on ${BUILDPLATFORM}, building for ${TARGETPLATFORM}"
 RUN mkdir -p /build /build-python
 
 # Build Resoto
+COPY requirements.txt /usr/src/requirements.txt
 COPY resotolib /usr/src/resotolib
 COPY resotocore /usr/src/resotocore
 COPY resotoworker /usr/src/resotoworker
@@ -35,10 +36,8 @@ RUN . /build/jupyterlite-venv-python3/bin/activate && python -m jupyter lite bui
 
 # Build binary wheels
 WORKDIR /usr/src
-RUN . /usr/local/resoto-venv-python3/bin/activate && find plugins/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 python -m pip wheel -w /build-python -f /build-python ./resotolib ./resotocore ./resotoworker ./resotometrics ./resotoshell
-
-# Install all wheels
-RUN . /usr/local/resoto-venv-python3/bin/activate && python -m pip install -f /build-python /build-python/*.whl
+RUN . /usr/local/resoto-venv-python3/bin/activate && pip install -r requirements.txt
+RUN . /usr/local/resoto-venv-python3/bin/activate && find plugins/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 python -m pip install ./resotolib ./resotocore ./resotoworker ./resotometrics ./resotoshell
 
 # Download and install ArangoDB client on x86 builds (there are no official ArangoDB binaries for arm64)
 WORKDIR /tmp/arangodb

diff --git a/bump.sh b/bump.sh
@@ -12,7 +12,7 @@ bump_to=$2
 git ls-files | grep \
     -e setup.py \
     -e __init__.py \
-    -e requirements.txt \
+    -e pyproject.toml \
 | xargs grep "$bump_from" \
 | cut -d : -f 1 \
 | xargs sed -i -e "s/$bump_from/$bump_to/g"

diff --git a/plugins/aws/MANIFEST.in b/plugins/aws/MANIFEST.in
@@ -1,2 +1 @@
 include README.md
-include requirements.txt