Merge pull request #2221 from valory-xyz/feat/dev-mode #8279
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main_workflow | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - main | |
| pull_request: | |
| env: | |
| TENDERMINT_IMAGE_VERSION: latest | |
| jobs: | |
| lock_check: | |
| continue-on-error: False | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, macos-latest, windows-latest ] | |
| python-version: ["3.10.9"] | |
| timeout-minutes: 18 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - if: matrix.os != 'windows-latest' | |
| name: Pipenv install requirements and check it can be locked | |
| run: | | |
| pip install pipenv==2023.7.23 | |
| pipenv --clear | |
| time pipenv install --verbose --dev --skip-lock | |
| time pipenv run pip install -e .[all] | |
| time pipenv lock | |
| - if: matrix.os == 'windows-latest' | |
| name: Pipenv install requirements and check it can be locked | |
| run: | | |
| choco install ptime | |
| pip install pipenv==2023.7.23 | |
| pipenv --clear | |
| ptime pipenv install --verbose --dev --skip-lock | |
| ptime pipenv run pip install -e .[all] | |
| ptime pipenv lock | |
| install_check: | |
| continue-on-error: False | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest ] | |
| python-version: ["3.10.9"] | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check clean installation | |
| run: | | |
| pip install .[all] --no-cache | |
| autonomy --help | |
| copyright_doc_and_dependencies_check: | |
| continue-on-error: False | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.10.9"] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update --fix-missing | |
| sudo apt-get autoremove | |
| sudo apt-get autoclean | |
| pip install tomte[tox,cli]==0.2.15 | |
| pip install --user --upgrade setuptools | |
| - name: Check copyright headers | |
| run: tomte format-copyright --author valory --author fetchai | |
| - name: License compatibility check | |
| run: tox -e liccheck | |
| - name: Check docs | |
| run: tox -e check-api-docs | |
| - name: Check doc links | |
| run: tomte check-doc-links --http-skips http://host.docker.internal:8545 --http-skips http://www.fipa.org/repository/ips.php3 --url-skips "https://gateway.autonolas.tech/ipfs/<hash>," --url-skips "https://github.com/valory-xyz/open-autonomy/trunk/infrastructure" --url-skips "http://host.docker.internal:8545" --url-skips "https://github.com/valory-xyz/open-operator" | |
| - name: Check doc IPFS hashes | |
| run: tox -e check-doc-hashes | |
| - name: Check dependencies | |
| run: tox -e check-dependencies | |
| linter_checks: | |
| continue-on-error: False | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.10.9"] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.17.7" | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update --fix-missing | |
| sudo apt-get autoremove | |
| sudo apt-get autoclean | |
| pip install tomte[tox]==0.2.15 | |
| pip install --user --upgrade setuptools | |
| # install Protobuf compiler | |
| wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip | |
| unzip protoc-24.3-linux-x86_64.zip -d protoc | |
| sudo mv protoc/bin/protoc /usr/local/bin/protoc | |
| # install IPFS | |
| sudo apt-get install -y wget | |
| sudo npm install -g markdown-spellcheck | |
| wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz | |
| tar xvfz go-ipfs.tar.gz | |
| sudo mv go-ipfs/ipfs /usr/local/bin/ipfs | |
| ipfs init | |
| make protolint_install | |
| - name: Check generate protocols | |
| run: tox -e check-generate-all-protocols | |
| - name: Security checks | |
| run: | | |
| tox -e bandit | |
| tox -e safety | |
| - name: Check packages | |
| run: tox -e check-packages | |
| - name: Check hash | |
| run: tox -e check-hash | |
| - name: Code style check | |
| run: | | |
| tox -e black-check | |
| tox -e isort-check | |
| tox -e flake8 | |
| tox -e vulture | |
| tox -e darglint | |
| - name: Static type check | |
| run: tox -e mypy | |
| - name: Pylint | |
| run: tox -e pylint | |
| - name: Check spelling | |
| run: tox -e spell-check | |
| - name: AbciApp consistency checks | |
| run: | | |
| tox -e check-abci-docstrings | |
| tox -e check-abciapp-specs | |
| tox -e check-handlers | |
| tox -e check-dialogues | |
| scan: | |
| name: gitleaks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.17.7" | |
| - run: | | |
| wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \ | |
| tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \ | |
| sudo install gitleaks /usr/bin && \ | |
| gitleaks detect --report-format json --report-path leak_report -v | |
| test: | |
| continue-on-error: True | |
| needs: | |
| - lock_check | |
| - install_check | |
| - copyright_doc_and_dependencies_check | |
| - linter_checks | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, macos-latest, windows-latest ] | |
| python-version: [ "3.8", "3.9", "3.10.9", "3.11" ] | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.17.7" | |
| - if: matrix.os == 'ubuntu-latest' | |
| name: Install dependencies (ubuntu-latest) | |
| run: | | |
| sudo apt-get update --fix-missing | |
| sudo apt-get autoremove | |
| sudo apt-get autoclean | |
| pip install tomte[tox]==0.2.15 | |
| pip install --user --upgrade setuptools | |
| # install Protobuf compiler | |
| wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip | |
| unzip protoc-24.3-linux-x86_64.zip -d protoc | |
| sudo mv protoc/bin/protoc /usr/local/bin/protoc | |
| # install IPFS | |
| sudo apt-get install -y wget | |
| wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz | |
| tar xvfz go-ipfs.tar.gz | |
| sudo mv go-ipfs/ipfs /usr/local/bin/ipfs | |
| ipfs init | |
| # install Tendermint | |
| wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz | |
| tar -xf tendermint.tar.gz | |
| sudo mv tendermint /usr/local/bin/tendermint | |
| # install skaffold | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-linux-amd64 && \ | |
| sudo chmod +x skaffold | |
| sudo install skaffold /usr/local/bin/ | |
| skaffold --help | |
| # build required images | |
| skaffold build --push=false | |
| skaffold build -p ci --push=false | |
| TM_DEPLOYMENT_PATH="deployments/Dockerfiles/tendermint/" | |
| TM_IMG_NAME="valory/open-autonomy-tendermint" | |
| docker build $TM_DEPLOYMENT_PATH -t $TM_IMG_NAME:0.1.0 -t $TM_IMG_NAME:1.0.0 -t $TM_IMG_NAME:latest | |
| # pull pre-built images | |
| docker pull valory/autonolas-registries:latest | |
| docker pull valory/contracts-amm:latest | |
| docker pull valory/safe-contract-net:latest | |
| docker pull valory/acn-node:latest | |
| docker pull tendermint/tendermint:v0.34.19 | |
| docker pull trufflesuite/ganache:beta | |
| - if: matrix.os == 'ubuntu-latest' | |
| name: Framework unit tests ubuntu-latest | |
| run: | | |
| tox -e py${{ matrix.python-version }}-linux -- -m 'not e2e' | |
| - if: matrix.os == 'ubuntu-latest' | |
| name: Packages unit tests ubuntu-latest | |
| run: | | |
| tox -e packages-py${{ matrix.python-version }}-linux -- -m 'not e2e' | |
| - if: matrix.os == 'macos-latest' | |
| name: Install dependencies (macos-latest) | |
| run: | | |
| pip install tomte[tox]==0.2.15 | |
| brew install gcc | |
| # brew install protobuf | |
| # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb | |
| wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-osx-x86_64.zip | |
| unzip protoc-24.3-osx-x86_64.zip -d protoc | |
| sudo mv protoc/bin/protoc /usr/local/bin/protoc | |
| brew tap yoheimuta/protolint | |
| brew install protolint | |
| # install ipfs | |
| curl -O https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_darwin-amd64.tar.gz | |
| tar -xvzf go-ipfs_v0.6.0_darwin-amd64.tar.gz | |
| cd go-ipfs | |
| sudo ./install.sh | |
| cd .. | |
| rm -rf go-ipfs | |
| ipfs init | |
| # install Tendermint | |
| wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_darwin_amd64.tar.gz | |
| tar -xf tendermint.tar.gz | |
| sudo mv tendermint /usr/local/bin/tendermint | |
| # install skaffold | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-darwin-amd64 && \ | |
| sudo chmod +x skaffold | |
| sudo install skaffold /usr/local/bin/ | |
| skaffold --help | |
| - if: matrix.os == 'macos-latest' | |
| name: Framework unit tests macos-latest | |
| run: | | |
| tox -e py${{ matrix.python-version }}-darwin -- -m 'not e2e' | |
| - if: matrix.os == 'macos-latest' | |
| name: Packages unit tests macos-latest | |
| run: | | |
| tox -e packages-py${{ matrix.python-version }}-darwin -- -m 'not e2e' | |
| - if: matrix.os == 'windows-latest' | |
| name: Install dependencies (windows-latest) | |
| env: | |
| ACTIONS_ALLOW_UNSECURE_COMMANDS: true | |
| run: | | |
| python -m pip install -U pip | |
| echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64" | |
| choco source add -n chocolatey -s 'https://chocolatey.org/api/v2/' | |
| choco source enable -n chocolatey | |
| choco install wget -y | |
| choco install protoc --version 24.3 | |
| choco install mingw -y | |
| choco install make -y | |
| # to check make was installed | |
| make --version | |
| pip install tomte[tox]==0.2.15 | |
| # wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-win64.zip | |
| # unzip protoc-24.3-win64.zip -d protoc | |
| # sudo mv protoc/bin/protoc /usr/local/bin/protoc | |
| # TODO: install protolint | |
| #echo "::set-env name=GOPATH::$(go env GOPATH)" | |
| #echo "::add-path::$(go env GOPATH)/bin" | |
| #make protolint_install_win | |
| # just check protolint runs | |
| #protolint version | |
| # install ipfs | |
| curl https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_windows-amd64.zip -o go-ipfs_v0.6.0.zip | |
| Expand-Archive -Path go-ipfs_v0.6.0.zip -DestinationPath ~\Apps\ | |
| cp ~\Apps\go-ipfs\ipfs.exe C:\Users\runneradmin\go\bin\ | |
| ipfs init | |
| # install Tendermint | |
| wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_windows_amd64.tar.gz | |
| tar -xvzf tendermint.tar.gz | |
| cp tendermint.exe C:\Users\runneradmin\go\bin\ | |
| # install skaffold | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-windows-amd64.exe | |
| cp skaffold C:\Users\runneradmin\go\bin\ | |
| skaffold --help | |
| - if: matrix.os == 'windows-latest' | |
| name: Framework unit tests windows-latest | |
| run: | | |
| tox -e py${{ matrix.python-version }}-win -- -m 'not e2e' | |
| - if: matrix.os == 'windows-latest' | |
| name: Packages unit tests windows-latest | |
| run: | | |
| tox -e packages-py${{ matrix.python-version }}-win -- -m 'not e2e' | |
| - if: matrix.os == 'ubuntu-latest' | |
| name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| e2e: | |
| continue-on-error: True | |
| needs: | |
| - lock_check | |
| - install_check | |
| - copyright_doc_and_dependencies_check | |
| - linter_checks | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest ] | |
| python-version: [ "3.10.9" ] | |
| timeout-minutes: 150 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@master | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update --fix-missing | |
| sudo apt-get autoremove | |
| sudo apt-get autoclean | |
| pip install tomte[tox]==0.2.15 | |
| pip install --user --upgrade setuptools | |
| # install Protobuf compiler | |
| wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip | |
| unzip protoc-24.3-linux-x86_64.zip -d protoc | |
| sudo mv protoc/bin/protoc /usr/local/bin/protoc | |
| # install IPFS | |
| sudo apt-get install -y wget | |
| wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz | |
| tar xvfz go-ipfs.tar.gz | |
| sudo mv go-ipfs/ipfs /usr/local/bin/ipfs | |
| ipfs init | |
| # install Tendermint | |
| wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz | |
| tar -xf tendermint.tar.gz | |
| sudo mv tendermint /usr/local/bin/tendermint | |
| # install Skaffold needed in order to build the Flask with Tendermint image | |
| curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-linux-amd64 &&\ | |
| sudo install skaffold /usr/local/bin/ | |
| # build the Flask with Tendermint image | |
| TM_DEPLOYMENT_PATH="deployments/Dockerfiles/tendermint/" | |
| TM_IMG_NAME="valory/open-autonomy-tendermint" | |
| docker build $TM_DEPLOYMENT_PATH -t $TM_IMG_NAME:0.1.0 -t $TM_IMG_NAME:1.0.0 -t $TM_IMG_NAME:latest | |
| # pull pre-built images | |
| docker pull valory/autonolas-registries:latest | |
| docker pull valory/contracts-amm:latest | |
| docker pull valory/safe-contract-net:latest | |
| docker pull tendermint/tendermint:v0.34.19 | |
| docker pull valory/slow-tendermint-server:0.1.0 | |
| - name: Unit tests | |
| run: | | |
| tox --develop -e e2e-py${{ matrix.python-version }}-linux -- -m 'e2e' |