main_workflow.yml

name: main_workflow

on:
  push:
    branches:
      - develop
      - main
  pull_request:

env:
  TENDERMINT_IMAGE_VERSION: latest

jobs:
  lock_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: ["3.10.9"]

    timeout-minutes: 18

    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@master
      with:
        python-version: ${{ matrix.python-version }}
    - if: matrix.os != 'windows-latest'
      name: Pipenv install requirements and check it can be locked
      run: |
        pip install pipenv==2023.7.23
        pipenv --clear
        time pipenv install --verbose --dev --skip-lock
        time pipenv run pip install -e .[all]
        time pipenv lock
    - if: matrix.os == 'windows-latest'
      name: Pipenv install requirements and check it can be locked
      run: |
        choco install ptime
        pip install pipenv==2023.7.23
        pipenv --clear
        ptime pipenv install --verbose --dev --skip-lock
        ptime pipenv run pip install -e .[all]
        ptime pipenv lock

  install_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: ["3.10.9"]
    timeout-minutes: 10
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@master
      with:
        python-version: ${{ matrix.python-version }}
    - name: Check clean installation
      run: |
        pip install .[all] --no-cache
        autonomy --help

  copyright_doc_and_dependencies_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10.9"]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - uses: actions/setup-python@master
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run:  |
        sudo apt-get update --fix-missing
        sudo apt-get autoremove
        sudo apt-get autoclean
        pip install tomte[tox,cli]==0.2.15
        pip install --user --upgrade setuptools
    - name: Check copyright headers
      run: tomte format-copyright --author valory --author fetchai
    - name: License compatibility check
      run: tox -e liccheck
    - name: Check docs
      run: tox -e check-api-docs
    - name: Check doc links
      run: tomte check-doc-links --http-skips http://host.docker.internal:8545 --http-skips http://www.fipa.org/repository/ips.php3 --url-skips "https://gateway.autonolas.tech/ipfs/<hash>," --url-skips "https://github.com/valory-xyz/open-autonomy/trunk/infrastructure" --url-skips "http://host.docker.internal:8545" --url-skips "https://github.com/valory-xyz/open-operator"
    - name: Check doc IPFS hashes
      run: tox -e check-doc-hashes
    - name: Check dependencies
      run: tox -e check-dependencies

  linter_checks:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10.9"]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@master
      with:
        python-version: ${{ matrix.python-version }}
    - uses: actions/setup-go@v3
      with:
        go-version: "1.17.7"
    - name: Install dependencies
      run:  |
        sudo apt-get update --fix-missing
        sudo apt-get autoremove
        sudo apt-get autoclean
        pip install tomte[tox]==0.2.15
        pip install --user --upgrade setuptools
        # install Protobuf compiler
        wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
        unzip protoc-24.3-linux-x86_64.zip -d protoc
        sudo mv protoc/bin/protoc /usr/local/bin/protoc
        # install IPFS
        sudo apt-get install -y wget
        sudo npm install -g markdown-spellcheck
        wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
        tar xvfz go-ipfs.tar.gz
        sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
        ipfs init
        make protolint_install
    - name: Check generate protocols
      run: tox -e check-generate-all-protocols
    - name: Security checks
      run: |
        tox -e bandit
        tox -e safety
    - name: Check packages
      run: tox -e check-packages
    - name: Check hash
      run: tox -e check-hash
    - name: Code style check
      run: |
        tox -e black-check
        tox -e isort-check
        tox -e flake8
        tox -e vulture
        tox -e darglint
    - name: Static type check
      run: tox -e mypy
    - name: Pylint
      run: tox -e pylint
    - name: Check spelling
      run: tox -e spell-check
    - name: AbciApp consistency checks
      run: |
        tox -e check-abci-docstrings
        tox -e check-abciapp-specs
        tox -e check-handlers
        tox -e check-dialogues

  scan:
    name: gitleaks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - run: |
          wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \
          tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \
          sudo install gitleaks /usr/bin && \
          gitleaks detect --report-format json --report-path leak_report -v

  test:
    continue-on-error: True
    needs:
      - lock_check
      - install_check
      - copyright_doc_and_dependencies_check
      - linter_checks
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.8", "3.9", "3.10.9", "3.11" ]

    timeout-minutes: 90

    steps:
      - uses: actions/checkout@v2
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: actions/setup-python@master
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - if: matrix.os == 'ubuntu-latest'
        name: Install dependencies (ubuntu-latest)
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox]==0.2.15
          pip install --user --upgrade setuptools

          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # install IPFS
          sudo apt-get install -y wget
          wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
          tar xvfz go-ipfs.tar.gz
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

          # install skaffold
          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-linux-amd64 && \
          sudo chmod +x skaffold
          sudo install skaffold /usr/local/bin/
          skaffold --help

          # build required images
          skaffold build --push=false
          skaffold build -p ci --push=false
          TM_DEPLOYMENT_PATH="deployments/Dockerfiles/tendermint/"
          TM_IMG_NAME="valory/open-autonomy-tendermint"
          docker build $TM_DEPLOYMENT_PATH -t $TM_IMG_NAME:0.1.0 -t $TM_IMG_NAME:1.0.0 -t $TM_IMG_NAME:latest

          # pull pre-built images
          docker pull valory/autonolas-registries:latest
          docker pull valory/contracts-amm:latest
          docker pull valory/safe-contract-net:latest
          docker pull valory/acn-node:latest
          docker pull tendermint/tendermint:v0.34.19
          docker pull trufflesuite/ganache:beta

      - if: matrix.os == 'ubuntu-latest'
        name: Framework unit tests ubuntu-latest
        run: |
          tox -e py${{ matrix.python-version }}-linux -- -m 'not e2e'

      - if: matrix.os == 'ubuntu-latest'
        name: Packages unit tests ubuntu-latest
        run: |
          tox -e packages-py${{ matrix.python-version }}-linux -- -m 'not e2e'

      - if: matrix.os == 'macos-latest'
        name: Install dependencies (macos-latest)
        run: |
          pip install tomte[tox]==0.2.15
          brew install gcc
          # brew install protobuf
          # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-osx-x86_64.zip
          unzip protoc-24.3-osx-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc
          brew tap yoheimuta/protolint
          brew install protolint

          # install ipfs
          curl -O https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_darwin-amd64.tar.gz
          tar -xvzf go-ipfs_v0.6.0_darwin-amd64.tar.gz
          cd go-ipfs
          sudo ./install.sh
          cd ..
          rm -rf go-ipfs
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_darwin_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

          # install skaffold
          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-darwin-amd64 && \
          sudo chmod +x skaffold
          sudo install skaffold /usr/local/bin/
          skaffold --help

      - if: matrix.os == 'macos-latest'
        name: Framework unit tests macos-latest
        run: |
          tox -e py${{ matrix.python-version }}-darwin -- -m 'not e2e'

      - if: matrix.os == 'macos-latest'
        name: Packages unit tests macos-latest
        run: |
          tox -e packages-py${{ matrix.python-version }}-darwin -- -m 'not e2e'

      - if: matrix.os == 'windows-latest'
        name: Install dependencies (windows-latest)
        env:
          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
        run: |
          python -m pip install -U pip
          echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64"

          choco source add -n chocolatey -s 'https://chocolatey.org/api/v2/'
          choco source enable -n chocolatey

          choco install wget -y
          choco install protoc --version 24.3
          choco install mingw -y
          choco install make -y
          # to check make was installed
          make --version
          pip install tomte[tox]==0.2.15
          # wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-win64.zip
          # unzip protoc-24.3-win64.zip -d protoc
          # sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # TODO: install protolint
          #echo "::set-env name=GOPATH::$(go env GOPATH)"
          #echo "::add-path::$(go env GOPATH)/bin"
          #make protolint_install_win
          # just check protolint runs
          #protolint version

          # install ipfs
          curl https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_windows-amd64.zip -o go-ipfs_v0.6.0.zip
          Expand-Archive -Path go-ipfs_v0.6.0.zip -DestinationPath ~\Apps\
          cp ~\Apps\go-ipfs\ipfs.exe C:\Users\runneradmin\go\bin\
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_windows_amd64.tar.gz
          tar -xvzf tendermint.tar.gz
          cp tendermint.exe C:\Users\runneradmin\go\bin\

          # install skaffold
          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-windows-amd64.exe
          cp skaffold C:\Users\runneradmin\go\bin\
          skaffold --help

      - if: matrix.os == 'windows-latest'
        name: Framework unit tests windows-latest
        run: |
          tox -e py${{ matrix.python-version }}-win -- -m 'not e2e'

      - if: matrix.os == 'windows-latest'
        name: Packages unit tests windows-latest
        run: |
          tox -e packages-py${{ matrix.python-version }}-win -- -m 'not e2e'

      - if: matrix.os == 'ubuntu-latest'
        name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false

  e2e:
    continue-on-error: True
    needs:
      - lock_check
      - install_check
      - copyright_doc_and_dependencies_check
      - linter_checks
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10.9" ]

    timeout-minutes: 150

    steps:
      - uses: actions/checkout@v2
        with:
          submodules: recursive
      - uses: actions/setup-python@master
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox]==0.2.15
          pip install --user --upgrade setuptools

          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # install IPFS
          sudo apt-get install -y wget
          wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
          tar xvfz go-ipfs.tar.gz
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

          # install Skaffold needed in order to build the Flask with Tendermint image
          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v1.39.1/skaffold-linux-amd64 &&\
            sudo install skaffold /usr/local/bin/

          # build the Flask with Tendermint image
          TM_DEPLOYMENT_PATH="deployments/Dockerfiles/tendermint/"
          TM_IMG_NAME="valory/open-autonomy-tendermint"
          docker build $TM_DEPLOYMENT_PATH -t $TM_IMG_NAME:0.1.0 -t $TM_IMG_NAME:1.0.0 -t $TM_IMG_NAME:latest

          # pull pre-built images
          docker pull valory/autonolas-registries:latest
          docker pull valory/contracts-amm:latest
          docker pull valory/safe-contract-net:latest
          docker pull tendermint/tendermint:v0.34.19
          docker pull valory/slow-tendermint-server:0.1.0

      - name: Unit tests
        run: |
          tox --develop -e e2e-py${{ matrix.python-version }}-linux -- -m 'e2e'