main_workflow.yml

name: main_workflow

on:
  push:
    branches:
      - develop
      - main
  pull_request:

jobs:
  lock_check:
    continue-on-error: False
    runs-on: ubuntu-latest

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 10

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          pip install pipenv==2023.7.23
      - name: Pipenv install requirements
        run: |
          pipenv install --dev --skip-lock
      - name: Pipenv lock
        run: |
          pipenv lock

  copyright_and_doc_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox,cli]==0.2.14
          pip install --upgrade setuptools==60.10.0
      - name: Check copyright headers
        run: tomte check-copyright --author valory --exclude-part abci  --exclude-part http_client  --exclude-part ipfs  --exclude-part ledger  --exclude-part p2p_libp2p_client  --exclude-part gnosis_safe  --exclude-part gnosis_safe_proxy_factory  --exclude-part multisend  --exclude-part service_registry  --exclude-part acn  --exclude-part contract_api  --exclude-part http  --exclude-part ipfs  --exclude-part ledger_api --exclude-part tendermint --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci  --exclude-part http_server
      - name: License compatibility check
        run: tox -e liccheck
      - name: Check doc links
        run: tomte check-doc-links --url-skips "https://goerli.infura.io/v3/<infura_api_key>" --url-skips "https://twitter.com/autonolas" --url-skips "https://developer.twitter.com/en/portal/dashboard" --url-skips "https://ceramic-clay.3boxlabs.com/" --url-skips "https://api.twitter.com/" --url-skips "http://host.docker.internal:5000/twitter/create_tweet"  --http-skips "http://host.docker.internal:5000/twitter/create_tweet"  --url-skips "https://safe-transaction-goerli.safe.global/api/v1/messages/{message_hash}/"
      - name: Check doc IPFS hashes
        run: tox -e check-doc-hashes

  linter_checks:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10.6" ]

    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox,cli]==0.2.14
          pip install --upgrade setuptools==60.10.0
          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc
          # install IPFS
          sudo apt-get install -y wget
          sudo npm install -g markdown-spellcheck
          wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
          tar xvfz go-ipfs.tar.gz
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
          ipfs init
      - name: Security checks
        run: tomte check-security
      - name: Check packages
        run: tox -e check-packages
      - name: Check FSM Spec
        run: tox -e check-abciapp-specs
      - name: Check hash
        run: tox -e check-hash
      - name: Code checks
        run: tomte check-code
      - name: Check spelling
        run: tomte check-spelling

  scan:
    name: gitleaks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - run: |
          wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \
          tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \
          sudo install gitleaks /usr/bin && \
          gitleaks detect --report-format json --report-path leak_report -v

  test:
    continue-on-error: True
    needs:
      - lock_check
      - copyright_and_doc_check
      - linter_checks
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.8", "3.9", "3.10", "3.11" ]

    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          submodules: recursive
          python-version: ${{ matrix.python-version }}
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - if: matrix.os == 'ubuntu-latest'
        name: Install dependencies (ubuntu-latest)
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox]==0.2.14
          pip install --upgrade setuptools==60.10.0

          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # install IPFS
          sudo apt-get install -y wget
          wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
          tar xvfz go-ipfs.tar.gz
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

          # pull node image needed for the integration tests
          docker pull node:16.7.0
          docker pull trufflesuite/ganache:beta
          docker pull valory/autonomous-fund-contracts:latest

      - if: matrix.os == 'ubuntu-latest'
        name: Unit tests (ubuntu-latest)
        run: |
          tox -e py${{ matrix.python-version }}-linux -- -m 'not e2e'

      - if: matrix.os == 'macos-latest'
        name: Install dependencies (macos-latest)
        run: |
          pip install tomte[tox]==0.2.14
          brew install gcc
          # brew install protobuf
          # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-osx-x86_64.zip
          unzip protoc-24.3-osx-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc
          brew tap yoheimuta/protolint
          brew install protolint

          # install ipfs
          curl -O https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_darwin-amd64.tar.gz
          tar -xvzf go-ipfs_v0.6.0_darwin-amd64.tar.gz
          cd go-ipfs
          sudo ./install.sh
          cd ..
          rm -rf go-ipfs
          ipfs init

      - if: matrix.os == 'macos-latest'
        name: Skills unit tests macos-latest
        run: |
          tox -e py${{ matrix.python-version }}-darwin -- -m 'not e2e'

      - if: matrix.os == 'windows-latest'
        name: Install dependencies (windows-latest)
        env:
          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
        run: |
          python -m pip install -U pip
          echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64"
          choco install wget -y
          choco install protoc --version 24.3
          choco install mingw -y
          choco install make -y
          # to check make was installed
          make --version
          pip install tomte[tox]==0.2.14
          # wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-win64.zip
          # unzip protoc-24.3-win64.zip -d protoc
          # sudo mv protoc/bin/protoc /usr/local/bin/protoc

          python scripts\update_symlinks_cross_platform.py

          # install ipfs
          curl https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_windows-amd64.zip -o go-ipfs_v0.6.0.zip
          Expand-Archive -Path go-ipfs_v0.6.0.zip -DestinationPath ~\Apps\
          cp ~\Apps\go-ipfs\ipfs.exe C:\Users\runneradmin\go\bin\
          ipfs init

      - if: matrix.os == 'windows-latest'
        name: Skills unit tests windows-latest
        run: |
          tox -e py${{ matrix.python-version }}-win -- -m 'not e2e'

      - if: matrix.os == 'ubuntu-latest'
        name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false

  e2e-tests:
    continue-on-error: True
    needs:
      - test
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          submodules: recursive
          python-version: ${{ matrix.python-version }}
      - if: matrix.os == 'ubuntu-latest'
        name: Install dependencies (ubuntu-latest)
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox]==0.2.14
          pip install --upgrade setuptools==60.10.0

          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # install IPFS
          sudo apt-get install -y wget
          wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
          tar xvfz go-ipfs.tar.gz
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
          ipfs init

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

          # pull node image needed for the integration tests
          docker pull node:16.7.0

          docker pull valory/autonomous-fund-contracts:latest

      - name: e2e tests
        run: |
          tox -e e2e-py${{ matrix.python-version }}-linux