Merge branch 'tests/test_data_types' into fix/data_types__eq__

.github/pull_request_template.md

@@ -50,7 +50,7 @@ _Put an `x` in the boxes that apply._
 - [ ] I have read the [CONTRIBUTING](../CONTRIBUTING.md) doc
 - [ ] I am making a pull request against the `main` branch (left side), from `develop`
 - [ ] Lint and unit tests pass locally and in CI
-- [ ] I have checked the fingerprint hashes are correct by running (`aea hash all` and `aea packages lock --check`)
+- [ ] I have checked the fingerprint hashes are correct by running (`aea packages lock --check`)
 - [ ] I have regenerated the latest API docs
 - [ ] I built the documentation and updated it with the latest changes
 - [ ] I have added an item in `HISTORY.md` for this release

.github/workflows/workflow.yml

@@ -63,7 +63,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Security Check
         run: tox -e bandit
       - name: Safety Check
@@ -102,7 +102,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Pylint check
         run: |
           tox -e pylint
@@ -116,12 +116,15 @@ jobs:
       - uses: actions/setup-python@v3
         with:
           python-version: "3.10"
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.17.7"
       - name: Install dependencies (ubuntu-latest)
         run: |
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
           pip install --user --upgrade setuptools
           # install Protobuf compiler
           wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
@@ -163,7 +166,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
           pip install --user --upgrade setuptools
       - name: Copyright Check
         run: tox -e check-copyright
@@ -188,16 +191,38 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Install markdown-spellcheck
         run: sudo npm install -g markdown-spellcheck
-      - name: Check Docs links and IPFS hashes
-        run: tox -e check-doc-links-hashes
       - name: Check API Docs updated
         run: tox -e check-api-docs
       - name: Check spelling
         run: tox -e spell-check
 
+  docs_link_check:
+    continue-on-error: True
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: github.base_ref == 'main'
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - name: Install dependencies (ubuntu-latest)
+        run: |
+          sudo apt-get update --fix-missing
+          sudo apt-get autoremove
+          sudo apt-get autoclean
+          pip install tomte[tox]==0.1.5
+      - name: Check Docs links and IPFS hashes
+        run: tox -e check-doc-links-hashes
+
   scan:
     name: gitleaks
     runs-on: ubuntu-latest
@@ -229,7 +254,7 @@ jobs:
           python-version: ${{ matrix.python_version }}
       - name: Install tox
         run: |
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Check Pipfile and tox.ini consistency
         run: |
           python ./scripts/check_pipfile_and_toxini.py
@@ -260,7 +285,7 @@ jobs:
           python-version: ${{ matrix.python_version }}
       - name: Install tox
         run: |
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Check plugin aea-ledger-cosmos
         run: |
           tox -r -e plugins_env -- sh -c "pip install ./plugins/aea-ledger-cosmos && aea generate-key cosmos && echo aea-ledger-cosmos checked!"
@@ -320,7 +345,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
           # install Protobuf compiler
           wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
           unzip protoc-3.19.4-linux-x86_64.zip -d protoc
@@ -357,7 +382,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
           # install Protobuf compiler
           wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
           unzip protoc-3.19.4-linux-x86_64.zip -d protoc
@@ -386,7 +411,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
       - name: Framework integration tests
         run: tox -e py3.10 -- -m 'integration and not unstable and ledger and not profiling'
       - name: Packages integration tests
@@ -406,7 +431,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python_version: ['3.7', '3.8', '3.9', '3.10']
-    timeout-minutes: 90
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@master
       - uses: actions/setup-python@v3
@@ -421,7 +446,7 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
+          pip install tomte[tox]==0.1.5
           # install Protobuf compiler
           wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
           unzip protoc-3.19.4-linux-x86_64.zip -d protoc
@@ -442,7 +467,7 @@ jobs:
       - if: matrix.os == 'macos-latest'
         name: Install dependencies (macos-latest)
         run: |
-          pip install tox
+          pip install tomte[tox]==0.1.5
           brew install gcc
           # brew install protobuf
           # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
@@ -473,7 +498,7 @@ jobs:
           choco install make -y
           # to check make was installed
           make --version
-          pip install tox
+          pip install tomte[tox]==0.1.5
           # wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-win64.zip
           # unzip protoc-3.19.4-win64.zip -d protoc
           # sudo mv protoc/bin/protoc /usr/local/bin/protoc
@@ -547,8 +572,10 @@ jobs:
           sudo apt-get update --fix-missing
           sudo apt-get autoremove
           sudo apt-get autoclean
-          pip install tox
-          pip install coverage
+          pip install tomte[tox]==0.1.5
+          # we only need "coverage" from the "tests" group
+          # but this makes it easier to stay in sync
+          pip install tomte[tests]==0.1.5
           # install Protobuf compiler
           wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
           unzip protoc-3.19.4-linux-x86_64.zip -d protoc

.pylintrc

@@ -36,7 +36,7 @@ disable=C0103,C0201,C0301,C0302,W0105,W0707,W1202,W1203,R0801,E1136,E0611,C0209,
 # C0206: consider-using-dict-items
 
 [IMPORTS]
-ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker
+ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker,signal
 
 [DESIGN]
 min-public-methods=1

HISTORY.md

@@ -1,5 +1,46 @@
 # Release History - open AEA
 
+## 1.26.0 (2022-12-15)
+
+AEA:
+- Adds support for hashing byte strings on `IPFSHashOnly` tool
+- Introduces `CliTest` tool to help with the `CLI` testing
+- Extends aea packages lock command to update fingerprints
+- Adds support for appending test coverage with previous runs on `aea test` command and fixes the coverage on `aea test packages` command
+- Updates the `BasePackageManager.add_package` to fetch packages recursively
+
+Plugins:
+- Updates the `cosmos` and `fetchai` ledger plugins to use `PyCryptodome` for `ripemd160` hash generation
+- Adds support for publishing byte strings directly to IPFS daemons without intermediate file storage on the `IPFS` plugin
+
+Chores:
+- Pins `tox` version using `tomte` in the CI to maintain version consistency
+- Pins `go` version to `v1.17.7` on the CI
+
+## 1.25.0 (2022-12-01)
+
+AEA:
+- Fixes the mechanism to convert the `json` path to environment variable string
+- Updates the process of agent subprocess termination to make sure we properly terminate agents across the various operating systems
+- Introduces `reraise_as_click_exception` to re-raise exceptions as `click.ClickExceptions` on command definitions
+- Extends `CliRunner` to allow usage of `capfd` to capture test output
+- Introduces `generate_env_vars_recursively` method to auto generate the environment variable names for component overrides
+- Extends `aea generate-key` to support creating multiple keys
+- Extends the package manager API to
+  - Update the hashes for third party packages with a warning
+  - Update the dependency hashes when locking packages
+  - Verifying the dependency hashes when verifying packages
+- Adds deprecation warning for `aea hash all` command since the same functionality is now being provided by `aea packages lock` command
+
+Tests:
+- Updates `libp2p` tests to use `capsys` to read `stdout` instead of patching `sys.stdout`
+- Re enables tests skipped with `# need remote registry` comment
+- Adds tests for package manager API
+
+Chores:
+- Updates the `tox` environment setting for unit tests to report duration of tests
+- Deprecates the usage of `aea hash all` command from the workflow
+
 ## 1.24.0 (2022-11-15)
 
 AEA:

Makefile

@@ -71,8 +71,6 @@ test-sub-p:
 
 .PHONY: hashes
 hashes:
-	python -m aea.cli hash all
-	python -m aea.cli hash all --packages-dir=./tests/data/packages
 	python -m aea.cli packages lock
 	python -m aea.cli --registry-path=./tests/data/packages packages lock
 
@@ -129,11 +127,11 @@ new_env: clean
 		echo "In a virtual environment! Exit first: 'exit'.";\
 	fi
 protolint_install:
-	GO111MODULE=on GOPATH=~/go go get -u -v github.com/yoheimuta/protolint/cmd/protolint@v0.27.0
+	GO111MODULE=on GOPATH=~/go go install -v github.com/yoheimuta/protolint/cmd/protolint@v0.27.0
 protolint:
 	PATH=${PATH}:${GOPATH}/bin/:~/go/bin protolint lint -config_path=./protolint.yaml -fix ./aea/mail ./packages/fetchai/protocols ./packages/valory/protocols
 protolint_install_win:
-	powershell -command '$$env:GO111MODULE="on"; go get -u -v github.com/yoheimuta/protolint/cmd/protolint@v0.27.0'
+	powershell -command '$$env:GO111MODULE="on"; go install -v github.com/yoheimuta/protolint/cmd/protolint@v0.27.0'
 protolint_win:
 	protolint lint -config_path=./protolint.yaml -fix ./aea/mail ./packages/fetchai/protocols ./packages/valory/protocols
 

SECURITY.md

@@ -8,8 +8,8 @@ The following table shows which versions of `open-aea` are currently being suppo
 
 | Version   | Supported          |
 | --------- | ------------------ |
-| `1.24.x`   | :white_check_mark: |
-| `< 1.24.0` | :x:                |
+| `1.26.x`   | :white_check_mark: |
+| `< 1.26.0` | :x:                |
 
 ## Reporting a Vulnerability
 

aea/__version__.py

@@ -23,7 +23,7 @@
 __title__ = "open-aea"
 __description__ = "Open Autonomous Economic Agent framework (without vendor lock-in)"
 __url__ = "https://github.com/valory-xyz/open-aea.git"
-__version__ = "1.24.0"
+__version__ = "1.26.0"
 __author__ = "Valory AG"
 __license__ = "Apache-2.0"
 __copyright__ = "2021 Valory AG, 2019 Fetch.AI Limited"

aea/cli/generate_key.py

@@ -27,7 +27,7 @@
 from aea.cli.utils.click_utils import password_option
 from aea.cli.utils.decorators import _check_aea_project
 from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA
-from aea.crypto.helpers import create_private_key
+from aea.crypto.helpers import create_private_key, generate_multiple_keys
 from aea.crypto.registries import crypto_registry
 
 
@@ -59,6 +59,7 @@
     required=False,
     default="",
 )
+@click.option("-n", type=int, help="Number of keys to generate")
 @click.pass_context
 def generate_key(
     click_context: click.core.Context,
@@ -68,8 +69,40 @@ def generate_key(
     add_key: bool = False,
     connection: bool = False,
     extra_entropy: Union[str, bytes, int] = "",
+    n: Optional[int] = None,
 ) -> None:
     """Generate a private key and place it in a file."""
+    if n is None:
+        _generate_one(
+            click_context=click_context,
+            type_=type_,
+            file=file,
+            password=password,
+            add_key=add_key,
+            connection=connection,
+            extra_entropy=extra_entropy,
+        )
+        return
+
+    generate_multiple_keys(
+        n=n,
+        type_=type_,
+        password=password,
+        extra_entropy=extra_entropy,
+        file=file,
+    )
+
+
+def _generate_one(
+    click_context: click.core.Context,
+    type_: str,
+    file: str,
+    password: Optional[str],
+    add_key: bool = False,
+    connection: bool = False,
+    extra_entropy: Union[str, bytes, int] = "",
+) -> None:
+    """Generate one key."""
     keys_generated = _generate_private_key(type_, file, password, extra_entropy)
     if add_key:
         _check_aea_project((click_context,))

aea/cli/get_wealth.py

@@ -39,7 +39,9 @@
 @click.argument(
     "type_",
     metavar="TYPE",
-    type=click.Choice(ledger_apis_registry.supported_ids),
+    type=click.Choice(
+        choices=list(ledger_apis_registry.supported_ids),
+    ),
     required=True,
 )
 @password_option()

aea/cli/ipfs_hash.py

@@ -29,6 +29,7 @@
 import traceback
 from pathlib import Path
 from typing import Callable, Dict, Optional, Tuple, cast
+from warnings import warn
 
 import click
 
@@ -260,6 +261,16 @@ def generate_all(
     no_wrap: bool,
 ) -> None:
     """Generate IPFS hashes."""
+    message = (
+        "`aea hash all` command has been deprecated and will be removed on v2.0.0, "
+        "please use `aea packages lock` command to perform package hash updates"
+    )
+    warn(
+        message=message,
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    click.echo(message=message)
     packages_dir = Path(packages_dir).absolute()
     return_code = update_hashes(packages_dir, no_wrap, vendor=vendor)
     sys.exit(return_code)