Merge pull request #2221 from valory-xyz/feat/dev-mode

Fix development mode

.github/workflows/release.yml

@@ -191,3 +191,30 @@ jobs:
           docker buildx build --platform linux/amd64,linux/arm64 -t valory/open-autonomy-hardhat:$TAG deployments/Dockerfiles/hardhat --push
           docker buildx build --platform linux/amd64,linux/arm64 -t valory/open-autonomy-hardhat:latest deployments/Dockerfiles/hardhat --push
 
+  publish-dev-image:
+    name: Publish Develop Image
+    runs-on: ubuntu-latest
+    needs:
+      - publish-autonomy-packages
+    steps:
+      - uses: actions/checkout@v2
+      - name: Docker login
+        env:
+          DOCKER_USER: ${{secrets.DOCKER_USER}}
+          DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+      - name: Set up support for multi platform build
+        run: |
+          docker run --privileged --rm tonistiigi/binfmt --install all
+          docker buildx create --use --name multibuild
+          docker buildx inspect --bootstrap
+      - name: Set up tag
+        run: echo export TAG=$(python3 -c "from setup import about; print(about[\"__version__\"])") > env.sh
+      - name: Build and push version tagged images
+        run: |
+          # export `TAG` variable
+          source env.sh
+          
+          docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t valory/open-autonomy-dev:$TAG deployments/Dockerfiles/development --push
+          docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t valory/open-autonomy-dev:latest deployments/Dockerfiles/development --push

autonomy/cli/build_images.py

@@ -79,14 +79,17 @@ def build_image(  # pylint: disable=too-many-arguments
     image_author: Optional[str] = None,
 ) -> None:
     """Build runtime images for autonomous agents."""
+    if dev:
+        click.echo(
+            "`--dev` flag is deprecated, development mode does not require you to build docker image manually anymore."
+        )
 
     with reraise_as_click_exception(ImageBuildFailed, FileNotFoundError):
         _build_image(
             agent=agent,
             service_dir=service_dir,
             extra_dependencies=extra_dependencies,
             pull=pull,
-            dev=dev,
             version=version,
             image_author=image_author,
             dockerfile=dockerfile,

autonomy/cli/deploy.py

@@ -24,6 +24,7 @@
 from typing import Optional, cast
 
 import click
+from aea import AEA_DIR
 from aea.cli.registry.settings import REGISTRY_REMOTE
 from aea.cli.utils.click_utils import (
     password_option,
@@ -59,9 +60,27 @@
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
 
 
-PACKAGES_DIR = "packages_dir"
-OPEN_AEA_DIR = "open_aea_dir"
-OPEN_AUTONOMY_DIR = "open_autonomy_dir"
+def _validate_packages_path(path: Optional[Path] = None) -> Path:
+    """Find packages dir for development mode."""
+    path = Path(path or Path.cwd()).resolve()
+    while path != path.root:
+        if path.name == "packages" and (path / "packages.json").exists():
+            return path
+        path = path.parent
+    raise click.ClickException(
+        "Please provide path to packages directory for development mode"
+    )
+
+
+def _validate_open_aea_dir(path: Optional[Path] = None) -> Path:
+    """Validate open-aea directory."""
+    path = Path(path or Path(AEA_DIR).parent).resolve()
+    for child in ("aea", "plugins", "setup.py"):
+        if not (path / child).exists():
+            raise click.ClickException(
+                "Please provide valid open-aea repository path for development mode"
+            )
+    return path
 
 
 @click.group(name="deploy")
@@ -136,11 +155,6 @@ def deploy_group(
     type=click.Path(),
     help="Path to open-aea repo (Use with dev mode)",
 )
-@click.option(
-    "--open-autonomy-dir",
-    type=click.Path(),
-    help="Path to open-autonomy repo (Use with dev mode)",
-)
 @click.option(
     "--aev",
     is_flag=True,
@@ -207,7 +221,6 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     password: Optional[str] = None,
     open_aea_dir: Optional[Path] = None,
     packages_dir: Optional[Path] = None,
-    open_autonomy_dir: Optional[Path] = None,
     log_level: str = INFO,
     aev: bool = False,
     image_version: Optional[str] = None,
@@ -228,28 +241,14 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
         )
 
     keys_file = Path(keys_file or DEFAULT_KEYS_FILE).absolute()
-    if not keys_file.exists():
+    if not keys_file.exists() and not dev_mode:
         message = f"No such file or directory: {keys_file}. Please provide valid path for keys file."
         raise click.ClickException(message)
 
     build_dir = Path(output_dir or DEFAULT_BUILD_FOLDER).absolute()
-    packages_dir = Path(packages_dir or Path.cwd() / "packages").absolute()
-    open_aea_dir = Path(open_aea_dir or Path.home() / "open-aea").absolute()
-    open_autonomy_dir = Path(
-        open_autonomy_dir or Path.home() / "open-autonomy"
-    ).absolute()
-
     if dev_mode:
-        for name, path in (
-            (PACKAGES_DIR, packages_dir),
-            (OPEN_AEA_DIR, open_aea_dir),
-            (OPEN_AUTONOMY_DIR, open_autonomy_dir),
-        ):
-            if not path.exists():
-                flag = "--" + "-".join(name.split("_"))
-                raise click.ClickException(
-                    f"Path does not exist @ {path} for {name}; Please provide proper value for {flag}"
-                )
+        packages_dir = _validate_packages_path(path=packages_dir)
+        open_aea_dir = _validate_open_aea_dir(path=open_aea_dir)
 
     ctx = cast(Context, click_context.obj)
     ctx.registry_type = registry
@@ -263,7 +262,6 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
             number_of_agents=number_of_agents,
             packages_dir=packages_dir,
             open_aea_dir=open_aea_dir,
-            open_autonomy_dir=open_autonomy_dir,
             log_level=log_level,
             apply_environment_variables=aev,
             image_version=image_version,

autonomy/cli/helpers/deployment.py

@@ -183,7 +183,6 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     number_of_agents: Optional[int] = None,
     packages_dir: Optional[Path] = None,
     open_aea_dir: Optional[Path] = None,
-    open_autonomy_dir: Optional[Path] = None,
     agent_instances: Optional[List[str]] = None,
     multisig_address: Optional[str] = None,
     consensus_threshold: Optional[int] = None,
@@ -219,7 +218,6 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
         dev_mode=dev_mode,
         packages_dir=packages_dir,
         open_aea_dir=open_aea_dir,
-        open_autonomy_dir=open_autonomy_dir,
         agent_instances=agent_instances,
         multisig_address=multisig_address,
         consensus_threshold=consensus_threshold,

autonomy/cli/helpers/image.py

@@ -33,7 +33,6 @@ def build_image(  # pylint: disable=too-many-arguments
     agent: Optional[PublicId],
     service_dir: Optional[Path],
     pull: bool = False,
-    dev: bool = False,
     version: Optional[str] = None,
     image_author: Optional[str] = None,
     extra_dependencies: Optional[Tuple[Dependency, ...]] = None,
@@ -51,7 +50,6 @@ def build_image(  # pylint: disable=too-many-arguments
     _build_image(
         agent=agent,
         pull=pull,
-        dev=dev,
         version=version,
         image_author=image_author,
         extra_dependencies=extra_dependencies,

autonomy/constants.py

@@ -32,6 +32,9 @@
 AUTONOMY_IMAGE_VERSION = os.environ.get(
     "AUTONOMY_IMAGE_VERSION", DEFAULT_AUTONOMY_VERSION
 )
+DEVELOPMENT_IMAGE_VERSION = os.environ.get(
+    "DEVELOPMENT_IMAGE_VERSION", DEFAULT_AUTONOMY_VERSION
+)
 TENDERMINT_IMAGE_VERSION = os.environ.get(
     "TENDERMINT_IMAGE_VERSION", DEFAULT_AUTONOMY_VERSION
 )
@@ -42,6 +45,9 @@
 ACN_IMAGE_VERSION = os.environ.get("ACN_IMAGE_VERSION", DEFAULT_IMAGE_VERSION)
 
 AUTONOMY_IMAGE_NAME = os.environ.get("AUTONOMY_IMAGE_NAME", "valory/open-autonomy")
+DEVELOPMENT_IMAGE_NAME = os.environ.get(
+    "AUTONOMY_IMAGE_NAME", "valory/open-autonomy-dev"
+)
 TENDERMINT_IMAGE_NAME = os.environ.get(
     "TENDERMINT_IMAGE_NAME", "valory/open-autonomy-tendermint"
 )
@@ -51,6 +57,7 @@
 SERVICE_REGISTRY_IMAGE_NAME = os.environ.get(
     "SERVICE_REGISTRY_IMAGE_NAME", f"valory/{SERVICE_REGISTRY_CONTRACT_CONTAINER_NAME}"
 )
+DEVELOPMENT_IMAGE = f"{DEVELOPMENT_IMAGE_NAME}:{DEVELOPMENT_IMAGE_VERSION}"
 
 DEFAULT_SERVICE_REGISTRY_CONTRACTS_IMAGE = (
     f"{SERVICE_REGISTRY_IMAGE_NAME}:{SERVICE_REGISTRY_IMAGE_VERSION}"

autonomy/deploy/base.py

@@ -27,6 +27,7 @@
 from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
 from warnings import warn
 
+from aea.cli.generate_key import _generate_multiple_keys
 from aea.configurations.base import (
     ConnectionConfig,
     ContractConfig,
@@ -252,6 +253,7 @@ def from_dir(  # pylint: disable=too-many-arguments
         number_of_agents: Optional[int] = None,
         agent_instances: Optional[List[str]] = None,
         apply_environment_variables: bool = False,
+        dev_mode: bool = False,
     ) -> "ServiceBuilder":
         """Service builder from path."""
         service = load_service_config(
@@ -266,6 +268,13 @@ def from_dir(  # pylint: disable=too-many-arguments
         )
 
         if keys_file is not None:
+            if dev_mode and not keys_file.exists():
+                # TODO: Add multiledger support
+                _generate_multiple_keys(
+                    n=service.number_of_agents,
+                    type_="ethereum",
+                    file=keys_file,
+                )
             service_builder.read_keys(keys_file=keys_file)
 
         if agent_instances is not None:
@@ -742,9 +751,8 @@ class BaseDeploymentGenerator(abc.ABC):  # pylint: disable=too-many-instance-att
     dev_mode: bool
     use_tm_testnet_setup: bool
 
-    packages_dir: Path
-    open_aea_dir: Path
-    open_autonomy_dir: Path
+    packages_dir: Optional[Path]
+    open_aea_dir: Optional[Path]
     resources: Resources
 
     def __init__(  # pylint: disable=too-many-arguments
@@ -755,7 +763,6 @@ def __init__(  # pylint: disable=too-many-arguments
         dev_mode: bool = False,
         packages_dir: Optional[Path] = None,
         open_aea_dir: Optional[Path] = None,
-        open_autonomy_dir: Optional[Path] = None,
         image_author: Optional[str] = None,
         resources: Optional[Resources] = None,
     ):
@@ -765,11 +772,8 @@ def __init__(  # pylint: disable=too-many-arguments
         self.build_dir = build_dir
         self.use_tm_testnet_setup = use_tm_testnet_setup
         self.dev_mode = dev_mode
-        self.packages_dir = packages_dir or Path.cwd().absolute() / "packages"
-        self.open_aea_dir = open_aea_dir or Path.home().absolute() / "open-aea"
-        self.open_autonomy_dir = (
-            open_autonomy_dir or Path.home().absolute() / "open-autonomy"
-        )
+        self.packages_dir = packages_dir
+        self.open_aea_dir = open_aea_dir
 
         self.tendermint_job_config: Optional[str] = None
         self.image_author = image_author or DEFAULT_DOCKER_IMAGE_AUTHOR

autonomy/deploy/build.py

@@ -41,7 +41,6 @@ def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     dev_mode: bool = False,
     packages_dir: Optional[Path] = None,
     open_aea_dir: Optional[Path] = None,
-    open_autonomy_dir: Optional[Path] = None,
     agent_instances: Optional[List[str]] = None,
     multisig_address: Optional[str] = None,
     consensus_threshold: Optional[int] = None,
@@ -55,13 +54,18 @@ def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     resources: Optional[Resources] = None,
 ) -> str:
     """Generate the deployment for the service."""
+    if dev_mode and type_of_deployment != DockerComposeGenerator.deployment_type:
+        raise RuntimeError(
+            "Development mode can only be used with docker-compose deployments"
+        )
 
     service_builder = ServiceBuilder.from_dir(
         path=service_path,
         keys_file=keys_file,
         number_of_agents=number_of_agents,
         agent_instances=agent_instances,
         apply_environment_variables=apply_environment_variables,
+        dev_mode=dev_mode,
     )
     service_builder.deplopyment_type = type_of_deployment
     service_builder.log_level = log_level
@@ -79,7 +83,6 @@ def generate_deployment(  # pylint: disable=too-many-arguments, too-many-locals
         dev_mode=dev_mode,
         packages_dir=packages_dir,
         open_aea_dir=open_aea_dir,
-        open_autonomy_dir=open_autonomy_dir,
         use_tm_testnet_setup=use_tm_testnet_setup,
         image_author=image_author,
         resources=resources,