patch addrman bucketing, remove build

.github/workflows/test.yml

@@ -11,17 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        backend: [compose, k8s]
+        backend: [compose]
     steps:
       - uses: actions/checkout@v3
-      - if: matrix.backend == 'compose'
-        uses: ./.github/actions
+      - uses: ./.github/actions
       - if: matrix.backend == 'k8s'
         uses: extractions/setup-just@v1
       - if: matrix.backend == 'k8s'
         uses: medyagh/setup-minikube@master
       - if: matrix.backend == 'k8s'
         run: |
+          eval $(minikube docker-env)
           pip install --upgrade pip
           pip install -e .
           just start

docs/graph.md

@@ -70,7 +70,7 @@ These should be built using the `warcli image build` command to ensure they have
 <node id="0">
 <data key="x">5.5</data>
 <data key="y">2.5</data>
-<data key="image">bitcoindevproject/bitcoin-core:26.0</data>
+<data key="image">bitcoindevproject/bitcoin:26.0</data>
 <data key="bitcoin_config">uacomment=warnet0_v24,debugexclude=libevent</data>
 <data key="tc_netem"></data>
 </node>

justfile

@@ -74,3 +74,13 @@ stopd:
 # port forward
 p:
     kubectl port-forward svc/rpc 9276:9276
+
+registry := 'bitcoindevproject/bitcoin'
+repo := 'bitcoin/bitcoin'
+arches := 'amd64'
+build-args := "--disable-tests --without-gui --disable-bench --disable-fuzz-binary --enable-suppress-external-warnings"
+load := "load"
+
+# Build docker image and optionally push to registry
+build branch tag registry=registry repo=repo build-args=build-args action=load:
+    warcli image build --registry={{registry}} --repo={{repo}} --branch={{branch}} --arches={{arches}} --tag={{tag}} --build-args="{{build-args}}" --action={{action}}

scripts/apidocs.py

@@ -2,9 +2,9 @@
 import re
 from pathlib import Path
 
+from cli.main import cli
 from click import Context
 from tabulate import tabulate
-from warnet.cli.main import cli
 
 doc = ""
 

src/backends/backend_interface.py

@@ -45,7 +45,7 @@ def get_status(self, tank_index: int, service: ServiceType):
         raise NotImplementedError("This method should be overridden by child class")
 
     @abstractmethod
-    def exec_run(self, tank_index: int, service: ServiceType, cmd: str, user: str):
+    def exec_run(self, tank_index: int, service: ServiceType, cmd: str):
         """
         Exectute a command on tank [tank_index] in service [service]
         """

src/backends/compose/compose_backend.py

@@ -9,6 +9,7 @@
 import docker
 import yaml
 from backends import BackendInterface, ServiceType
+from cli.image import build_image
 from docker.models.containers import Container
 from templates import TEMPLATES
 from warnet.lnnode import LNNode
@@ -33,7 +34,8 @@
 DOCKERFILE_NAME = "Dockerfile"
 TORRC_NAME = "torrc"
 ENTRYPOINT_NAME = "entrypoint.sh"
-DOCKER_REGISTRY = "bitcoindevproject/bitcoin-core"
+DOCKER_REGISTRY = "bitcoindevproject/bitcoin"
+LOCAL_REGISTRY = "warnet/bitcoin-core"
 GRAFANA_PROVISIONING = "grafana-provisioning"
 CONTAINER_PREFIX_BITCOIND = "tank-bitcoin"
 CONTAINER_PREFIX_LN = "tank-ln"
@@ -135,9 +137,9 @@ def get_status(self, tank_index: int, service: ServiceType) -> RunningStatus:
             case _:
                 return RunningStatus.PENDING
 
-    def exec_run(self, tank_index: int, service: ServiceType, cmd: str, user: str = "root") -> str:
+    def exec_run( self, tank_index: int, service: ServiceType, cmd: str) -> str:
         c = self.get_container(tank_index, service)
-        result = c.exec_run(cmd=cmd, user=user)
+        result = c.exec_run(cmd=cmd)
         if result.exit_code != 0:
             raise Exception(
                 f"Command failed with exit code {result.exit_code}: {result.output.decode('utf-8')} {cmd}"
@@ -168,7 +170,7 @@ def get_bitcoin_cli(self, tank: Tank, method: str, params=None):
             cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method} {' '.join(map(str, params))}"
         else:
             cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method}"
-        return self.exec_run(tank.index, ServiceType.BITCOIN, cmd, user="bitcoin")
+        return self.exec_run(tank.index, ServiceType.BITCOIN, cmd)
 
     def get_file(self, tank_index: int, service: ServiceType, file_path: str):
         container = self.get_container(tank_index, service)
@@ -358,18 +360,10 @@ def add_services(self, tank: Tank, services):
         if "/" and "#" in tank.version:
             # it's a git branch, building step is necessary
             repo, branch = tank.version.split("#")
-            build = {
-                "context": str(tank.config_dir),
-                "dockerfile": str(tank.config_dir / DOCKERFILE_NAME),
-                "args": {
-                    "REPO": repo,
-                    "BRANCH": branch,
-                    "BUILD_ARGS": f"{tank.DEFAULT_BUILD_ARGS + tank.extra_build_args}",
-                },
-            }
-            services[container_name]["build"] = build
+            services[container_name]["image"] = f"{LOCAL_REGISTRY}:{branch}"
+            build_image(repo, branch, LOCAL_REGISTRY, branch, tank.DEFAULT_BUILD_ARGS + tank.extra_build_args, arches="amd64")
             self.copy_configs(tank)
-        elif tank.is_custom_build and tank.image:
+        elif tank.image:
             # Pre-built custom image
             image = tank.image
             services[container_name]["image"] = image

src/backends/kubernetes/kubernetes_backend.py

@@ -6,6 +6,7 @@
 
 import yaml
 from backends import BackendInterface, ServiceType
+from cli.image import build_image
 from kubernetes import client, config
 from kubernetes.client.models.v1_pod import V1Pod
 from kubernetes.client.rest import ApiException
@@ -15,8 +16,9 @@
 from warnet.tank import Tank
 from warnet.utils import default_bitcoin_conf_args, parse_raw_messages
 
-DOCKER_REGISTRY_CORE = "bitcoindevproject/k8s-bitcoin-core"
+DOCKER_REGISTRY_CORE = "bitcoindevproject/bitcoin"
 DOCKER_REGISTRY_LND = "lightninglabs/lnd:v0.17.0-beta"
+LOCAL_REGISTRY = "warnet/bitcoin-core"
 POD_PREFIX = "tank"
 BITCOIN_CONTAINER_NAME = "bitcoin"
 LN_CONTAINER_NAME = "ln"
@@ -157,13 +159,12 @@ def get_status(self, tank_index: int, service: ServiceType) -> RunningStatus:
                 break
         return RunningStatus.RUNNING if ready else RunningStatus.PENDING
 
-    def exec_run(self, tank_index: int, service: ServiceType, cmd: str, user: str = "root"):
-        # k8s doesn't let us run exec commands as a user, but we can use su
-        # because its installed in the bitcoin containers. we will need to rework
-        # this command if we decided to remove gosu from the containers
-        # TODO: change this if we remove gosu
+    def exec_run(self, tank_index: int, service: ServiceType, cmd: str):
         pod_name = self.get_pod_name(tank_index, service)
-        exec_cmd = ["/bin/sh", "-c", f"su - {user} -c '{cmd}'"]
+        if service == ServiceType.BITCOIN:
+            exec_cmd = ["/bin/bash", "-c", f"{cmd}"]
+        elif service == ServiceType.LIGHTNING:
+            exec_cmd = ["/bin/sh", "-c", f"{cmd}"]
         result = stream(
             self.client.connect_get_namespaced_pod_exec,
             pod_name,
@@ -211,7 +212,7 @@ def get_bitcoin_cli(self, tank: Tank, method: str, params=None):
             cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method} {' '.join(map(str, params))}"
         else:
             cmd = f"bitcoin-cli -regtest -rpcuser={tank.rpc_user} -rpcport={tank.rpc_port} -rpcpassword={tank.rpc_password} {method}"
-        return self.exec_run(tank.index, ServiceType.BITCOIN, cmd, user="bitcoin")
+        return self.exec_run(tank.index, ServiceType.BITCOIN, cmd)
 
     def get_messages(
         self,
@@ -319,7 +320,6 @@ def tank_from_deployment(self, pod, pods_by_name, warnet):
                         c_branch = env.value
                 if c_repo and c_branch:
                     t.version = f"{c_repo}#{c_branch}"
-                    t.is_custom_build = True
 
         # check if we can find a corresponding lnd pod
         lnd_pod = pods_by_name.get(self.get_pod_name(index, ServiceType.LIGHTNING))
@@ -340,17 +340,37 @@ def default_bitcoind_config_args(self, tank):
 
     def create_bitcoind_container(self, tank) -> client.V1Container:
         container_name = BITCOIN_CONTAINER_NAME
-        container_image = (
-            tank.image if tank.is_custom_build else f"{DOCKER_REGISTRY_CORE}:{tank.version}"
-        )
+        container_image = None
+
+        # Prebuilt image
+        if tank.image:
+            container_image = tank.image
+        # On-demand built image
+        elif "/" and "#" in tank.version:
+        # We don't have docker installed on the RPC server, where this code will be run from,
+        # and it's currently unclear to me if having the RPC pod build images is a good idea.
+        # Don't support this for now in CI by disabling in the workflow.
+
+        # This can be re-enabled by enabling in the workflow file and installing docker and
+        # docker-buildx on the rpc server image.
+
+            # it's a git branch, building step is necessary
+            repo, branch = tank.version.split("#")
+            build_image(
+                repo,
+                branch,
+                LOCAL_REGISTRY,
+                branch,
+                tank.DEFAULT_BUILD_ARGS + tank.extra_build_args,
+                arches="amd64",
+            )
+        # Prebuilt major version
+        else:
+            container_image = f"{DOCKER_REGISTRY_CORE}:{tank.version}"
+
         container_env = [
             client.V1EnvVar(name="BITCOIN_ARGS", value=self.default_bitcoind_config_args(tank))
         ]
-        # TODO: support custom builds
-        if tank.is_custom_build:
-            # TODO: check if the build already exists in the registry
-            # Annoyingly the api differs between providers, so this is annoying
-            pass
 
         return client.V1Container(
             name=container_name,

src/cli/image.py

@@ -1,7 +1,8 @@
 import sys
 
 import click
-from utils.image_build import build_and_upload_images
+
+from .image_build import build_image
 
 
 @click.group(name="image")
@@ -16,11 +17,12 @@ def image():
 @click.option("--tag", required=True, type=str)
 @click.option("--build-args", required=False, type=str)
 @click.option("--arches", required=False, type=str)
-def build(repo, branch, registry, tag, build_args, arches):
+@click.option("--action", required=False, type=str)
+def build(repo, branch, registry, tag, build_args, arches, action="load"):
     """
-    Build bitcoind and bitcoin-cli from <repo>/<branch> and deploy to <registry> as <tag>
-    This requires docker and buildkit to be enabled.
+    Build bitcoind and bitcoin-cli from <repo>/<branch> as <registry>:<tag>.
+    Optionally deploy to remote registry using --action=push, otherwise image is loaded to local registry.
     """
-    res = build_and_upload_images(repo, branch, registry, tag, build_args, arches)
+    res = build_image(repo, branch, registry, tag, build_args, arches, action)
     if not res:
         sys.exit(1)