From 53f4aa33004a0e251ee9c8243733223a9198c6dc Mon Sep 17 00:00:00 2001 From: Tim Serong Date: Thu, 19 Aug 2021 16:07:54 +1000 Subject: [PATCH] gravel: remove etcd Signed-off-by: Tim Serong --- doc/dev/gravel/systemd-boot.md | 2 +- doc/project-plan/roadmap.rst | 8 +- doc/project-plan/testing-plan.rst | 4 +- images/README.md | 2 +- images/aquarium/config.sh | 3 +- images/aquarium/config.xml | 1 - images/aquarium/disk.sh | 1 - src/boot/aqrbootsetup.sh | 1 - src/gravel/controllers/config.py | 9 - src/gravel/controllers/nodes/deployment.py | 35 ---- src/gravel/controllers/nodes/etcd.py | 163 ----------------- src/gravel/controllers/nodes/messages.py | 1 - src/gravel/controllers/nodes/mgr.py | 51 +----- src/gravel/controllers/nodes/systemdisk.py | 1 - src/gravel/tests/conftest.py | 35 +--- .../unit/controllers/nodes/test_deployment.py | 1 - .../tests/unit/controllers/nodes/test_etcd.py | 140 --------------- .../tests/unit/controllers/nodes/test_mgr.py | 61 ------- src/gravel/typings/aetcd3/__init__.pyi | 17 -- src/gravel/typings/aetcd3/client.pyi | 168 ------------------ src/gravel/typings/aetcd3/events.pyi | 7 - src/gravel/typings/aetcd3/locks.pyi | 20 --- src/gravel/typings/aetcd3/members.pyi | 17 -- src/mypy.ini | 3 - src/requirements.txt | 1 - 25 files changed, 17 insertions(+), 735 deletions(-) delete mode 100644 src/gravel/controllers/nodes/etcd.py delete mode 100644 src/gravel/tests/unit/controllers/nodes/test_etcd.py delete mode 100644 src/gravel/typings/aetcd3/__init__.pyi delete mode 100644 src/gravel/typings/aetcd3/client.pyi delete mode 100644 src/gravel/typings/aetcd3/events.pyi delete mode 100644 src/gravel/typings/aetcd3/locks.pyi delete mode 100644 src/gravel/typings/aetcd3/members.pyi diff --git a/doc/dev/gravel/systemd-boot.md b/doc/dev/gravel/systemd-boot.md index 1e3a55b30..23899a9e6 100644 --- a/doc/dev/gravel/systemd-boot.md +++ b/doc/dev/gravel/systemd-boot.md @@ -17,7 +17,7 @@ These are * `/var/log`, so logs are persisted between runs. -* `/var/lib/etcd`, where we keep this node's etcd persistent db. +* `/var/lib/aquarium`, where we keep this node's local kvstore cache. * `/var/lib/containers`, where container images are kept. diff --git a/doc/project-plan/roadmap.rst b/doc/project-plan/roadmap.rst index a739aa84d..ea4737906 100644 --- a/doc/project-plan/roadmap.rst +++ b/doc/project-plan/roadmap.rst @@ -73,7 +73,7 @@ M6 * Add events widget (frontend) - * Basic event stashing on etcd (backend) + * Basic event stashing (backend) * node join * ??? @@ -171,7 +171,7 @@ A list of items brought to you in some unkempt, roughly prioritized order. * Obtain cluster events (backend) - * Store them in etcd? + * Store them somewhere? * Mutual exclusion access to ceph cluster operations * Dashboard @@ -185,7 +185,7 @@ A list of items brought to you in some unkempt, roughly prioritized order. * Figure out what is an event (backend) * Figure out how to display Ceph status updates as events (backend) - * Store events in etcd (backend) + * Store events (backend) * Display events (frontend) * Hosts @@ -199,7 +199,7 @@ A list of items brought to you in some unkempt, roughly prioritized order. * Obtain logs for each node (frontend, backend) * Obtain all logs (frontend, backend) - * Likely rely on etcd to do keep obtained logs from all nodes (backend) + * Where to keep obtained logs from all nodes? (backend) * or on websockets to connect to each node and obtain those logs? diff --git a/doc/project-plan/testing-plan.rst b/doc/project-plan/testing-plan.rst index c01b5a50c..c4e54df39 100644 --- a/doc/project-plan/testing-plan.rst +++ b/doc/project-plan/testing-plan.rst @@ -10,7 +10,7 @@ ceph tests should either be sufficient in this regard or expanded upon. Our primary scope is Aquarium. That is, we want to ensure aquarium does the right things, makes the right/expected decisions, and executes the orchestrated plan as expected. We also want to test the integration between Aquarium, the -host OS, Ceph, and any other services (etcd for example) to ensure that the +host OS, Ceph, and any other services to ensure that the deployment and operation of such services is functional and works with Aquarium's directives (ie, opinionated configuration). @@ -130,4 +130,4 @@ aqrtest This is not run as part of any CI yet. -TODO \ No newline at end of file +TODO diff --git a/images/README.md b/images/README.md index 777f96718..5799c2da2 100644 --- a/images/README.md +++ b/images/README.md @@ -24,7 +24,7 @@ will be then be started on boot. The latter, `disk.sh`, is run within a chrooted image mount, but before the image is finalized. During this step we will be obtaining container images needed for Aquarium's execution, so we have them available upon first run, -including an image for `Ceph` and an image for `etcd`. +including an image for `Ceph`. ## Containerized build environment diff --git a/images/aquarium/config.sh b/images/aquarium/config.sh index 348d1d983..cf958741f 100644 --- a/images/aquarium/config.sh +++ b/images/aquarium/config.sh @@ -81,8 +81,7 @@ if [[ "$kiwi_profiles" == *"Vagrant"* ]]; then fi pip install fastapi==0.63.0 uvicorn==0.13.3 websockets==8.1 \ - bcrypt==3.2.0 pyjwt==2.1.0 python-multipart==0.0.5 \ - git+https://github.com/aquarist-labs/aetcd3/@edf633045ce61c7bbac4d4a6ca15b14f8acfe9cd + bcrypt==3.2.0 pyjwt==2.1.0 python-multipart==0.0.5 baseInsertService aquarium-boot baseInsertService sshd baseInsertService aquarium diff --git a/images/aquarium/config.xml b/images/aquarium/config.xml index 39cdf42ac..7f031c44f 100644 --- a/images/aquarium/config.xml +++ b/images/aquarium/config.xml @@ -179,7 +179,6 @@ - diff --git a/images/aquarium/disk.sh b/images/aquarium/disk.sh index b88526da5..a04ec6769 100644 --- a/images/aquarium/disk.sh +++ b/images/aquarium/disk.sh @@ -39,7 +39,6 @@ mount -t tmpfs none /run # setting "--events-backend none" means podman doesn't try # (and fail) to log a "system refresh" event to the journal -/usr/bin/podman --events-backend none pull quay.io/coreos/etcd:latest # we don't get to use cephadm directly because it will # try running a container inside the chroot, and that # fails with a bang. diff --git a/src/boot/aqrbootsetup.sh b/src/boot/aqrbootsetup.sh index bf530232d..9de75ea7b 100644 --- a/src/boot/aqrbootsetup.sh +++ b/src/boot/aqrbootsetup.sh @@ -50,7 +50,6 @@ mount /dev/mapper/aquarium-systemdisk /aquarium # overlay overlay /etc etc || exit 1 overlay /var/log logs || exit 1 -overlay /var/lib/etcd etcd || exit 1 overlay /var/lib/aquarium aquarium || exit 1 overlay /var/lib/containers containers || exit 1 overlay /root roothome || exit 1 diff --git a/src/gravel/controllers/config.py b/src/gravel/controllers/config.py index f805758b9..16b21715c 100644 --- a/src/gravel/controllers/config.py +++ b/src/gravel/controllers/config.py @@ -53,14 +53,6 @@ class ServicesOptionsModel(BaseModel): probe_interval: float = Field(1.0, title="Services Probe Interval") -class EtcdOptionsModel(BaseModel): - registry: str = Field( - "quay.io/coreos/etcd", title="Container Image Registry" - ) - version: str = Field("latest", title="Container Version Label") - data_dir: str = Field("/var/lib/etcd", title="Etcd Data Dir") - - class AuthOptionsModel(BaseModel): jwt_secret: str = Field( title="The access token secret", @@ -81,7 +73,6 @@ class OptionsModel(BaseModel): devices: DevicesOptionsModel = Field(DevicesOptionsModel()) status: StatusOptionsModel = Field(StatusOptionsModel()) services: ServicesOptionsModel = Field(ServicesOptionsModel()) - etcd: EtcdOptionsModel = Field(EtcdOptionsModel()) auth: AuthOptionsModel = Field(AuthOptionsModel()) diff --git a/src/gravel/controllers/nodes/deployment.py b/src/gravel/controllers/nodes/deployment.py index e3efbb791..3d3622f2b 100644 --- a/src/gravel/controllers/nodes/deployment.py +++ b/src/gravel/controllers/nodes/deployment.py @@ -36,7 +36,6 @@ NodeHasBeenDeployedError, NodeHasJoinedError, ) -from gravel.controllers.nodes.etcd import spawn_etcd from gravel.controllers.nodes.host import HostnameCtlError, set_hostname from gravel.controllers.nodes.messages import ( ErrorMessageModel, @@ -307,7 +306,6 @@ async def join( assert welcome.pubkey assert welcome.cephconf assert welcome.keyring - assert welcome.etcd_peer # create system disk after we are certain we are joining. # ensure all state writes happen only after the disk has been created. @@ -321,17 +319,6 @@ async def join( self._state.mark_join() await self._set_hostname(hostname) - my_url: str = f"{hostname}=http://{address}:2380" - initial_cluster: str = f"{welcome.etcd_peer},{my_url}" - await spawn_etcd( - self._gstate, - new=False, - token=None, - hostname=hostname, - address=address, - initial_cluster=initial_cluster, - ) - authorized_keys: Path = Path("/root/.ssh/authorized_keys") if not authorized_keys.parent.exists(): authorized_keys.parent.mkdir(0o700) @@ -380,23 +367,6 @@ async def join( self._state.mark_ready() return True - async def _prepare_etcd( - self, hostname: str, address: str, token: str - ) -> None: - assert self._state - if self._state.bootstrapping: - raise NodeCantDeployError("node being deployed") - elif not self._state.nostage: - raise NodeCantDeployError("node can't be deployed") - - await spawn_etcd( - self._gstate, - new=True, - token=token, - hostname=hostname, - address=address, - ) - async def deploy( self, config: DeploymentConfig, @@ -485,11 +455,6 @@ async def _assimilate_devices() -> None: self._progress = ProgressEnum.PREPARING await self._set_hostname(hostname) - try: - await self._prepare_etcd(hostname, address, token) - except NodeError as e: - logger.error(f"bootstrap prepare error: {e.message}") - raise e await self._set_ntp_addr(ntp_addr) diff --git a/src/gravel/controllers/nodes/etcd.py b/src/gravel/controllers/nodes/etcd.py deleted file mode 100644 index 10b974854..000000000 --- a/src/gravel/controllers/nodes/etcd.py +++ /dev/null @@ -1,163 +0,0 @@ -# project aquarium's backend -# Copyright (C) 2021 SUSE, LLC. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -import asyncio -import multiprocessing -import shlex -from logging import Logger -from pathlib import Path -from typing import List, Optional - -from fastapi.logger import logger as fastapi_logger - -from gravel.controllers.errors import GravelError -from gravel.controllers.gstate import GlobalState - - -class ContainerFetchError(GravelError): - pass - - -logger: Logger = fastapi_logger - - -# We need to rely on "spawn" because otherwise the subprocess' signal -# handler will play nasty tricks with uvicorn's and fastapi's signal -# handlers. -# For future reference, -# - uvicorn issue: https://github.com/encode/uvicorn/issues/548 -# - python bug report: https://bugs.python.org/issue43064 -# - somewhat of a solution, using "spawn" for multiprocessing: -# https://github.com/tiangolo/fastapi/issues/1487#issuecomment-657290725 -# -# And because we need to rely on spawn, which will create a new python -# interpreter to execute what we are specifying, the function we're passing -# needs to be pickled. And nested functions, apparently, can't be pickled. Thus, -# we need to have the functions at the top-level scope. -# -def _bootstrap_etcd_process(cmd: List[str]): - async def _run_etcd(): - process = await asyncio.create_subprocess_exec(*cmd) - await process.wait() - - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(_run_etcd()) - except KeyboardInterrupt: - pass - - -async def spawn_etcd( - gstate: GlobalState, - new: bool, - token: Optional[str], - hostname: str, - address: str, - initial_cluster: Optional[str] = None, -) -> None: - - assert hostname - assert address - - data_dir: Path = Path(gstate.config.options.etcd.data_dir) - if not data_dir.exists(): - data_dir.mkdir(0o700) - - logger.info(f"starting etcd, hostname: {hostname}, addr: {address}") - - def _get_etcd_args() -> List[str]: - client_url: str = f"http://{address}:2379" - peer_url: str = f"http://{address}:2380" - - nonlocal initial_cluster - if not initial_cluster: - initial_cluster = f"{hostname}={peer_url}" - - args: List[str] = [ - "--name", - hostname, - "--initial-advertise-peer-urls", - peer_url, - "--listen-peer-urls", - peer_url, - "--listen-client-urls", - f"{client_url},http://127.0.0.1:2379", - "--advertise-client-urls", - client_url, - "--initial-cluster", - initial_cluster, - "--data-dir", - str(data_dir), - ] - - if new: - assert token - args += ["--initial-cluster-state", "new"] - args += ["--initial-cluster-token", token] - else: - args += ["--initial-cluster-state", "existing"] - - return args - - def _get_container_cmd() -> List[str]: - registry = gstate.config.options.etcd.registry - version = gstate.config.options.etcd.version - - return [ - "podman", - "run", - "--rm", - "--replace", - "--net=host", - "--entrypoint", - "/usr/local/bin/etcd", - "-v", - f"{data_dir}:{data_dir}", - "--name", - f"etcd.{hostname}", - f"{registry}:{version}", - ] + _get_etcd_args() - - cmd = _get_container_cmd() - - ctx = multiprocessing.get_context("spawn") - logger.debug("spawn etcd: " + shlex.join(cmd)) - process = ctx.Process(target=_bootstrap_etcd_process, args=(cmd,)) - process.start() - - logger.info(f"started etcd process pid = {process.pid}") - await gstate.init_store() - - -async def etcd_pull_image(gstate: GlobalState) -> None: - registry = gstate.config.options.etcd.registry - version = gstate.config.options.etcd.version - - logger.debug(f"fetching etcd image from {registry}:{version}") - - cmd = shlex.split(f"podman pull {registry}:{version}") - process = await asyncio.create_subprocess_exec( - *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - - try: - # wait for 5 minutes - retcode = await asyncio.wait_for(process.wait(), 300) - if retcode != 0: - stderr = "unknown error" - if process.stderr: - stderr = (await process.stderr.read()).decode("utf-8") - raise ContainerFetchError(stderr) - except TimeoutError: - raise ContainerFetchError("timed out") diff --git a/src/gravel/controllers/nodes/messages.py b/src/gravel/controllers/nodes/messages.py index 3faec08bf..b430c2d44 100644 --- a/src/gravel/controllers/nodes/messages.py +++ b/src/gravel/controllers/nodes/messages.py @@ -41,7 +41,6 @@ class WelcomeMessageModel(BaseModel): pubkey: str cephconf: str keyring: str - etcd_peer: str class ReadyToAddMessageModel(BaseModel): diff --git a/src/gravel/controllers/nodes/mgr.py b/src/gravel/controllers/nodes/mgr.py index 57bcc36e7..1766cc2e9 100644 --- a/src/gravel/controllers/nodes/mgr.py +++ b/src/gravel/controllers/nodes/mgr.py @@ -19,7 +19,6 @@ from typing import Dict, List, Optional from uuid import UUID, uuid4 -import aetcd3 from fastapi import status from fastapi.logger import logger as fastapi_logger from pydantic import BaseModel, Field @@ -50,11 +49,6 @@ NodeNotStartedError, NodeShuttingDownError, ) -from gravel.controllers.nodes.etcd import ( - ContainerFetchError, - etcd_pull_image, - spawn_etcd, -) from gravel.controllers.nodes.messages import ( ErrorMessageModel, JoinMessageModel, @@ -92,7 +86,6 @@ # stage = AVAILABLE # # _node_start() [state: none || available] -# obtain etcd state # load state # start connmgr # stage = STARTED @@ -201,13 +194,6 @@ async def start(self) -> None: assert self.deployment_state.ready or self.deployment_state.deployed assert self._state.hostname assert self._state.address - await spawn_etcd( - self.gstate, - new=False, - token=None, - hostname=self._state.hostname, - address=self._state.address, - ) await self._start_ceph() await self._node_start() @@ -217,12 +203,11 @@ async def shutdown(self) -> None: async def _obtain_images(self) -> bool: cephadm = self.gstate.cephadm try: - await asyncio.gather( - cephadm.pull_images(), etcd_pull_image(self.gstate) - ) - except ContainerFetchError as e: - logger.error(f"unable to fetch containers: {e.message}") - return False + # Since removing etcd this gather structure is redundant + # (we're only invoking one awaitable), but I've left it + # here anyway in case it turns out we want to add other + # container images in future. + await asyncio.gather(cephadm.pull_images()) except CephadmError as e: logger.error(f"unable to fetch ceph containers: {str(e)}") return False @@ -623,32 +608,8 @@ async def _handle_join( logger.debug(f"handle join > pubkey: {pubkey}") - logger.debug("handle join > connect etcd client") - etcd: aetcd3.Etcd3Client = aetcd3.client() - peer_url: str = f"http://{msg.address}:2380" - logger.debug(f"handle join > add '{peer_url}' to etcd") - member, nodes = await etcd.add_member([peer_url]) - await etcd.close() - assert member is not None - assert nodes is not None - assert len(nodes) > 0 - - member_urls: str = ",".join( - [ - f"{m.name}={m.peer_urls[0]}" - for m in nodes - if (len(m.peer_urls) > 0 and len(m.name) > 0) - ] - ) - logger.debug( - f"{member_urls=}, member: {member.name}={member.peer_urls[0]}" - ) - welcome = WelcomeMessageModel( - pubkey=pubkey, - cephconf=cephconf, - keyring=keyring, - etcd_peer=member_urls, + pubkey=pubkey, cephconf=cephconf, keyring=keyring ) try: logger.debug(f"handle join > send welcome: {welcome}") diff --git a/src/gravel/controllers/nodes/systemdisk.py b/src/gravel/controllers/nodes/systemdisk.py index 27b00880f..610a5f968 100644 --- a/src/gravel/controllers/nodes/systemdisk.py +++ b/src/gravel/controllers/nodes/systemdisk.py @@ -78,7 +78,6 @@ class SystemDisk: _overlaydirs: Dict[str, str] = { "etc": "/etc", "logs": "/var/log", - "etcd": "/var/lib/etcd", "aquarium": "/var/lib/aquarium", "containers": "/var/lib/containers", "roothome": "/root", diff --git a/src/gravel/tests/conftest.py b/src/gravel/tests/conftest.py index 722d4ee7f..c7139d81c 100644 --- a/src/gravel/tests/conftest.py +++ b/src/gravel/tests/conftest.py @@ -52,27 +52,6 @@ def __str__(self): ) -def mock_aetcd_modules(mocker: MockerFixture): - class MockAetcd3Error(Exception): - def __init__(self, message: str, errno: Optional[int] = None): - super().__init__(message) - self.errno = errno - - def __str__(self): - msg = super().__str__() - if self.errno is None: - return msg - return f"[errno {self.errno}] {msg}" - - sys.modules.update( - { - "aetcd3.etcdrpc": mocker.MagicMock( - Error=MockAetcd3Error, OSError=MockAetcd3Error - ) - } - ) - - @pytest.fixture(params=["default_ceph.conf"]) def ceph_conf_file_fs(request: SubRequest, fs: fake_filesystem.FakeFilesystem): """This fixture uses pyfakefs to stub filesystem calls and return @@ -213,17 +192,8 @@ async def startup(aquarium_app: FastAPI, aquarium_api: FastAPI): logger: logging.Logger = fastapi_logger class FakeNodeDeployment(NodeDeployment): - async def _prepare_etcd( - self, hostname: str, address: str, token: str - ) -> None: - assert self._state - if self._state.bootstrapping: - raise NodeCantDeployError("node bootstrapping") - elif not self._state.nostage: - raise NodeCantDeployError("node can't be bootstrapped") - - # We don't need to spawn etcd, just allow gstate to init store - await self._gstate.init_store() + # Do we still need this thing since removing etcd? + pass class FakeNodeMgr(NodeMgr): def __init__(self, gstate: GlobalState): @@ -248,7 +218,6 @@ async def start(self) -> None: ) assert self._state.hostname assert self._state.address - # We don't need to spawn etcd, just allow gstate to init store await self.gstate.init_store() async def _obtain_images(self) -> bool: diff --git a/src/gravel/tests/unit/controllers/nodes/test_deployment.py b/src/gravel/tests/unit/controllers/nodes/test_deployment.py index b5254ddb5..badff2ca7 100644 --- a/src/gravel/tests/unit/controllers/nodes/test_deployment.py +++ b/src/gravel/tests/unit/controllers/nodes/test_deployment.py @@ -206,7 +206,6 @@ def mock_host_exists(cls, hostname: str) -> bool: # type: ignore fake_connmgr: ConnMgr = cast(ConnMgr, mocker.MagicMock()) deployment = NodeDeployment(gstate, fake_connmgr) - deployment._prepare_etcd = mocker.AsyncMock() deployment._set_hostname = mock_set_hostname deployment._set_ntp_addr = mock_set_ntpaddr diff --git a/src/gravel/tests/unit/controllers/nodes/test_etcd.py b/src/gravel/tests/unit/controllers/nodes/test_etcd.py deleted file mode 100644 index e9ec92baf..000000000 --- a/src/gravel/tests/unit/controllers/nodes/test_etcd.py +++ /dev/null @@ -1,140 +0,0 @@ -# project aquarium's backend -# Copyright (C) 2021 SUSE, LLC. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# pyright: reportPrivateUsage=false, reportUnknownMemberType=false -# pyright: reportMissingTypeStubs=false - - -from typing import Callable, List, cast - -import pytest -from pyfakefs import fake_filesystem -from pytest_mock import MockerFixture - -from gravel.controllers.gstate import GlobalState -from gravel.tests.unit.asyncio import FakeProcess - - -def test_bootstrap_process(mocker: MockerFixture) -> None: - async def mock_subprocess(*args: List[str]) -> FakeProcess: - assert len(args) > 0 - assert "foo" in args - return FakeProcess(stdout=None, stderr=None, ret=0) - - mocker.patch("asyncio.create_subprocess_exec", new=mock_subprocess) - from gravel.controllers.nodes.etcd import _bootstrap_etcd_process - - _bootstrap_etcd_process(["foo", "bar"]) - - -@pytest.mark.asyncio -async def test_spawn_etcd( - mocker: MockerFixture, - fs: fake_filesystem.FakeFilesystem, - gstate: GlobalState, -) -> None: - class FakeProcessCtx: - pid = 1234 - - def __init__( - self, target: Callable[[List[str]], None], args: List[str] - ) -> None: - assert target is not None - assert len(args) > 0 - - def start(self) -> None: - pass - - async def mock_subprocess(*args: List[str]) -> FakeProcess: - assert len(args) > 0 - assert "podman" in args - return FakeProcess(stdout=None, stderr=None, ret=0) - - import multiprocessing.context - - mocker.patch.object( - multiprocessing.context.SpawnContext, "Process", new=FakeProcessCtx - ) - mocker.patch("asyncio.create_subprocess_exec", new=mock_subprocess) - from gravel.controllers.nodes.etcd import spawn_etcd - - fs.makedirs("/var/lib") - await spawn_etcd(gstate, True, "asd-fgh-jkl", "foobar", "1.1.1.1") - assert fs.exists(gstate.config.options.etcd.data_dir) - - -def assert_pull_image( - *args: List[str], stdout: int, stderr: int, registry: str, version: str -) -> None: - import asyncio - - assert len(args) > 0 - assert args[0] == "podman" - assert args[1] == "pull" - assert len(args[2]) > 0 - image = cast(str, args[2]).split(":") - assert len(image) == 2 - assert image[0] == registry - assert image[1] == version - assert stdout == asyncio.subprocess.PIPE - assert stderr == asyncio.subprocess.PIPE - - -@pytest.mark.asyncio -async def test_etcd_pull_image( - mocker: MockerFixture, gstate: GlobalState -) -> None: - async def mock_subprocess( - *args: List[str], stdout: int, stderr: int - ) -> FakeProcess: - assert_pull_image( - *args, - stdout=stdout, - stderr=stderr, - registry=gstate.config.options.etcd.registry, - version=gstate.config.options.etcd.version, - ) - return FakeProcess(stdout="", stderr="", ret=0) - - mocker.patch("asyncio.create_subprocess_exec", new=mock_subprocess) - from gravel.controllers.nodes.etcd import etcd_pull_image - - await etcd_pull_image(gstate) - - -@pytest.mark.asyncio -async def test_fail_etcd_pull_image( - mocker: MockerFixture, gstate: GlobalState -) -> None: - async def mock_subprocess( - *args: List[str], stdout: int, stderr: int - ) -> FakeProcess: - assert_pull_image( - *args, - stdout=stdout, - stderr=stderr, - registry=gstate.config.options.etcd.registry, - version=gstate.config.options.etcd.version, - ) - return FakeProcess(stdout=None, stderr="foobar", ret=1) - - mocker.patch("asyncio.create_subprocess_exec", new=mock_subprocess) - from gravel.controllers.nodes.etcd import ( - ContainerFetchError, - etcd_pull_image, - ) - - try: - await etcd_pull_image(gstate) - except ContainerFetchError as e: - assert e.message == "foobar" diff --git a/src/gravel/tests/unit/controllers/nodes/test_mgr.py b/src/gravel/tests/unit/controllers/nodes/test_mgr.py index e7823816d..f70d61d44 100644 --- a/src/gravel/tests/unit/controllers/nodes/test_mgr.py +++ b/src/gravel/tests/unit/controllers/nodes/test_mgr.py @@ -138,30 +138,10 @@ async def test_mgr_start( hostname="foobar", ) - called_etcd_spawn = False - - async def mock_spawn_etcd( - gstate: GlobalState, - new: bool, - token: Optional[str], - hostname: str, - address: str, - initial_cluster: Optional[str] = None, - ) -> None: - assert not new - assert token is None - assert hostname == "foobar" - assert address == "1.2.3.4" - assert initial_cluster is None - nonlocal called_etcd_spawn - called_etcd_spawn = True - - mocker.patch("gravel.controllers.nodes.mgr.spawn_etcd", new=mock_spawn_etcd) nodemgr._start_ceph = mocker.AsyncMock() nodemgr._node_start = mocker.AsyncMock() await nodemgr.start() - assert called_etcd_spawn nodemgr._start_ceph.assert_called_once() # type: ignore nodemgr._node_start.assert_called_once() # type: ignore @@ -171,15 +151,6 @@ async def test_obtain_images( gstate: GlobalState, mocker: MockerFixture ) -> None: - called_etcd_pull_image = False - - async def mock_etcd_pull_img(gstate: GlobalState) -> None: - nonlocal called_etcd_pull_image - called_etcd_pull_image = True - - mocker.patch( - "gravel.controllers.nodes.mgr.etcd_pull_image", new=mock_etcd_pull_img - ) orig_cephadm_pull_img = gstate.cephadm.pull_images gstate.cephadm.pull_images = mocker.AsyncMock() @@ -187,7 +158,6 @@ async def mock_etcd_pull_img(gstate: GlobalState) -> None: ret = await nodemgr._obtain_images() assert ret gstate.cephadm.pull_images.assert_called_once() # type: ignore - assert called_etcd_pull_image from gravel.cephadm.cephadm import CephadmError @@ -198,25 +168,6 @@ async def mock_etcd_pull_img(gstate: GlobalState) -> None: assert not ret gstate.cephadm.pull_images.assert_called_once() # type: ignore - from gravel.controllers.nodes.etcd import ContainerFetchError - - called_etcd_pull_image_fail = False - - async def fail_etcd_pull_img(gstate: GlobalState) -> None: - nonlocal called_etcd_pull_image_fail - called_etcd_pull_image_fail = True - raise ContainerFetchError("barbaz") - - mocker.patch( - "gravel.controllers.nodes.mgr.etcd_pull_image", new=fail_etcd_pull_img - ) - gstate.cephadm.pull_images = mocker.AsyncMock() - - ret = await nodemgr._obtain_images() - assert not ret - assert called_etcd_pull_image_fail - gstate.cephadm.pull_images.assert_called_once() # type: ignore - gstate.cephadm.pull_images = orig_cephadm_pull_img @@ -934,7 +885,6 @@ def conn_cb(data: MessageModel) -> None: assert msg.pubkey == "mypubkey" assert msg.cephconf == "mycephconf" assert msg.keyring == "mycephkeyring" - assert msg.etcd_peer == "asd=10.11.12.13" nonlocal called_conn_cb called_conn_cb = True @@ -950,17 +900,6 @@ def conn_cb(data: MessageModel) -> None: with open("/etc/ceph/ceph.client.admin.keyring", mode="w") as f: f.write("mycephkeyring") - class FakeAETCD: - async def add_member( - self, urls: List[str] - ) -> Tuple[Member, List[Member]]: - return await mock_add_member(urls) - - async def close(self) -> None: - pass - - mocker.patch("aetcd3.client", new=FakeAETCD) - conn = fake_conn(conn_cb) await nodemgr._handle_join( conn, diff --git a/src/gravel/typings/aetcd3/__init__.pyi b/src/gravel/typings/aetcd3/__init__.pyi deleted file mode 100644 index 3680b088b..000000000 --- a/src/gravel/typings/aetcd3/__init__.pyi +++ /dev/null @@ -1,17 +0,0 @@ -# ignore because we're not using it: from . import etcdrpc -from .client import Etcd3Client, Transactions, client -from .exceptions import Etcd3Exception # type: ignore -from .leases import Lease # type: ignore -from .locks import Lock -from .members import Member # type: ignore - -__all__ = ( - "Etcd3Client", - "Etcd3Exception", - "Lease", - "Lock", - "Member", - "Transactions", - "client", - # 'etcdrpc', -) diff --git a/src/gravel/typings/aetcd3/client.pyi b/src/gravel/typings/aetcd3/client.pyi deleted file mode 100644 index 08023995b..000000000 --- a/src/gravel/typings/aetcd3/client.pyi +++ /dev/null @@ -1,168 +0,0 @@ -# project aquarium's backend -# Copyright (C) 2021 SUSE, LLC. - -# pyright: reportMissingTypeStubs=false -from typing import ( - IO, - Any, - AsyncIterator, - Awaitable, - Callable, - Dict, - Generator, - List, - Optional, - Tuple, - Union, -) - -from aetcd3.etcdrpc.rpc_pb2 import ( - Compare, - DeleteRangeResponse, - LeaseKeepAliveResponse, - LeaseRevokeResponse, - LeaseTimeToLiveResponse, - PutResponse, - RequestOp, - ResponseOp, -) -from aetcd3.events import Event -from aetcd3.leases import Lease -from aetcd3.locks import Lock -from aetcd3.members import Member - -class Transactions: ... - -class KVMetadata: - key: bytes - create_revision: int - mod_revision: int - version: int - lease_id: int - ... - -class Status: ... -class Alarm: ... - -class Etcd3Client: - def __init__( - self, - host: str = ..., - port: int = ..., - ca_cert: Optional[str] = ..., - cert_key: Optional[str] = ..., - cert_cert: Optional[str] = ..., - timeout: Optional[float] = ..., - user: Optional[str] = ..., - password: Optional[str] = ..., - gprc_options: Dict[Any, Any] = ..., - ) -> None: ... - async def open(self) -> None: ... - async def close(self) -> None: ... - async def __aenter__(self) -> Any: ... - async def __aexit__(self, *args: Any) -> Any: ... - async def get( - self, key: str, serializable: bool = ... - ) -> Tuple[bytes, KVMetadata]: ... - async def get_prefix( - self, - key_prefix: str, - sort_order: Optional[str] = ..., - sort_target: str = ..., - keys_only: bool = ..., - ) -> Generator[Tuple[bytes, KVMetadata], None, None]: ... - async def get_range( - self, - range_start: str, - range_end: str, - sort_order: Optional[str] = ..., - sort_target: str = ..., - **kwargs: Any, - ) -> Generator[Tuple[bytes, KVMetadata], None, None]: ... - async def get_all( - self, - sort_order: Optional[str] = ..., - sort_target: Optional[str] = ..., - keys_only: bool = ..., - ) -> Generator[Tuple[bytes, KVMetadata], None, None]: ... - ... - async def put( - self, - key: str, - value: Union[bytes, str], - lease: Optional[Union[Lease, int]] = ..., - prev_kv: bool = ..., - ) -> PutResponse: ... - async def replace( - self, - key: str, - initial_value: Union[bytes, str], - new_value: Union[bytes, str], - ) -> bool: ... - async def delete( - self, key: str, prev_kv: bool = ..., return_response: bool = ... - ) -> Union[bool, DeleteRangeResponse]: ... - async def delete_prefix(self, prefix: str) -> DeleteRangeResponse: ... - async def status(self) -> Status: ... - async def add_watch_callback( - self, - key: str, - callback: Callable[[Event], Awaitable[None]], - **kwargs: Any, - ) -> int: ... - async def watch( - self, key: str, **kwargs: Any - ) -> Tuple[AsyncIterator[Event], Awaitable[None]]: ... - async def watch_prefix( - self, key_prefix: str, **kwargs: Any - ) -> Tuple[AsyncIterator[Event], Awaitable[None]]: ... - async def watch_once( - self, key: str, timeout: Optional[float] = ..., **kwargs: Any - ) -> Any: ... - async def watch_prefix_once( - self, key_prefix: str, timeout: Optional[float] = ..., **kwargs: Any - ) -> Any: ... - async def cancel_watch(self, watch_id: int) -> None: ... - async def transaction( - self, - compare: List[Compare], - success: Optional[List[RequestOp]] = ..., - failure: Optional[List[RequestOp]] = ..., - ) -> Tuple[bool, List[ResponseOp]]: ... - async def lease(self, ttl: int, id: int) -> Lease: ... - async def revoke_lease(self, lease_id: int) -> LeaseRevokeResponse: ... - async def refresh_lease(self, lease_id: int) -> LeaseKeepAliveResponse: ... - async def get_lease_info( - self, lease_id: int, *, keys: bool = ... - ) -> LeaseTimeToLiveResponse: ... - def lock(self, name: str, ttl: int = ...) -> Lock: ... - async def add_member( - self, urls: List[str] - ) -> Tuple[Member, List[Member]]: ... - async def remove_member(self, member_id: int) -> None: ... - async def update_member( - self, member_id: int, peer_urls: List[str] - ) -> None: ... - async def members(self) -> Generator[Member, None, None]: ... - async def compact(self, revision: int, physical: bool = ...) -> None: ... - async def defragment(self) -> None: ... - async def hash(self) -> int: ... - async def create_alarm(self, member_id: int = ...) -> List[Alarm]: ... - async def list_alarms( - self, member_id: int, alarm_type: str = ... - ) -> Generator[Alarm, None, None]: ... - async def disarm_alarm(self, member_id: int = ...) -> List[Alarm]: ... - async def snapshot(self, file_obj: IO[bytes]) -> None: ... - ... - -def client( - host: str = ..., - port: int = ..., - ca_cert: Optional[str] = ..., - cert_key: Optional[str] = ..., - timeout: Optional[float] = ..., - cert_cert: Optional[str] = ..., - user: Optional[str] = ..., - password: Optional[str] = ..., - **kwargs: Any, -) -> Etcd3Client: ... diff --git a/src/gravel/typings/aetcd3/events.pyi b/src/gravel/typings/aetcd3/events.pyi deleted file mode 100644 index ae59389cf..000000000 --- a/src/gravel/typings/aetcd3/events.pyi +++ /dev/null @@ -1,7 +0,0 @@ -# project aquarium's backend -# Copyright (C) 2021 SUSE, LLC. - -class Event: - - key: bytes - value: bytes diff --git a/src/gravel/typings/aetcd3/locks.pyi b/src/gravel/typings/aetcd3/locks.pyi deleted file mode 100644 index 20aeb26d3..000000000 --- a/src/gravel/typings/aetcd3/locks.pyi +++ /dev/null @@ -1,20 +0,0 @@ -# project aquarium's backend -# Copyright (C) 2021 SUSE, LLC. - -# pyright: reportMissingTypeStubs=false -from typing import Optional - -import aetcd3 -from aetcd3.etcdrpc.rpc_pb2 import LeaseKeepAliveResponse - -class Lock: - def __init__( - self, - name: str, - ttl: int = ..., - etcd_client: Optional[aetcd3.Etcd3Client] = ..., - ) -> None: ... - async def acquire(self, timeout: int = ...) -> bool: ... - async def release(self) -> bool: ... - async def refresh(self) -> LeaseKeepAliveResponse: ... - async def is_acquired(self) -> bool: ... diff --git a/src/gravel/typings/aetcd3/members.pyi b/src/gravel/typings/aetcd3/members.pyi deleted file mode 100644 index ca2da22f3..000000000 --- a/src/gravel/typings/aetcd3/members.pyi +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Any, List - -class Member: - - id: int - name: str - peer_urls: List[str] - client_urls: List[str] - _etcd_client: Any - def __init__( - self, - id: int, - name: str, - peer_urls: List[str], - client_urls: List[str], - etcd_client: Any = ..., - ) -> None: ... diff --git a/src/mypy.ini b/src/mypy.ini index e1beefc7b..dee4ca45a 100644 --- a/src/mypy.ini +++ b/src/mypy.ini @@ -9,6 +9,3 @@ allow_redefinition = True [mypy-pyfakefs.*] ignore_missing_imports = True - -[mypy-aetcd3.*] -ignore_missing_imports = True diff --git a/src/requirements.txt b/src/requirements.txt index df862aeba..7bf0238d4 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -11,6 +11,5 @@ uvicorn==0.13.3 pip websockets==8.1 PyYAML==5.4.1 -git+https://github.com/aquarist-labs/aetcd3/@edf633045ce61c7bbac4d4a6ca15b14f8acfe9cd kiwi kiwi-boxed-plugin