Merge 'Introduce a scylla-native nodetool' from Botond Dénes

This series introduces a scylla-native nodetool.  It is invokable via the main scylla executable as the other native tools we have. It uses the seastar's new `http::client` to connect to the specified node and execute the desired commands.
For now a single command is implemented: `nodetool compact`, invokable as `scylla nodetool compact`. Once all the boilerplate is added to create a new tool, implementing a single command is not too bad, in terms of code-bloat. Certainly not as clean as a python implementation would be, but good enough. The advantages of a C++ implementation is that all of us in the core team know C++ and that it is shipped right as part of the scylla executable..

Closes #14841

* github.com:scylladb/scylladb:
  test: add nodetool tests
  test.py: add ToolTestSuite and ToolTest
  tools/scylla-nodetool: implement compact operation
  tools/scylla-nodetool: implement basic scylla_rest_api_client
  tools: introduce scylla-nodetool
  utils: export dns_connection_factory from s3/client.cc to http.hh
  utils/s3/client: pass logger to dns_connection_factory in constructor
  tools/utils: tool_app_template::run_async(): also detect --help* as --help

configure.py

@@ -1245,7 +1245,7 @@ def generate_compdb(compdb, buildfile, modes):
 
 scylla_raft_dependencies = scylla_raft_core + ['utils/uuid.cc', 'utils/error_injection.cc']
 
-scylla_tools = ['tools/scylla-types.cc', 'tools/scylla-sstable.cc', 'tools/schema_loader.cc', 'tools/utils.cc', 'tools/lua_sstable_consumer.cc']
+scylla_tools = ['tools/scylla-types.cc', 'tools/scylla-sstable.cc', 'tools/scylla-nodetool.cc', 'tools/schema_loader.cc', 'tools/utils.cc', 'tools/lua_sstable_consumer.cc']
 scylla_perfs = ['test/perf/perf_fast_forward.cc',
                 'test/perf/perf_row_cache_update.cc',
                 'test/perf/perf_simple_query.cc',

main.cc

@@ -1959,6 +1959,7 @@ int main(int ac, char** av) {
         {"server", scylla_main, "the scylladb server"},
         {"types", tools::scylla_types_main, "a command-line tool to examine values belonging to scylla types"},
         {"sstable", tools::scylla_sstable_main, "a multifunctional command-line tool to examine the content of sstables"},
+        {"nodetool", tools::scylla_nodetool_main, "a command-line tool to administer local or remote ScyllaDB nodes"},
         {"perf-fast-forward", perf::scylla_fast_forward_main, "run performance tests by fast forwarding the reader on this server"},
         {"perf-row-cache-update", perf::scylla_row_cache_update_main, "run performance tests by updating row cache on this server"},
         {"perf-tablets", perf::scylla_tablets_main, "run performance tests of tablet metadata management"},

test.py

@@ -500,6 +500,31 @@ def pattern(self) -> str:
         return "run"
 
 
+class ToolTestSuite(TestSuite):
+    """A collection of Python pytests that test tools
+
+    These tests do not need an cluster setup for them. They invoke scylla
+    manually, in tool mode.
+    """
+
+    def __init__(self, path, cfg: dict, options: argparse.Namespace, mode: str) -> None:
+        super().__init__(path, cfg, options, mode)
+
+    def build_test_list(self) -> List[str]:
+        """For pytest, search for directories recursively"""
+        path = self.suite_path
+        pytests = itertools.chain(path.rglob("*_test.py"), path.rglob("test_*.py"))
+        return [os.path.splitext(t.relative_to(self.suite_path))[0] for t in pytests]
+
+    @property
+    def pattern(self) -> str:
+        assert False
+
+    async def add_test(self, shortname) -> None:
+        test = ToolTest(self.next_id((shortname, self.suite_key)), shortname, self)
+        self.tests.append(test)
+
+
 class Test:
     """Base class for CQL, Unit and Boost tests"""
     def __init__(self, test_no: int, shortname: str, suite) -> None:
@@ -949,6 +974,53 @@ async def run(self, options: argparse.Namespace) -> Test:
         return self
 
 
+class ToolTest(Test):
+    """Run a collection of pytest test cases
+
+    That do not need a scylla cluster set-up for them."""
+
+    def __init__(self, test_no: int, shortname: str, suite) -> None:
+        super().__init__(test_no, shortname, suite)
+        self.path = "pytest"
+        self.xmlout = os.path.join(self.suite.options.tmpdir, self.mode, "xml", self.uname + ".xunit.xml")
+        ToolTest._reset(self)
+
+    def _prepare_pytest_params(self, options: argparse.Namespace):
+        self.args = [
+            "-s",  # don't capture print() output inside pytest
+            "--log-level=DEBUG",   # Capture logs
+            "-o",
+            "junit_family=xunit2",
+            "--junit-xml={}".format(self.xmlout),
+            f"--mode={self.mode}"]
+        if options.markers:
+            self.args.append(f"-m={options.markers}")
+
+            # https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
+            no_tests_selected_exit_code = 5
+            self.valid_exit_codes = [0, no_tests_selected_exit_code]
+        self.args.append(str(self.suite.suite_path / (self.shortname + ".py")))
+
+    def _reset(self) -> None:
+        """Reset the test before a retry, if it is retried as flaky"""
+        pass
+
+    def print_summary(self) -> None:
+        print("Output of {} {}:".format(self.path, " ".join(self.args)))
+
+    async def run(self, options: argparse.Namespace) -> Test:
+        self._prepare_pytest_params(options)
+
+        loggerPrefix = self.mode + '/' + self.uname
+        logger = LogPrefixAdapter(logging.getLogger(loggerPrefix), {'prefix': loggerPrefix})
+        self.success = await run_test(self, options)
+        logger.info("Test %s %s", self.uname, "succeeded" if self.success else "failed ")
+        return self
+
+    def write_junit_failure_report(self, xml_res: ET.Element) -> None:
+        super().write_junit_failure_report(xml_res)
+
+
 class TabularConsoleOutput:
     """Print test progress to the console"""
 

test/nodetool/conftest.py

@@ -0,0 +1,172 @@
+#
+# Copyright 2023-present ScyllaDB
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+
+import os
+import pytest
+import random
+import rest_api_mock
+import subprocess
+import sys
+import requests.exceptions
+import time
+
+from rest_api_mock import expected_request
+
+
+def pytest_addoption(parser):
+    parser.addoption('--mode', action='store', default='dev',
+                     help='Scylla build mode to use')
+    parser.addoption('--nodetool', action='store', choices=["scylla", "cassandra"], default="scylla",
+                     help="Which nodetool implementation to run the tests against")
+    parser.addoption('--nodetool-path', action='store', default=None,
+                     help="Path to the nodetool binary,"
+                     " with --nodetool=scylla, this should be the scylla binary,"
+                     " with --nodetool=cassandra, this should be the nodetool binary")
+    parser.addoption('--jmx-path', action='store', default=None,
+                     help="Path to the jmx binary, only used with --nodetool=cassandra")
+
+
+@pytest.fixture(scope="session")
+def rest_api_mock_server():
+    ip = f"127.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
+    port = random.randint(10000, 65535)
+
+    server_process = subprocess.Popen([
+        sys.executable,
+        os.path.join(os.path.dirname(__file__), "rest_api_mock.py"),
+        ip,
+        str(port)])
+
+    server = (ip, port)
+
+    i = 0
+    while True:
+        try:
+            rest_api_mock.get_expected_requests(server)
+            break
+        except requests.exceptions.ConnectionError:
+            if i == 50:  # 5 seconds
+                raise
+            time.sleep(0.1)
+            i += 1
+
+    try:
+        yield server
+    finally:
+        server_process.terminate()
+        server_process.wait()
+
+
+@pytest.fixture(scope="session")
+def jmx(request, rest_api_mock_server):
+    if request.config.getoption("nodetool") == "scylla":
+        yield
+        return
+
+    jmx_path = request.config.getoption("jmx_path")
+    if jmx_path is None:
+        jmx_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scylla-jmx", "scripts",
+                                                "scylla-jmx"))
+    else:
+        jmx_path = os.path.abspath(jmx_path)
+
+    workdir = os.path.join(os.path.dirname(jmx_path), "..")
+    ip, api_port = rest_api_mock_server
+    expected_requests = [
+            expected_request(
+                "GET",
+                "/column_family/",
+                response=[{"ks": "system_schema",
+                           "cf": "columns",
+                           "type": "ColumnFamilies"},
+                          {"ks": "system_schema",
+                           "cf": "computed_columns",
+                           "type": "ColumnFamilies"}]),
+            expected_request(
+                "GET",
+                "/stream_manager/",
+                response=[])]
+    rest_api_mock.set_expected_requests(rest_api_mock_server, expected_requests)
+
+    # Our nodetool launcher script ignores the host param, so this has to be 127.0.0.1, matching the internal default.
+    jmx_ip = "127.0.0.1"
+    jmx_port = random.randint(10000, 65535)
+    while jmx_port == api_port:
+        jmx_port = random.randint(10000, 65535)
+
+    jmx_process = subprocess.Popen(
+            [
+                jmx_path,
+                "-a", ip,
+                "-p", str(api_port),
+                "-ja", jmx_ip,
+                "-jp", str(jmx_port),
+            ],
+            cwd=workdir, text=True)
+
+    # Wait until jmx starts up
+    # We rely on the expected requests being consumed for this
+    i = 0
+    while len(rest_api_mock.get_expected_requests(rest_api_mock_server)) > 0:
+        if i == 50:  # 5 seconds
+            raise RuntimeError("timed out waiting for JMX to start")
+        time.sleep(0.1)
+        i += 1
+
+    yield jmx_ip, jmx_port
+
+    jmx_process.terminate()
+    jmx_process.wait()
+
+
+@pytest.fixture(scope="session")
+def nodetool_path(request):
+    if request.config.getoption("nodetool") == "scylla":
+        mode = request.config.getoption("mode")
+        return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "build", mode, "scylla"))
+
+    path = request.config.getoption("nodetool_path")
+    if path is not None:
+        return os.path.abspath(path)
+
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "tools", "java", "bin", "nodetool"))
+
+
+@pytest.fixture(scope="function")
+def scylla_only(request):
+    if request.config.getoption("nodetool") != "scylla":
+        pytest.skip('Scylla-only test skipped')
+
+
+@pytest.fixture(scope="module")
+def nodetool(request, jmx, nodetool_path, rest_api_mock_server):
+    def invoker(method, *args, expected_requests=None):
+        if expected_requests is not None:
+            rest_api_mock.set_expected_requests(rest_api_mock_server, expected_requests)
+
+        if request.config.getoption("nodetool") == "scylla":
+            api_ip, api_port = rest_api_mock_server
+            cmd = [nodetool_path, "nodetool", method,
+                   "--logger-log-level", "scylla-nodetool=trace",
+                   "-h", api_ip,
+                   "-p", str(api_port)]
+        else:
+            jmx_ip, jmx_port = jmx
+            cmd = [nodetool_path, "-h", jmx_ip, "-p", str(jmx_port), method]
+        cmd += list(args)
+        res = subprocess.run(cmd, capture_output=True, text=True)
+        sys.stdout.write(res.stdout)
+        sys.stderr.write(res.stderr)
+
+        unconsumed_expected_requests = rest_api_mock.get_expected_requests(rest_api_mock_server)
+        # Clear up any unconsumed requests, so the next test starts with a clean slate
+        rest_api_mock.clear_expected_requests(rest_api_mock_server)
+
+        # Check the return-code first, if the command failed probably not all requests were consumed
+        res.check_returncode()
+        assert len(unconsumed_expected_requests) == 0
+
+    return invoker