Merge pull request #24728 from qoega/remove-enable-dictionaries

Remove explicit enable_dictionaries files in integration tests

docker/test/integration/runner/compose/docker_compose_hdfs.yml

@@ -4,9 +4,9 @@ services:
         image: sequenceiq/hadoop-docker:2.7.0
         hostname: hdfs1
         restart: always
-        ports:
-            - ${HDFS_NAME_EXTERNAL_PORT}:${HDFS_NAME_INTERNAL_PORT} #50070
-            - ${HDFS_DATA_EXTERNAL_PORT}:${HDFS_DATA_INTERNAL_PORT} #50075
+        expose:
+            - ${HDFS_NAME_PORT}
+            - ${HDFS_DATA_PORT}
         entrypoint: /etc/bootstrap.sh -d
         volumes:
             - type: ${HDFS_FS:-tmpfs}

docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml

@@ -14,9 +14,9 @@ services:
       - type: ${KERBERIZED_HDFS_FS:-tmpfs}
         source: ${KERBERIZED_HDFS_LOGS:-}
         target: /var/log/hadoop-hdfs
-    ports:
-      - ${KERBERIZED_HDFS_NAME_EXTERNAL_PORT}:${KERBERIZED_HDFS_NAME_INTERNAL_PORT} #50070
-      - ${KERBERIZED_HDFS_DATA_EXTERNAL_PORT}:${KERBERIZED_HDFS_DATA_INTERNAL_PORT} #1006
+    expose:
+      - ${KERBERIZED_HDFS_NAME_PORT}
+      - ${KERBERIZED_HDFS_DATA_PORT}
     depends_on:
       - hdfskerberos
     entrypoint: /etc/bootstrap.sh -d
@@ -28,4 +28,4 @@ services:
       - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab
       - ${KERBERIZED_HDFS_DIR}/../../kerberos_image_config.sh:/config.sh
       - /dev/urandom:/dev/random
-    ports: [88, 749]
+    expose: [88, 749]

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -10,7 +10,7 @@ echo '{
     "storage-driver": "overlay2",
     "insecure-registries" : ["dockerhub-proxy.sas.yp-c.yandex.net:5000"],
     "registry-mirrors" : ["http://dockerhub-proxy.sas.yp-c.yandex.net:5000"]
-}' | dd of=/etc/docker/daemon.json
+}' | dd of=/etc/docker/daemon.json 2>/dev/null
 
 dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log &
 

tests/integration/conftest.py

@@ -1,32 +1,31 @@
 import subprocess
+from helpers.cluster import run_and_check
 import pytest
 import logging
+import os
 from helpers.test_tools import TSV
 from helpers.network import _NetworkManager
 
 @pytest.fixture(autouse=True, scope="session")
 def cleanup_environment():
     _NetworkManager.clean_all_user_iptables_rules()
     try:
-        result = subprocess.run(['docker', 'container', 'list', '-a', '|', 'wc', '-l'])
-        if result.returncode != 0:
-            logging.error(f"docker ps returned error:{str(result.stderr)}")
-        else:
-            if int(result.stdout) > 1:
-                if env["PYTEST_CLEANUP_CONTAINERS"] != 1:
-                    logging.warning(f"Docker containters({result.stdout}) are running before tests run. They can be left from previous pytest run and cause test failures.\n"\
-                                    "You can set env PYTEST_CLEANUP_CONTAINERS=1 or use runner with --cleanup-containers argument to enable automatic containers cleanup.")
-                else:
-                    logging.debug("Trying to kill unstopped containers...")
-                    subprocess.run(['docker', 'kill', f'`docker container list -a`'])
-                    subprocess.run(['docker', 'rm', f'`docker container list -a`'])
-                    logging.debug("Unstopped containers killed")
-                    r = subprocess.run(['docker-compose', 'ps', '--services', '--all'])
-                    logging.debug(f"Docker ps before start:{r.stdout}")
+        result = run_and_check(['docker ps | wc -l'], shell=True)
+        if int(result) > 1:
+            if int(os.environ.get("PYTEST_CLEANUP_CONTAINERS")) != 1:
+                logging.warning(f"Docker containters({int(result)}) are running before tests run. They can be left from previous pytest run and cause test failures.\n"\
+                                "You can set env PYTEST_CLEANUP_CONTAINERS=1 or use runner with --cleanup-containers argument to enable automatic containers cleanup.")
             else:
-                logging.debug(f"No running containers")
+                logging.debug("Trying to kill unstopped containers...")
+                run_and_check([f'docker kill $(docker container list  --all  --quiet)'], shell=True, nothrow=True)
+                run_and_check([f'docker rm $docker container list  --all  --quiet)'], shell=True, nothrow=True)
+                logging.debug("Unstopped containers killed")
+                r = run_and_check(['docker-compose', 'ps', '--services', '--all'])
+                logging.debug(f"Docker ps before start:{r.stdout}")
+        else:
+            logging.debug(f"No running containers")
     except Exception as e:
-        logging.error(f"cleanup_environment:{str(e)}")
+        logging.exception(f"cleanup_environment:{str(e)}")
         pass
 
     yield

tests/integration/helpers/cluster.py

@@ -62,12 +62,14 @@ def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=su
     err = res.stderr.decode('utf-8')
     if res.returncode != 0:
         # check_call(...) from subprocess does not print stderr, so we do it manually
+        logging.debug(f"Command:{args}")
         logging.debug(f"Stderr:{err}")
         logging.debug(f"Stdout:{out}")
         logging.debug(f"Env: {env}")
         if not nothrow:
             raise Exception(f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}")
     else:
+        logging.debug(f"Command:{args}")
         logging.debug(f"Stderr: {err}")
         logging.debug(f"Stdout: {out}")
         return out
@@ -263,15 +265,17 @@ def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=N
         # available when with_hdfs == True
         self.hdfs_host = "hdfs1"
         self.hdfs_ip = None
-        self.hdfs_name_port = get_free_port()
-        self.hdfs_data_port = get_free_port()
+        self.hdfs_name_port = 50070
+        self.hdfs_data_port = 50075
         self.hdfs_dir = p.abspath(p.join(self.instances_dir, "hdfs"))
         self.hdfs_logs_dir = os.path.join(self.hdfs_dir, "logs")
+        self.hdfs_api = None # also for kerberized hdfs
 
         # available when with_kerberized_hdfs == True
         self.hdfs_kerberized_host = "kerberizedhdfs1"
-        self.hdfs_kerberized_name_port = get_free_port()
-        self.hdfs_kerberized_data_port = get_free_port()
+        self.hdfs_kerberized_ip = None
+        self.hdfs_kerberized_name_port = 50070
+        self.hdfs_kerberized_data_port = 1006
         self.hdfs_kerberized_dir = p.abspath(p.join(self.instances_dir, "kerberized_hdfs"))
         self.hdfs_kerberized_logs_dir = os.path.join(self.hdfs_kerberized_dir, "logs")
 
@@ -375,11 +379,11 @@ def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=N
     def cleanup(self):
         # Just in case kill unstopped containers from previous launch
         try:
-            result = run_and_check(['docker', 'container', 'list', '-a', '-f name={self.project_name}', '|', 'wc', '-l'])
+            result = run_and_check(f'docker container list --all --filter name={self.project_name} | wc -l', shell=True)
             if int(result) > 1:
-                logging.debug("Trying to kill unstopped containers...")
-                run_and_check(['docker', 'kill', f'`docker container list -a -f name={self.project_name}`'])
-                run_and_check(['docker', 'rm', f'`docker container list -a -f name={self.project_name}`'])
+                logging.debug(f"Trying to kill unstopped containers for project{self.project_name}...")
+                run_and_check(f'docker kill $(docker container list --all --quiet --filter name={self.project_name})', shell=True)
+                run_and_check(f'docker rm $(docker container list --all  --quiet --filter name={self.project_name})', shell=True)
                 logging.debug("Unstopped containers killed")
                 run_and_check(['docker-compose', 'ps', '--services', '--all'])
             else:
@@ -409,8 +413,10 @@ def cleanup(self):
         try:
             logging.debug("Trying to prune unused volumes...")
 
-            run_and_check(['docker', 'volume', 'prune', '-f'])
-            logging.debug("Volumes pruned")
+            result = run_and_check(['docker volume ls | wc -l'], shell=True)
+            if int(result>0):
+                run_and_check(['docker', 'volume', 'prune', '-f'])
+            logging.debug(f"Volumes pruned: {result}")
         except:
             pass
 
@@ -564,10 +570,8 @@ def setup_postgres_cluster_cmd(self, instance, env_variables, docker_compose_yml
     def setup_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir):
         self.with_hdfs = True
         env_variables['HDFS_HOST'] = self.hdfs_host
-        env_variables['HDFS_NAME_EXTERNAL_PORT'] = str(self.hdfs_name_port)
-        env_variables['HDFS_NAME_INTERNAL_PORT'] = "50070"
-        env_variables['HDFS_DATA_EXTERNAL_PORT'] = str(self.hdfs_data_port)
-        env_variables['HDFS_DATA_INTERNAL_PORT'] = "50075"
+        env_variables['HDFS_NAME_PORT'] = str(self.hdfs_name_port)
+        env_variables['HDFS_DATA_PORT'] = str(self.hdfs_data_port)
         env_variables['HDFS_LOGS'] = self.hdfs_logs_dir
         env_variables['HDFS_FS'] = "bind"
         self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')])
@@ -579,10 +583,8 @@ def setup_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir):
     def setup_kerberized_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir):
         self.with_kerberized_hdfs = True
         env_variables['KERBERIZED_HDFS_HOST'] = self.hdfs_kerberized_host
-        env_variables['KERBERIZED_HDFS_NAME_EXTERNAL_PORT'] = str(self.hdfs_kerberized_name_port)
-        env_variables['KERBERIZED_HDFS_NAME_INTERNAL_PORT'] = "50070"
-        env_variables['KERBERIZED_HDFS_DATA_EXTERNAL_PORT'] = str(self.hdfs_kerberized_data_port)
-        env_variables['KERBERIZED_HDFS_DATA_INTERNAL_PORT'] = "1006"
+        env_variables['KERBERIZED_HDFS_NAME_PORT'] = str(self.hdfs_kerberized_name_port)
+        env_variables['KERBERIZED_HDFS_DATA_PORT'] = str(self.hdfs_kerberized_data_port)
         env_variables['KERBERIZED_HDFS_LOGS'] = self.hdfs_kerberized_logs_dir
         env_variables['KERBERIZED_HDFS_FS'] = "bind"
         env_variables['KERBERIZED_HDFS_DIR'] = instance.path + '/'
@@ -1128,30 +1130,26 @@ def wait_zookeeper_to_start(self, timeout=180):
         raise Exception("Cannot wait ZooKeeper container")
 
     def make_hdfs_api(self, timeout=180, kerberized=False):
-        hdfs_api = None
         if kerberized:
             keytab = p.abspath(p.join(self.instances['node1'].path, "secrets/clickhouse.keytab"))
             krb_conf = p.abspath(p.join(self.instances['node1'].path, "secrets/krb_long.conf"))
-            hdfs_ip = self.get_instance_ip('kerberizedhdfs1')
-            # logging.debug("kerberizedhdfs1 ip ", hdfs_ip)
+            self.hdfs_kerberized_ip = self.get_instance_ip(self.hdfs_kerberized_host)
             kdc_ip = self.get_instance_ip('hdfskerberos')
-            # logging.debug("kdc_ip ", kdc_ip)
-            hdfs_api = HDFSApi(user="root",
-                              timeout=timeout,
-                              kerberized=True,
-                              principal="root@TEST.CLICKHOUSE.TECH",
-                              keytab=keytab,
-                              krb_conf=krb_conf,
-                              host="localhost",
-                              protocol="http",
-                              proxy_port=self.hdfs_kerberized_name_port,
-                              data_port=self.hdfs_kerberized_data_port,
-                              hdfs_ip=hdfs_ip,
-                              kdc_ip=kdc_ip)                         
+            self.hdfs_api = HDFSApi(user="root",
+                                    timeout=timeout,
+                                    kerberized=True,
+                                    principal="root@TEST.CLICKHOUSE.TECH",
+                                    keytab=keytab,
+                                    krb_conf=krb_conf,
+                                    host=self.hdfs_kerberized_host,
+                                    protocol="http",
+                                    proxy_port=self.hdfs_kerberized_name_port,
+                                    data_port=self.hdfs_kerberized_data_port,
+                                    hdfs_ip=self.hdfs_kerberized_ip,
+                                    kdc_ip=kdc_ip)                         
         else:
-            logging.debug("Create HDFSApi host={}".format("localhost"))
-            hdfs_api = HDFSApi(user="root", host="localhost", data_port=self.hdfs_data_port, proxy_port=self.hdfs_name_port)
-        return hdfs_api
+            self.hdfs_ip = self.get_instance_ip(self.hdfs_host)
+            self.hdfs_api = HDFSApi(user="root", host=self.hdfs_host, data_port=self.hdfs_data_port, proxy_port=self.hdfs_name_port, hdfs_ip=self.hdfs_ip)
 
     def wait_kafka_is_available(self, kafka_docker_id, kafka_port, max_retries=50):
         retries = 0
@@ -1166,16 +1164,15 @@ def wait_kafka_is_available(self, kafka_docker_id, kafka_port, max_retries=50):
                 time.sleep(1)
 
 
-    def wait_hdfs_to_start(self, hdfs_api, timeout=300):
-        self.hdfs_ip = self.get_instance_ip(self.hdfs_host)
+    def wait_hdfs_to_start(self, timeout=300):
         start = time.time()
         while time.time() - start < timeout:
             try:
-                hdfs_api.write_data("/somefilewithrandomname222", "1")
+                self.hdfs_api.write_data("/somefilewithrandomname222", "1")
                 logging.debug("Connected to HDFS and SafeMode disabled! ")
                 return
             except Exception as ex:
-                logging.debug("Can't connect to HDFS " + str(ex))
+                logging.exception("Can't connect to HDFS " + str(ex))
                 time.sleep(1)
 
         raise Exception("Can't wait HDFS to start")
@@ -1413,16 +1410,16 @@ def start(self, destroy_dirs=True):
                 os.makedirs(self.hdfs_logs_dir)
                 os.chmod(self.hdfs_logs_dir, stat.S_IRWXO)
                 subprocess_check_call(self.base_hdfs_cmd + common_opts)
-                hdfs_api = self.make_hdfs_api()
-                self.wait_hdfs_to_start(hdfs_api)
+                self.make_hdfs_api()
+                self.wait_hdfs_to_start()
 
             if self.with_kerberized_hdfs and self.base_kerberized_hdfs_cmd:
                 logging.debug('Setup kerberized HDFS')
                 os.makedirs(self.hdfs_kerberized_logs_dir)
                 os.chmod(self.hdfs_kerberized_logs_dir, stat.S_IRWXO)
                 run_and_check(self.base_kerberized_hdfs_cmd + common_opts)
-                hdfs_api = self.make_hdfs_api(kerberized=True)
-                self.wait_hdfs_to_start(hdfs_api)
+                self.make_hdfs_api(kerberized=True)
+                self.wait_hdfs_to_start()
 
             if self.with_mongo and self.base_mongo_cmd:
                 logging.debug('Setup Mongo')