Merge PR #32129 into master

* refs/pull/32129/head:
	qa: Enable flake8 tox and fix failures

CMakeLists.txt

@@ -628,6 +628,8 @@ add_custom_target(check
 
 add_subdirectory(src)
 
+add_subdirectory(qa)
+
 add_subdirectory(doc)
 if(WITH_MANPAGE)
   add_subdirectory(man)

qa/CMakeLists.txt

@@ -0,0 +1,9 @@
+set(CEPH_BUILD_VIRTUALENV $ENV{TMPDIR})
+if(NOT CEPH_BUILD_VIRTUALENV)
+  set(CEPH_BUILD_VIRTUALENV ${CMAKE_BINARY_DIR})
+endif()
+
+if(WITH_TESTS)
+  include(AddCephTest)
+  add_tox_test(qa flake8)
+endif()

qa/standalone/special/ceph_objectstore_tool.py

@@ -46,7 +46,7 @@ def check_output(*popenargs, **kwargs):
     def decode(s):
         return s.decode('utf-8')
 
-    def check_output(*args, **kwargs):
+    def check_output(*args, **kwargs): # noqa
         return decode(subprocess.check_output(*args, **kwargs))
 else:
     def decode(s):
@@ -337,7 +337,7 @@ def check_entry_transactions(entry, enum):
 
 
 def check_transaction_ops(ops, enum, tnum):
-    if len(ops) is 0:
+    if len(ops) == 0:
         logging.warning("No ops found in entry {e} trans {t}".format(e=enum, t=tnum))
     errors = 0
     for onum in range(len(ops)):
@@ -376,7 +376,7 @@ def test_dump_journal(CFSD_PREFIX, osds):
         os.unlink(TMPFILE)
 
         journal_errors = check_journal(jsondict)
-        if journal_errors is not 0:
+        if journal_errors != 0:
             logging.error(jsondict)
         ERRORS += journal_errors
 
@@ -520,7 +520,7 @@ def get_osd_weights(CFSD_PREFIX, osd_ids, osd_path):
     for line in output.strip().split('\n'):
         print(line)
         linev = re.split('\s+', line)
-        if linev[0] is '':
+        if linev[0] == '':
             linev.pop(0)
         print('linev %s' % linev)
         weights.append(float(linev[2]))

qa/tasks/barbican.py

@@ -4,14 +4,12 @@
 import argparse
 import contextlib
 import logging
-import string
 import httplib
 from urlparse import urlparse
 import json
 
 from teuthology import misc as teuthology
 from teuthology import contextutil
-from teuthology import safepath
 from teuthology.orchestra import run
 from teuthology.exceptions import ConfigError
 
@@ -201,7 +199,6 @@ def run_barbican(ctx, config):
 
         # start the public endpoint
         client_public_with_id = 'barbican.public' + '.' + client_id
-        client_public_with_cluster = cluster_name + '.' + client_public_with_id
 
         run_cmd = ['cd', get_barbican_dir(ctx), run.Raw('&&'),
                    '.', '.barbicanenv/bin/activate', run.Raw('&&'),
@@ -248,8 +245,6 @@ def create_secrets(ctx, config):
 
     keystone_role = cconfig.get('use-keystone-role', None)
     keystone_host, keystone_port = ctx.keystone.public_endpoints[keystone_role]
-    keystone_url = 'http://{host}:{port}/v2.0'.format(host=keystone_host,
-                                                      port=keystone_port)
     barbican_host, barbican_port = ctx.barbican.endpoints[cclient]
     barbican_url = 'http://{host}:{port}'.format(host=barbican_host,
                                                  port=barbican_port)
@@ -482,7 +477,6 @@ def task(ctx, config):
         config = all_clients
     if isinstance(config, list):
         config = dict.fromkeys(config)
-    clients = config.keys()
 
     overrides = ctx.config.get('overrides', {})
     # merge each client section, not the top level.

qa/tasks/cbt.py

@@ -3,7 +3,6 @@
 import yaml
 
 from teuthology import misc
-from teuthology.config import config as teuth_config
 from teuthology.orchestra import run
 from teuthology.task import Task
 

qa/tasks/ceph.py

@@ -85,18 +85,18 @@ def ceph_crash(ctx, config):
             path = os.path.join(ctx.archive, 'remote')
             try:
                 os.makedirs(path)
-            except OSError as e:
+            except OSError:
                 pass
             for remote in ctx.cluster.remotes.keys():
                 sub = os.path.join(path, remote.shortname)
                 try:
                     os.makedirs(sub)
-                except OSError as e:
+                except OSError:
                     pass
                 try:
                     teuthology.pull_directory(remote, '/var/lib/ceph/crash',
                                               os.path.join(sub, 'crash'))
-                except ReadError as e:
+                except ReadError:
                     pass
 
 
@@ -270,13 +270,13 @@ def write_rotate_conf(ctx, daemons):
             path = os.path.join(ctx.archive, 'remote')
             try:
                 os.makedirs(path)
-            except OSError as e:
+            except OSError:
                 pass
             for remote in ctx.cluster.remotes.keys():
                 sub = os.path.join(path, remote.shortname)
                 try:
                     os.makedirs(sub)
-                except OSError as e:
+                except OSError:
                     pass
                 teuthology.pull_directory(remote, '/var/log/ceph',
                                           os.path.join(sub, 'log'))
@@ -397,8 +397,6 @@ def create_rbd_pool(ctx, config):
 @contextlib.contextmanager
 def cephfs_setup(ctx, config):
     cluster_name = config['cluster']
-    testdir = teuthology.get_testdir(ctx)
-    coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
 
     first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
     (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys()

qa/tasks/ceph_fuse.py

@@ -7,7 +7,6 @@
 
 from teuthology import misc as teuthology
 from cephfs.fuse_mount import FuseMount
-from tasks.cephfs.filesystem import Filesystem
 
 log = logging.getLogger(__name__)
 

qa/tasks/ceph_manager.py

@@ -679,7 +679,7 @@ def shrink_pool(self):
         Decrease the size of the pool
         """
         pool = self.ceph_manager.get_pool()
-        orig_pg_num = self.ceph_manager.get_pool_pg_num(pool)
+        _ = self.ceph_manager.get_pool_pg_num(pool)
         self.log("Shrinking pool %s" % (pool,))
         if self.ceph_manager.contract_pool(
                 pool,
@@ -906,7 +906,7 @@ def choose_action(self):
         Random action selector.
         """
         chance_down = self.config.get('chance_down', 0.4)
-        chance_test_min_size = self.config.get('chance_test_min_size', 0)
+        _ = self.config.get('chance_test_min_size', 0)
         chance_test_backfill_full = \
             self.config.get('chance_test_backfill_full', 0)
         if isinstance(chance_down, int):
@@ -1653,7 +1653,7 @@ def wait_run_admin_socket(self, service_type,
         while True:
             proc = self.admin_socket(service_type, service_id,
                                      args, check_status=False, stdout=stdout)
-            if proc.exitstatus is 0:
+            if proc.exitstatus == 0:
                 return proc
             else:
                 tries += 1

qa/tasks/cephadm.py

@@ -6,27 +6,17 @@
 import argparse
 import configobj
 import contextlib
-import errno
 import logging
 import os
 import json
-import time
-import gevent
-import re
-import socket
 import uuid
 
-from paramiko import SSHException
-from ceph_manager import CephManager, write_conf
+from ceph_manager import CephManager
 from tarfile import ReadError
-from tasks.cephfs.filesystem import Filesystem
 from teuthology import misc as teuthology
 from teuthology import contextutil
-from teuthology import exceptions
 from teuthology.orchestra import run
-import ceph_client as cclient
 from teuthology.orchestra.daemon import DaemonGroup
-from tasks.daemonwatchdog import DaemonWatchdog
 from teuthology.config import config as teuth_config
 
 # these items we use from ceph.py should probably eventually move elsewhere
@@ -93,7 +83,6 @@ def normalize_hostnames(ctx):
 @contextlib.contextmanager
 def download_cephadm(ctx, config, ref):
     cluster_name = config['cluster']
-    testdir = teuthology.get_testdir(ctx)
 
     if config.get('cephadm_mode') != 'cephadm-package':
         ref = config.get('cephadm_branch', ref)
@@ -179,13 +168,13 @@ def ceph_log(ctx, config):
             path = os.path.join(ctx.archive, 'remote')
             try:
                 os.makedirs(path)
-            except OSError as e:
+            except OSError:
                 pass
             for remote in ctx.cluster.remotes.keys():
                 sub = os.path.join(path, remote.name)
                 try:
                     os.makedirs(sub)
-                except OSError as e:
+                except OSError:
                     pass
                 teuthology.pull_directory(remote, '/var/log/ceph/' + fsid,
                                           os.path.join(sub, 'log'))
@@ -207,19 +196,19 @@ def ceph_crash(ctx, config):
             path = os.path.join(ctx.archive, 'remote')
             try:
                 os.makedirs(path)
-            except OSError as e:
+            except OSError:
                 pass
             for remote in ctx.cluster.remotes.keys():
                 sub = os.path.join(path, remote.name)
                 try:
                     os.makedirs(sub)
-                except OSError as e:
+                except OSError:
                     pass
                 try:
                     teuthology.pull_directory(remote,
                                               '/var/lib/ceph/%s/crash' % fsid,
                                               os.path.join(sub, 'crash'))
-                except ReadError as e:
+                except ReadError:
                     pass
 
 @contextlib.contextmanager
@@ -387,7 +376,6 @@ def ceph_mons(ctx, config):
     """
     cluster_name = config['cluster']
     fsid = ctx.ceph[cluster_name].fsid
-    testdir = teuthology.get_testdir(ctx)
     num_mons = 1
 
     try:
@@ -457,7 +445,6 @@ def ceph_mgrs(ctx, config):
     """
     cluster_name = config['cluster']
     fsid = ctx.ceph[cluster_name].fsid
-    testdir = teuthology.get_testdir(ctx)
 
     try:
         nodes = []
@@ -549,7 +536,6 @@ def ceph_mdss(ctx, config):
     """
     cluster_name = config['cluster']
     fsid = ctx.ceph[cluster_name].fsid
-    testdir = teuthology.get_testdir(ctx)
 
     nodes = []
     daemons = {}
@@ -628,7 +614,6 @@ def shell(ctx, config):
     """
     Execute (shell) commands
     """
-    testdir = teuthology.get_testdir(ctx)
     cluster_name = config.get('cluster', 'ceph')
 
     if 'all' in config and len(config) == 1:
@@ -724,7 +709,7 @@ def restart(ctx, config):
             healthy(ctx=ctx, config=dict(cluster=cluster))
     if config.get('wait-for-osds-up', False):
         for cluster in clusters:
-            wait_for_osds_up(ctx=ctx, config=dict(cluster=cluster))
+            ctx.managers[cluster].wait_for_all_osds_up()
     yield
 
 @contextlib.contextmanager

qa/tasks/cephfs/filesystem.py

@@ -1191,7 +1191,7 @@ def data_objects_absent(self, ino, size):
     def dirfrag_exists(self, ino, frag):
         try:
             self.rados(["stat", "{0:x}.{1:08x}".format(ino, frag)])
-        except CommandFailedError as e:
+        except CommandFailedError:
             return False
         else:
             return True

qa/tasks/cephfs/fuse_mount.py

@@ -1,4 +1,3 @@
-
 from StringIO import StringIO
 import json
 import time
@@ -10,7 +9,6 @@
 from teuthology.orchestra import run
 from teuthology.orchestra.run import CommandFailedError
 from .mount import CephFSMount
-from tasks.cephfs.filesystem import Filesystem
 
 log = logging.getLogger(__name__)
 
@@ -166,7 +164,7 @@ def gather_mount_info(self):
         try:
             self.inst = status['inst_str']
             self.addr = status['addr_str']
-        except KeyError as e:
+        except KeyError:
             sessions = self.fs.rank_asok(['session', 'ls'])
             for s in sessions:
                 if s['id'] == self.id:

qa/tasks/cephfs/test_cephfs_shell.py

@@ -12,8 +12,6 @@
 from time import sleep
 from StringIO import StringIO
 from tasks.cephfs.cephfs_test_case import CephFSTestCase
-from tasks.cephfs.fuse_mount import FuseMount
-from teuthology.exceptions import CommandFailedError
 from teuthology.misc import sudo_write_file
 
 log = logging.getLogger(__name__)
@@ -608,8 +606,8 @@ def test_du_with_path_in_args(self):
             path_prefix='')
 
         args = ['du', '/']
-        for path in path_to_files:
-            args.append(path)
+        for p in path_to_files:
+            args.append(p)
         du_output = self.get_cephfs_shell_cmd_output(args)
 
         for expected_output in expected_patterns_in_output:

qa/tasks/cephfs/test_client_recovery.py

@@ -591,7 +591,7 @@ def test_dont_mark_unresponsive_client_stale(self):
         SESSION_AUTOCLOSE = 50
         time_at_beg = time.time()
         mount_a_gid = self.mount_a.get_global_id()
-        mount_a_pid = self.mount_a.client_pid
+        _ = self.mount_a.client_pid
         self.fs.set_var('session_timeout', SESSION_TIMEOUT)
         self.fs.set_var('session_autoclose', SESSION_AUTOCLOSE)
         self.assert_session_count(2, self.fs.mds_asok(['session', 'ls']))

qa/tasks/cephfs/test_exports.py

@@ -150,7 +150,6 @@ def test_session_race(self):
         status = self.fs.wait_for_daemons()
 
         rank1 = self.fs.get_rank(rank=1, status=status)
-        name1 = 'mds.'+rank1['name']
 
         # Create a directory that is pre-exported to rank 1
         self.mount_a.run_shell(["mkdir", "-p", "a/aa"])