Skip to content
Permalink
Browse files
CASSANDRA-14076 Fix dtest code style check
  • Loading branch information
jay-zhuang authored and ptnapoleon committed Dec 4, 2017
1 parent c1bcc18 commit 44fd9be91d38345bf666c2118693489f9a46de20
Showing 18 changed files with 71 additions and 67 deletions.
@@ -2,19 +2,19 @@ language: python
python:
- "2.7"
install:
- pip install pep8 flake8
- pip install pycodestyle flake8
script:
# we want pyflakes to check all files for unused imports only
# we use flake8 because it allows us to ignore other warnings
# exclude the thrift directories - they contain auto-generated code
- flake8 --ignore=E501,F811,F812,F822,F823,F831,F841,N8,C9 --exclude=thrift_bindings,cassandra-thrift .
- git remote add apache git://github.com/apache/cassandra-dtest.git
- git fetch apache # fetch master for the next diff
# feed changed lines with no context around them to pep8
# feed changed lines with no context around them to pycodestyle
# I know we don't enforce line length but if you introduce
# 200-char lines you are doing something terribly wrong.
# lint all files for everything but line length errors
- git diff apache/master...HEAD -U0 | pep8 --ignore=E501 --diff
- git diff apache/master...HEAD -U0 | pycodestyle --ignore=E501 --diff
# lint all files except json_test.py for line length errors
- git diff apache/master...HEAD -U0 | pep8 --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
- git diff apache/master...HEAD -U0 | pycodestyle --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
sudo: false
@@ -544,7 +544,7 @@ def run():
valid_fcn(v)
except Queue.Empty:
pass
except:
except Exception:
exceptions_queue.put(sys.exc_info())

start = 0
@@ -973,12 +973,12 @@ def test_13880(self):
session.execute(query)

stmt = SimpleStatement("INSERT INTO test.test (id) VALUES (0);",
consistency_level = ConsistencyLevel.ALL)
consistency_level=ConsistencyLevel.ALL)
session.execute(stmt)

# with node2 down and hints disabled, delete the partition on node1
node2.stop(wait_other_notice=True)
session.execute("DELETE FROM test.test WHERE id = 0;");
session.execute("DELETE FROM test.test WHERE id = 0;")
node2.start(wait_other_notice=True)

# with both nodes up, do a CL.ALL query with per partition limit of 1;
@@ -1034,7 +1034,7 @@ def test_13747(self):
# prior to CASSANDRA-13747 this would cause an assertion in short read protection code
node2.start(wait_other_notice=True)
stmt = SimpleStatement("SELECT DISTINCT token(id), id FROM test.test;",
consistency_level = ConsistencyLevel.ALL)
consistency_level=ConsistencyLevel.ALL)
result = list(session.execute(stmt))
assert_length_equal(result, 5)

@@ -1051,7 +1051,7 @@ def test_13595(self):

cluster.populate(2)
node1, node2 = cluster.nodelist()
remove_perf_disable_shared_mem(node1) # necessary for jmx
remove_perf_disable_shared_mem(node1) # necessary for jmx
cluster.start(wait_other_notice=True)

session = self.patient_cql_connection(node1)
@@ -1100,7 +1100,7 @@ def test_13595(self):
assert_all(session,
'SELECT id FROM test.test LIMIT 1;',
[[3]],
cl = ConsistencyLevel.ALL)
cl=ConsistencyLevel.ALL)

srp = make_mbean('metrics', type='Table', name='ShortReadProtectionRequests', keyspace='test', scope='test')
with JolokiaAgent(node1) as jmx:
@@ -1172,7 +1172,7 @@ def test_12872(self):
assert_all(session,
'SELECT ck FROM test.test WHERE pk = 0 LIMIT 2;',
[[0], [4]],
cl = ConsistencyLevel.ALL)
cl=ConsistencyLevel.ALL)

def short_read_test(self):
"""
@@ -56,7 +56,7 @@ def test_13691(self):
#

cluster.set_install_dir(version='2.1.17')
cluster.start();
cluster.start()
cluster.nodetool("upgradesstables")

#
@@ -380,7 +380,7 @@ def drop_counter_column_test(self):

assert_invalid(session, "ALTER TABLE counter_bug add c counter", "Cannot re-add previously dropped counter column c")

@since("2.0", max_version="3.X") # Compact Storage
@since("2.0", max_version="3.X") # Compact Storage
def compact_counter_cluster_test(self):
"""
@jira_ticket CASSANDRA-12219
@@ -583,7 +583,7 @@ def tearDown(self):
for runner in self.runners:
try:
runner.stop()
except:
except Exception:
pass

failed = did_fail()
@@ -1127,5 +1127,6 @@ def run_scenarios(scenarios, handler, deferred_exceptions=tuple()):
if errors:
raise MultiError(errors, tracebacks)


def supports_v5_protocol(cluster_version):
return cluster_version >= LooseVersion('4.0')
@@ -1,17 +1,17 @@
#!/bin/bash

# linter_check ensures your changes will pass on travis.
# Requires pep8 and flake8: pip install pep8 flake8
# Requires pycodestyle and flake8: pip install pycodestyle flake8

flake8 --ignore=E501,F811,F812,F822,F823,F831,F841,N8,C9 --exclude=thrift_bindings,cassandra-thrift .
flake8_result=$?

# lint all files for everything but line length errors
git diff apache/master...HEAD -U0 | pep8 --ignore=E501 --diff
git diff apache/master...HEAD -U0 | pycodestyle --ignore=E501 --diff
pep8_style_check=$?

# lint all files except json_test.py for line length errors
git diff apache/master...HEAD -U0 | pep8 --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
git diff apache/master...HEAD -U0 | pycodestyle --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
pep8_line_length=$?

echo -e "\nflake8 exited with ${flake8_result}."
@@ -6,6 +6,7 @@
from tools.decorators import since
from tools.jmxutils import JolokiaAgent, make_mbean, remove_perf_disable_shared_mem


class TestNodetool(Tester):

def test_decommission_after_drain_is_invalid(self):
@@ -75,7 +76,7 @@ def test_nodetool_timeout_commands(self):
'truncate', 'misc']
if cluster.version() < '4.0':
types.append('streamingsocket')

# read all of the timeouts, make sure we get a sane response
for timeout_type in types:
out, err, _ = node.nodetool('gettimeout {}'.format(timeout_type))
@@ -171,7 +172,7 @@ def test_reloadlocalschema(self):
cluster = self.cluster
cluster.populate(1)
node = cluster.nodelist()[0]
remove_perf_disable_shared_mem(node) # for jmx
remove_perf_disable_shared_mem(node) # for jmx
cluster.start()

session = self.patient_cql_connection(node)
@@ -1,15 +1,12 @@
from collections import namedtuple

from pprint import pprint
import os
import inspect
from nose.plugins.base import Plugin
from nose.case import Test
import logging
import unittest

log = logging.getLogger(__name__)


class DTestCollect(Plugin):
"""
Collect and output test names only, don't run any tests.
@@ -41,6 +38,7 @@ def prepareTestCase(self, test):
log.debug("Preparing test case %s", test)
if not isinstance(test, Test):
return

def run(result):
# We need to make these plugin calls because there won't be
# a result proxy, due to using a stripped-down test suite
@@ -59,7 +57,8 @@ def describeTest(self, test):
else:
tag = test.test._testMethodName + "-" + tag
retval = "%s:%s.%s" % (test.test.__module__, test.test.__class__.__name__, tag)
return retval;
return retval


class TestSuiteFactory:
"""
@@ -1,10 +1,8 @@
from collections import namedtuple

from nose import plugins
from pprint import pprint
import os
import inspect


class DTestTag(plugins.Plugin):
enabled = True # if this plugin is loaded at all, we're using it
name = 'dtest_tag'
@@ -17,12 +15,12 @@ def configure(self, options, conf):

def nice_classname(self, obj):
"""Returns a nice name for class object or class instance.
>>> nice_classname(Exception()) # doctest: +ELLIPSIS
'...Exception'
>>> nice_classname(Exception) # doctest: +ELLIPSIS
'...Exception'
"""
if inspect.isclass(obj):
cls_name = obj.__name__
@@ -40,10 +40,8 @@
"""
import codecs
import doctest
import os
import sys
import traceback
import re
import inspect
from StringIO import StringIO
@@ -59,28 +57,32 @@

TEST_ID = re.compile(r'^(.*?)(\(.*\))$')


def xml_safe(value):
"""Replaces invalid XML characters with '?'."""
return CONTROL_CHARACTERS.sub('?', value)


def escape_cdata(cdata):
"""Escape a string for an XML CDATA section."""
return xml_safe(cdata).replace(']]>', ']]>]]&gt;<![CDATA[')


def id_split(idval):
m = TEST_ID.match(idval)
retval = []
if m:
name, fargs = m.groups()
head, tail = name.rsplit(".", 1)
retval = [head, tail+fargs]
retval = [head, tail + fargs]
else:
retval = idval.rsplit(".", 1)
tag = os.getenv('TEST_TAG', '')
if tag != '':
retval[-1] = retval[-1] + "-" + tag
return retval


def nice_classname(obj):
"""Returns a nice name for class object or class instance.
@@ -104,6 +106,7 @@ def nice_classname(obj):
else:
return cls_name


def exc_message(exc_info):
"""Return the exception's message."""
exc = exc_info[1]
@@ -123,6 +126,7 @@ def exc_message(exc_info):
result = force_unicode(result, 'UTF-8')
return xml_safe(result)


class Tee(object):
def __init__(self, encoding, *args):
self._encoding = encoding
@@ -215,8 +219,8 @@ def report(self, stream):
self.encoding, 'replace')
self.stats['encoding'] = self.encoding
self.stats['testsuite_name'] = self.xunit_testsuite_name
self.stats['total'] = (self.stats['errors'] + self.stats['failures']
+ self.stats['passes'] + self.stats['skipped'])
self.stats['total'] = (self.stats['errors'] + self.stats['failures'] +
self.stats['passes'] + self.stats['skipped'])
self.error_report_file.write(
u'<?xml version="1.0" encoding="%(encoding)s"?>'
u'<testsuite name="%(testsuite_name)s" tests="%(total)d" '
@@ -265,16 +269,14 @@ def _getCapturedStdout(self):
if self._currentStdout:
value = self._currentStdout.getvalue()
if value:
return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata(
value)
return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata(value)
return ''

def _getCapturedStderr(self):
if self._currentStderr:
value = self._currentStderr.getvalue()
if value:
return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata(
value)
return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata(value)
return ''

def addError(self, test, err, capt=None):
@@ -156,13 +156,13 @@ def _deprecated_repair_jmx(self, method, arguments):
# wait for log to start
node1.watch_log_for("Starting repair command")
# get repair parameters from the log
l = node1.grep_log(("Starting repair command #1" + (" \([^\)]+\)" if cluster.version() >= LooseVersion("3.10") else "") +
", repairing keyspace ks with repair options \(parallelism: (?P<parallelism>\w+), primary range: (?P<pr>\w+), "
"incremental: (?P<incremental>\w+), job threads: (?P<jobs>\d+), ColumnFamilies: (?P<cfs>.+), dataCenters: (?P<dc>.+), "
"hosts: (?P<hosts>.+), # of ranges: (?P<ranges>\d+)(, pull repair: (?P<pullrepair>true|false))?\)"))
line = node1.grep_log(("Starting repair command #1" + (" \([^\)]+\)" if cluster.version() >= LooseVersion("3.10") else "") +
", repairing keyspace ks with repair options \(parallelism: (?P<parallelism>\w+), primary range: (?P<pr>\w+), "
"incremental: (?P<incremental>\w+), job threads: (?P<jobs>\d+), ColumnFamilies: (?P<cfs>.+), dataCenters: (?P<dc>.+), "
"hosts: (?P<hosts>.+), # of ranges: (?P<ranges>\d+)(, pull repair: (?P<pullrepair>true|false))?\)"))

assert_length_equal(l, 1)
line, m = l[0]
assert_length_equal(line, 1)
line, m = line[0]

if supports_pull_repair:
self.assertEqual(m.group("pullrepair"), "false", "Pull repair cannot be enabled through the deprecated API so the pull repair option should always be false.")
@@ -769,9 +769,9 @@ def bootstrap_test(self):

@since('4.0')
def force_test(self):
"""
forcing an incremental repair should incrementally repair any nodes
that are up, but should not promote the sstables to repaired
"""
forcing an incremental repair should incrementally repair any nodes
that are up, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500})
@@ -801,9 +801,9 @@ def force_test(self):

@since('4.0')
def hosts_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500})
@@ -827,9 +827,9 @@ def hosts_test(self):

@since('4.0')
def subrange_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
@@ -852,7 +852,7 @@ def subrange_test(self):
self.assertNoRepairedSSTables(node, 'ks')

# only repair the partition k=0
token = Murmur3Token.from_key(str(bytearray([0,0,0,0])))
token = Murmur3Token.from_key(str(bytearray([0, 0, 0, 0])))
# import ipdb; ipdb.set_trace()
# run with force flag
node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)])
@@ -217,7 +217,7 @@ def product_of_values(d):
"import nose\n" +
"from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n" +
"from plugins.dtestxunit import DTestXunit\n" +
"from plugins.dtesttag import DTestTag\n" +
"from plugins.dtesttag import DTestTag\n" +
"from plugins.dtestcollect import DTestCollect\n" +
"import sys\n" +
"print sys.getrecursionlimit()\n" +

0 comments on commit 44fd9be

Please sign in to comment.