Skip to content
Permalink
Browse files
Merge pull request #1347 from riptano/speed-up-cql-tests
Speed up lwt cql tests and also dtest.py refactorings
  • Loading branch information
ptnapoleon committed Sep 27, 2016
2 parents 41bffab + de24b94 commit dec6140402b74f5cca6b83b9902ae591fc989cc5
Showing 57 changed files with 461 additions and 446 deletions.
@@ -5,7 +5,7 @@
from cassandra import ConsistencyLevel, Timeout, Unavailable
from cassandra.query import SimpleStatement

from dtest import CASSANDRA_DIR, Tester, debug
from dtest import CASSANDRA_DIR, Tester, debug, create_ks
from tools.assertions import (assert_all, assert_invalid, assert_one,
assert_unavailable)
from tools.decorators import since
@@ -383,7 +383,7 @@ def prepare(self, nodes=1, compression=True, version=None, protocol_version=None

def create_schema(self, session, rf):
debug('Creating schema...')
self.create_ks(session, 'ks', rf)
create_ks(session, 'ks', rf)

session.execute("""
CREATE TABLE clicks (
@@ -10,7 +10,7 @@
from cassandra.concurrent import execute_concurrent_with_args
from ccmlib.node import NodeError

from dtest import DISABLE_VNODES, Tester, debug
from dtest import DISABLE_VNODES, Tester, debug, create_ks, create_cf
from tools.assertions import (assert_almost_equal, assert_bootstrap_state, assert_not_running,
assert_one, assert_stderr_clean)
from tools.data import query_c1c2
@@ -62,8 +62,8 @@ def default_bootstrap(cluster, token):
cluster.start(wait_other_notice=True)

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
create_ks(session, 'ks', 1)
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

# record the size before inserting any of our own data
empty_size = node1.data_size()
@@ -15,7 +15,7 @@
from ccmlib.node import Node
from nose.tools import assert_equal, assert_less_equal

from dtest import Tester, debug
from dtest import Tester, debug, create_ks
from tools.data import rows_to_list
from tools.decorators import known_failure, since
from tools.files import size_of_files_in_dir
@@ -255,7 +255,7 @@ def prepare(self, ks_name,
self.cluster.start(wait_for_binary_proto=True)
node = self.cluster.nodelist()[0]
session = self.patient_cql_connection(node)
self.create_ks(session, ks_name, rf=1)
create_ks(session, ks_name, rf=1)

if table_name is not None:
self.assertIsNotNone(cdc_enabled_table, 'if creating a table in prepare, must specify whether or not CDC is enabled on it')
@@ -462,7 +462,7 @@ def _init_new_loading_node(self, ks_name, create_stmt):
loading_node.start(wait_for_binary_proto=True)
debug('recreating ks and table')
loading_session = self.patient_exclusive_cql_connection(loading_node)
self.create_ks(loading_session, ks_name, rf=1)
create_ks(loading_session, ks_name, rf=1)
debug('creating new table')
loading_session.execute(create_stmt)
debug('stopping new node')
@@ -1,6 +1,6 @@
import os

from dtest import Tester
from dtest import Tester, create_ks, create_cf


class TestCFID(Tester):
@@ -13,10 +13,10 @@ def cfid_test(self):
[node1] = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)

for x in range(0, 5):
self.create_cf(session, 'cf', gc_grace=0, key_type='int', columns={'c1': 'int'})
create_cf(session, 'cf', gc_grace=0, key_type='int', columns={'c1': 'int'})
session.execute('insert into cf (key, c1) values (1,1)')
session.execute('insert into cf (key, c1) values (2,1)')
node1.flush()
@@ -12,7 +12,7 @@
from ccmlib.node import Node, TimeoutError
from parse import parse

from dtest import Tester, debug
from dtest import Tester, debug, create_ks
from tools.assertions import assert_almost_equal, assert_none, assert_one
from tools.data import rows_to_list
from tools.decorators import known_failure, since
@@ -51,7 +51,7 @@ def prepare(self, configuration=None, create_test_keyspace=True, **kwargs):
self.session1 = self.patient_cql_connection(self.node1)
if create_test_keyspace:
self.session1.execute("DROP KEYSPACE IF EXISTS ks;")
self.create_ks(self.session1, 'ks', 1)
create_ks(self.session1, 'ks', 1)
self.session1.execute("DROP TABLE IF EXISTS test;")
query = """
CREATE TABLE test (
@@ -177,7 +177,7 @@ def test_mv_lock_contention_during_replay(self):
session = self.patient_cql_connection(node1)

debug("Creating schema")
self.create_ks(session, 'Test', 1)
create_ks(session, 'Test', 1)
session.execute("""
CREATE TABLE mytable (
a int,
@@ -239,7 +239,7 @@ def test_commitlog_replay_on_startup(self):

debug("Insert data")
session = self.patient_cql_connection(node1)
self.create_ks(session, 'Test', 1)
create_ks(session, 'Test', 1)
session.execute("""
CREATE TABLE users (
user_name varchar PRIMARY KEY,
@@ -472,7 +472,7 @@ def test_bad_crc(self):
self.cluster.start()

cursor = self.patient_cql_connection(self.cluster.nodelist()[0])
self.create_ks(cursor, 'ks', 1)
create_ks(cursor, 'ks', 1)
cursor.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")

for i in range(10):
@@ -552,7 +552,7 @@ def test_compression_error(self):
self.cluster.start()

cursor = self.patient_cql_connection(self.cluster.nodelist()[0])
self.create_ks(cursor, 'ks1', 1)
create_ks(cursor, 'ks1', 1)
cursor.execute("CREATE TABLE ks1.tbl (k INT PRIMARY KEY, v INT)")

for i in range(10):
@@ -7,7 +7,7 @@

import parse

from dtest import Tester, debug
from dtest import Tester, debug, create_ks
from tools.assertions import assert_length_equal, assert_none, assert_one
from tools.decorators import known_failure, since
from tools.misc import ImmutableMapping
@@ -34,7 +34,7 @@ def compaction_delete_test(self):
[node1] = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)

session.execute("create table ks.cf (key int PRIMARY KEY, val int) with compaction = {'class':'" + self.strategy + "'} and gc_grace_seconds = 30;")

@@ -166,7 +166,7 @@ def sstable_deletion_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node1] = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute("create table cf (key int PRIMARY KEY, val int) with gc_grace_seconds = 0 and compaction= {'class':'" + self.strategy + "'}")

for x in range(0, 100):
@@ -205,7 +205,7 @@ def dtcs_deletion_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node1] = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
# max sstable age is 0.5 minute:
session.execute("""create table cf (key int PRIMARY KEY, val int) with gc_grace_seconds = 0
and compaction= {'class':'DateTieredCompactionStrategy', 'max_sstable_age_days':0.00035, 'min_threshold':2}""")
@@ -301,7 +301,7 @@ def compaction_strategy_switching_test(self):

for strat in strategies:
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)

session.execute("create table ks.cf (key int PRIMARY KEY, val int) with gc_grace_seconds = 0 and compaction= {'class':'" + self.strategy + "'};")

@@ -337,7 +337,7 @@ def large_compaction_warning_test(self):
[node] = cluster.nodelist()

session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)

mark = node.mark_log()
strlen = (1024 * 1024) / 100
@@ -368,7 +368,7 @@ def disable_autocompaction_nodetool_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node] = cluster.nodelist()
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute('CREATE TABLE to_disable (id int PRIMARY KEY, d TEXT) WITH compaction = {{\'class\':\'{0}\'}}'.format(self.strategy))
node.nodetool('disableautocompaction ks to_disable')
for i in range(1000):
@@ -393,7 +393,7 @@ def disable_autocompaction_schema_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node] = cluster.nodelist()
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute('CREATE TABLE to_disable (id int PRIMARY KEY, d TEXT) WITH compaction = {{\'class\':\'{0}\', \'enabled\':\'false\'}}'.format(self.strategy))
for i in range(1000):
session.execute('insert into to_disable (id, d) values ({0}, \'{1}\')'.format(i, 'hello' * 100))
@@ -426,7 +426,7 @@ def disable_autocompaction_alter_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node] = cluster.nodelist()
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute('CREATE TABLE to_disable (id int PRIMARY KEY, d TEXT) WITH compaction = {{\'class\':\'{0}\'}}'.format(self.strategy))
session.execute('ALTER TABLE to_disable WITH compaction = {{\'class\':\'{0}\', \'enabled\':\'false\'}}'.format(self.strategy))
for i in range(1000):
@@ -454,7 +454,7 @@ def disable_autocompaction_alter_and_nodetool_test(self):
cluster.populate(1).start(wait_for_binary_proto=True)
[node] = cluster.nodelist()
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute('CREATE TABLE to_disable (id int PRIMARY KEY, d TEXT) WITH compaction = {{\'class\':\'{0}\'}}'.format(self.strategy))
node.nodetool('disableautocompaction ks to_disable')
for i in range(1000):
@@ -1,5 +1,6 @@
import os

from dtest import create_ks
from scrub_test import TestHelper
from tools.assertions import assert_crc_check_chance_equal
from tools.decorators import since
@@ -28,7 +29,7 @@ def disable_compression_cql_test(self):
[node] = cluster.nodelist()

session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute("create table disabled_compression_table (id uuid PRIMARY KEY ) WITH compression = {'enabled': false};")
session.cluster.refresh_schema_metadata()
meta = session.cluster.metadata.keyspaces['ks'].tables['disabled_compression_table']
@@ -58,7 +59,7 @@ def compression_cql_options_test(self):
[node] = cluster.nodelist()

session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute("""
create table compression_opts_table
(id uuid PRIMARY KEY )
@@ -119,7 +120,7 @@ def compression_cql_disabled_with_alter_test(self):
[node] = cluster.nodelist()

session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute("""
create table start_enabled_compression_table
(id uuid PRIMARY KEY )
@@ -150,7 +151,7 @@ def compression_cql_enabled_with_alter_test(self):
[node] = cluster.nodelist()

session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)
session.execute("create table start_disabled_compression_table (id uuid PRIMARY KEY ) WITH compression = {'enabled': false};")
meta = session.cluster.metadata.keyspaces['ks'].tables['start_disabled_compression_table']
self.assertEqual('false', meta.options['compression']['enabled'])
@@ -10,7 +10,7 @@
from cassandra.concurrent import execute_concurrent
from ccmlib.node import Node

from dtest import Tester, debug
from dtest import Tester, debug, create_ks
from tools.decorators import known_failure, since
from tools.misc import ImmutableMapping

@@ -36,7 +36,7 @@ def prepare_for_changes(self, session, namespace='ns1'):
"""
debug("prepare_for_changes() " + str(namespace))
# create a keyspace that will be used
self.create_ks(session, "ks_%s" % namespace, 2)
create_ks(session, "ks_%s" % namespace, 2)
session.execute('USE ks_%s' % namespace)

# create a column family with an index and a row of data
@@ -66,7 +66,7 @@ def prepare_for_changes(self, session, namespace='ns1'):
session.execute(query)

# make a keyspace that can be deleted
self.create_ks(session, "ks2_%s" % namespace, 2)
create_ks(session, "ks2_%s" % namespace, 2)

def make_schema_changes(self, session, namespace='ns1'):
"""
@@ -89,7 +89,7 @@ def make_schema_changes(self, session, namespace='ns1'):
wait(2)

# create keyspace
self.create_ks(session, "ks3_%s" % namespace, 2)
create_ks(session, "ks3_%s" % namespace, 2)
session.execute('USE ks_%s' % namespace)

wait(2)
@@ -3,7 +3,7 @@
import parse
from cassandra.concurrent import execute_concurrent_with_args

from dtest import Tester, debug
from dtest import Tester, debug, create_ks
from tools.jmxutils import (JolokiaAgent, make_mbean,
remove_perf_disable_shared_mem)

@@ -17,7 +17,7 @@ def compression_chunk_length_test(self):
cluster.populate(1).start()
node = cluster.nodelist()[0]
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
create_ks(session, 'ks', 1)

create_table_query = "CREATE TABLE test_table (row varchar, name varchar, value int, PRIMARY KEY (row, name));"
alter_chunk_len_query = "ALTER TABLE test_table WITH compression = {{'sstable_compression' : 'SnappyCompressor', 'chunk_length_kb' : {chunk_length}}};"
@@ -11,7 +11,7 @@
from nose.tools import assert_greater_equal

from tools.assertions import assert_length_equal, assert_none, assert_unavailable
from dtest import DISABLE_VNODES, MultiError, Tester, debug
from dtest import DISABLE_VNODES, MultiError, Tester, debug, create_ks, create_cf
from tools.data import (create_c1c2_table, insert_c1c2, insert_columns,
query_c1c2, rows_to_list)
from tools.decorators import known_failure, since
@@ -141,7 +141,7 @@ def _start_cluster(self, save_sessions=False, requires_local_reads=False):
self.ksname = 'mytestks'
session = self.patient_exclusive_cql_connection(cluster.nodelist()[0])

self.create_ks(session, self.ksname, rf)
create_ks(session, self.ksname, rf)
self.create_tables(session, requires_local_reads)

if save_sessions:
@@ -799,8 +799,8 @@ def short_read_test(self):
node1, node2, node3 = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', read_repair=0.0)
create_ks(session, 'ks', 3)
create_cf(session, 'cf', read_repair=0.0)

normal_key = 'normal'
reversed_key = 'reversed'
@@ -859,8 +859,8 @@ def short_read_delete_test(self):
node1, node2 = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', read_repair=0.0)
create_ks(session, 'ks', 3)
create_cf(session, 'cf', read_repair=0.0)
# insert 2 columns in one row
insert_columns(self, session, 0, 2)

@@ -895,7 +895,7 @@ def short_read_quorum_delete_test(self):
node1, node2, node3 = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
create_ks(session, 'ks', 3)

session.execute("CREATE TABLE t (id int, v int, PRIMARY KEY(id, v)) WITH read_repair_chance = 0.0")
# we write 1 and 2 in a partition: all nodes get it.
@@ -933,7 +933,7 @@ def readrepair_test(self):
node1, node2 = cluster.nodelist()

session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 2)
create_ks(session, 'ks', 2)
create_c1c2_table(self, session, read_repair=1.0)

node2.stop(wait_other_notice=True)
@@ -968,7 +968,7 @@ def quorum_available_during_failure_test(self):

debug("Set to talk to node 2")
session = self.patient_cql_connection(node2)
self.create_ks(session, 'ks', RF)
create_ks(session, 'ks', RF)
create_c1c2_table(self, session)

debug("Generating some data")

0 comments on commit dec6140

Please sign in to comment.