Merge branch 'cassandra-1.2.0' into cassandra-1.2

CHANGES.txt

@@ -17,6 +17,7 @@
 
 1.2.0
  * Disallow counters in collections (CASSANDRA-5082)
+ * cqlsh: add unit tests (CASSANDRA-3920)
 
 
 1.2.0-rc2

bin/cqlsh

@@ -53,10 +53,16 @@ import platform
 import warnings
 import csv
 
+
+readline = None
 try:
-    import readline
+    # check if tty first, cause readline doesn't check, and only cares
+    # about $TERM. we don't want the funky escape code stuff to be
+    # output if not a tty.
+    if sys.stdin.isatty():
+        import readline
 except ImportError:
-    readline = None
+    pass
 
 CQL_LIB_PREFIX = 'cql-internal-only-'
 THRIFT_LIB_PREFIX = 'thrift-python-internal-only-'
@@ -166,7 +172,7 @@ else:
 
 debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES')
 
-SYSTEM_KEYSPACES = ('system', 'system_traces')
+SYSTEM_KEYSPACES = ('system', 'system_traces', 'system_auth')
 
 # we want the cql parser to understand our cqlsh-specific commands too
 my_commands_ending_with_newline = (
@@ -368,24 +374,32 @@ class VersionNotSupported(Exception):
     pass
 
 class DecodeError(Exception):
+    verb = 'decode'
+
     def __init__(self, thebytes, err, expectedtype, colname=None):
         self.thebytes = thebytes
         self.err = err
+        if isinstance(expectedtype, type) and issubclass(expectedtype, CassandraType):
+            expectedtype = expectedtype.cql_parameterized_type()
         self.expectedtype = expectedtype
         self.colname = colname
 
     def __str__(self):
         return str(self.thebytes)
 
     def message(self):
-        what = 'column name %r' % (self.thebytes,)
+        what = 'value %r' % (self.thebytes,)
         if self.colname is not None:
             what = 'value %r (for column %r)' % (self.thebytes, self.colname)
-        return 'Failed to decode %s as %s: %s' % (what, self.expectedtype, self.err)
+        return 'Failed to %s %s as %s: %s' \
+               % (self.verb, what, self.expectedtype, self.err)
 
     def __repr__(self):
         return '<%s %s>' % (self.__class__.__name__, self.message())
 
+class FormatError(DecodeError):
+    verb = 'format'
+
 def full_cql_version(ver):
     while ver.count('.') < 2:
         ver += '.0'
@@ -397,9 +411,9 @@ def format_value(val, typeclass, output_encoding, addcolor=False, time_format=No
                  float_precision=None, colormap=None, nullval=None):
     if isinstance(val, DecodeError):
         if addcolor:
-            return colorme(val.thebytes, colormap, 'hex')
+            return colorme(repr(val.thebytes), colormap, 'error')
         else:
-            return FormattedValue(val.thebytes)
+            return FormattedValue(repr(val.thebytes))
     if not issubclass(typeclass, CassandraType):
         typeclass = lookup_casstype(typeclass)
     return format_by_type(typeclass, val, output_encoding, colormap=colormap,
@@ -449,7 +463,7 @@ class Shell(cmd.Cmd):
     def __init__(self, hostname, port, transport_factory, color=False,
                  username=None, password=None, encoding=None, stdin=None, tty=True,
                  completekey=DEFAULT_COMPLETEKEY, use_conn=None,
-                 cqlver=None, keyspace=None, tracing_enabled=False,
+                 cqlver=DEFAULT_CQLVER, keyspace=None, tracing_enabled=False,
                  display_time_format=DEFAULT_TIME_FORMAT,
                  display_float_precision=DEFAULT_FLOAT_PRECISION):
         cmd.Cmd.__init__(self, completekey=completekey)
@@ -530,9 +544,14 @@ class Shell(cmd.Cmd):
     def myformat_value(self, val, casstype, **kwargs):
         if isinstance(val, DecodeError):
             self.decoding_errors.append(val)
-        return format_value(val, casstype, self.output_codec.name,
-                            addcolor=self.color, time_format=self.display_time_format,
-                            float_precision=self.display_float_precision, **kwargs)
+        try:
+            return format_value(val, casstype, self.output_codec.name,
+                                addcolor=self.color, time_format=self.display_time_format,
+                                float_precision=self.display_float_precision, **kwargs)
+        except Exception, e:
+            err = FormatError(val, e, casstype)
+            self.decoding_errors.append(err)
+            return format_value(err, None, self.output_codec.name, addcolor=self.color)
 
     def myformat_colname(self, name, nametype):
         return self.myformat_value(name, nametype, colormap=COLUMN_NAME_COLORS)
@@ -644,7 +663,7 @@ class Shell(cmd.Cmd):
         raise ColumnFamilyNotFound("Unconfigured column family %r" % (cfname,))
 
     def get_columnfamily_names(self, ksname=None):
-        if self.cqlver_atleast(3):
+        if self.cqlver_atleast(3) and not self.is_cql3_beta():
             return self.get_columnfamily_names_cql3(ksname=ksname)
         return [c.name for c in self.get_columnfamilies(ksname)]
 
@@ -670,7 +689,7 @@ class Shell(cmd.Cmd):
     def get_column_names(self, ksname, cfname):
         if ksname is None:
             ksname = self.current_keyspace
-        if ksname not in SYSTEM_KEYSPACES and self.cqlver_atleast(3):
+        if self.cqlver_atleast(3) and not (self.is_cql3_beta() and ksname in SYSTEM_KEYSPACES):
             return self.get_column_names_from_layout(ksname, cfname)
         else:
             return self.get_column_names_from_cfdef(ksname, cfname)
@@ -898,7 +917,7 @@ class Shell(cmd.Cmd):
             print
         statement = self.statement.getvalue()
         if statement.strip():
-            if not self.onecmd(statement + ';'):
+            if not self.onecmd(statement):
                 self.printerr('Incomplete statement at end of file')
         self.do_exit()
 
@@ -960,7 +979,7 @@ class Shell(cmd.Cmd):
             print_trace_session(self, self.cursor, session_id)
             return result
         else:
-            return self.perform_statement_untraced(statement, decoder=None)
+            return self.perform_statement_untraced(statement, decoder=decoder)
 
     def perform_statement_untraced(self, statement, decoder=None):
         if not statement:
@@ -1048,6 +1067,7 @@ class Shell(cmd.Cmd):
         last_description = None
         for row in cursor:
             if last_description is not None and cursor.description != last_description:
+                cursor._reset()
                 return False
             last_description = cursor.description
         cursor._reset()
@@ -1085,13 +1105,13 @@ class Shell(cmd.Cmd):
                 widths[num] = max(widths[num], col.displaywidth)
 
         # print header
-        header = ' | '.join(hdr.color_ljust(w) for (hdr, w) in zip(formatted_names, widths))
+        header = ' | '.join(hdr.ljust(w, color=self.color) for (hdr, w) in zip(formatted_names, widths))
         self.writeresult(' ' + header.rstrip())
         self.writeresult('-%s-' % '-+-'.join('-' * w for w in widths))
 
         # print row data
         for row in formatted_values:
-            line = ' | '.join(col.color_rjust(w) for (col, w) in zip(row, widths))
+            line = ' | '.join(col.rjust(w, color=self.color) for (col, w) in zip(row, widths))
             self.writeresult(' ' + line)
 
     def print_dynamic_result(self, cursor):
@@ -1197,13 +1217,15 @@ class Shell(cmd.Cmd):
 
         # no metainfo available from system.schema_* for system CFs, so we have
         # to use cfdef-based description for those.
-
-        if self.cqlver_atleast(3) and not self.is_cql3_beta():
+        if self.cqlver_atleast(3) and not (self.is_cql3_beta() and ksname in SYSTEM_KEYSPACES):
             try:
                 layout = self.get_columnfamily_layout(ksname, cfname)
             except CQL_ERRORS:
                 # most likely a 1.1 beta where cql3 is supported, but not system.schema_*
-                pass
+                if self.debug:
+                    print 'warning: failed to use system.schema_* tables to describe cf'
+                    import traceback
+                    traceback.print_exc()
             else:
                 return self.print_recreate_columnfamily_from_layout(layout, out)
 
@@ -1293,14 +1315,22 @@ class Shell(cmd.Cmd):
         # work out how to determine that from a layout.
 
         cf_opts = []
+        compaction_strategy = trim_if_present(getattr(layout, 'compaction_strategy_class'),
+                                              'org.apache.cassandra.db.compaction.')
         for cql3option, layoutoption in cqlruleset.columnfamily_layout_options:
             if layoutoption is None:
                 layoutoption = cql3option
             optval = getattr(layout, layoutoption, None)
             if optval is None:
-                continue
+                if layoutoption == 'bloom_filter_fp_chance':
+                    if compaction_strategy == 'LeveledCompactionStrategy':
+                        optval = 0.1
+                    else:
+                        optval = 0.01
+                else:
+                    continue
             elif layoutoption == 'compaction_strategy_class':
-                optval = trim_if_present(optval, 'org.apache.cassandra.db.compaction.')
+                optval = compaction_strategy
             cf_opts.append((cql3option, self.cql_protect_value(optval)))
         for cql3option, layoutoption, _ in cqlruleset.columnfamily_layout_map_options:
             if layoutoption is None:
@@ -1311,6 +1341,9 @@ class Shell(cmd.Cmd):
                 if compclass is not None:
                     optmap['sstable_compression'] = \
                             trim_if_present(compclass, 'org.apache.cassandra.io.compress.')
+            if layoutoption == 'compaction_strategy_options':
+                optmap['class'] = compaction_strategy
+
             if self.cqlver_atleast(3) and not self.is_cql3_beta():
                 cf_opts.append((cql3option, optmap))
             else:
@@ -1356,8 +1389,9 @@ class Shell(cmd.Cmd):
         print
         if ksname is None:
             for k in self.get_keyspaces():
-                print 'Keyspace %s' % (k.name,)
-                print '---------%s' % ('-' * len(k.name))
+                name = self.cql_protect_name(k.name)
+                print 'Keyspace %s' % (name,)
+                print '---------%s' % ('-' * len(name))
                 cmd.Cmd.columnize(self, self.get_columnfamily_names(k.name))
                 print
         else:
@@ -2045,15 +2079,15 @@ def raw_option_with_default(configs, section, option, default=None):
 def should_use_color():
     if not sys.stdout.isatty():
         return False
-    if os.environ.get('TERM', 'dumb') == 'dumb':
+    if os.environ.get('TERM', '') in ('dumb', ''):
         return False
     try:
         import subprocess
         p = subprocess.Popen(['tput', 'colors'], stdout=subprocess.PIPE)
         stdout, _ = p.communicate()
         if int(stdout.strip()) < 8:
             return False
-    except (OSError, ImportError):
+    except (OSError, ImportError, ValueError):
         # oh well, we tried. at least we know there's a $TERM and it's
         # not "dumb".
         pass

pylib/cqlshlib/cql3handling.py

@@ -58,36 +58,34 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
     columnfamily_options = (
         # (CQL option name, Thrift option name (or None if same))
         ('comment', None),
+        ('compaction_strategy_class', 'compaction_strategy'),
         ('comparator', 'comparator_type'),
-        ('read_repair_chance', None),
-        ('gc_grace_seconds', None),
         ('default_validation', 'default_validation_class'),
+        ('gc_grace_seconds', None),
+        ('read_repair_chance', None),
         ('replicate_on_write', None),
-        ('compaction_strategy_class', 'compaction_strategy'),
     )
 
     old_columnfamily_layout_options = (
         # (CQL3 option name, schema_columnfamilies column name (or None if same))
-        ('comment', None),
         ('bloom_filter_fp_chance', None),
         ('caching', None),
-        ('read_repair_chance', None),
+        ('comment', None),
+        ('compaction_strategy_class', None),
         ('dclocal_read_repair_chance', 'local_read_repair_chance'),
         ('gc_grace_seconds', None),
+        ('read_repair_chance', None),
         ('replicate_on_write', None),
-        ('compaction_strategy_class', None),
     )
 
     new_columnfamily_layout_options = (
-        ('comment', None),
         ('bloom_filter_fp_chance', None),
         ('caching', None),
-        ('read_repair_chance', None),
+        ('comment', None),
         ('dclocal_read_repair_chance', 'local_read_repair_chance'),
         ('gc_grace_seconds', None),
+        ('read_repair_chance', None),
         ('replicate_on_write', None),
-        ('default_read_consistency', None),
-        ('default_write_consistency', None),
     )
 
     old_columnfamily_layout_map_options = (
@@ -103,18 +101,18 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
         # (CQL3 option name, schema_columnfamilies column name (or None if same),
         #  list of known map keys)
         ('compaction', 'compaction_strategy_options',
-            ('min_threshold', 'max_threshold')),
+            ('class', 'min_threshold', 'max_threshold')),
         ('compression', 'compression_parameters',
             ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')),
     )
 
     new_obsolete_cf_options = (
+        'compaction_parameters',
         'compaction_strategy_class',
         'compaction_strategy_options',
-        'min_compaction_threshold',
-        'max_compaction_threshold',
-        'compaction_parameters',
         'compression_parameters',
+        'max_compaction_threshold',
+        'min_compaction_threshold',
     )
 
     @staticmethod
@@ -1514,7 +1512,7 @@ def get_columns_compact(self):
         if len(self.column_aliases) == 0:
             if self.comparator is not UTF8Type:
                 warn(UnexpectedTableStructure("Compact storage CF %s has no column aliases,"
-                                              " but comparator is not UTF8Type." % (self,)))
+                                              " but comparator is not UTF8Type." % (self.name,)))
             colalias_types = []
         elif issubclass(self.comparator, CompositeType):
             colalias_types = self.comparator.subtypes

pylib/cqlshlib/displaying.py

@@ -53,20 +53,24 @@ def _pad(self, width, fill=' '):
         else:
             return ''
 
-    def ljust(self, width, fill=' '):
+    def ljust(self, width, fill=' ', color=False):
         """
         Similar to self.strval.ljust(width), but takes expected terminal
         display width into account for special characters, and does not
         take color escape codes into account.
         """
+        if color:
+            return self.color_ljust(width, fill=fill)
         return self.strval + self._pad(width, fill)
 
-    def rjust(self, width, fill=' '):
+    def rjust(self, width, fill=' ', color=False):
         """
         Similar to self.strval.rjust(width), but takes expected terminal
         display width into account for special characters, and does not
         take color escape codes into account.
         """
+        if color:
+            return self.color_rjust(width, fill=fill)
         return self._pad(width, fill) + self.strval
 
     def color_rjust(self, width, fill=' '):