Handle sqlalchemy-migrate for keystone and nova

Add a lib in devstack to install CockroachDB. This lib provides two
functions: `database_start_switch_cockroachdb` and
`database_stop_switch_cockroachdb`. The soft `stack.sh`, that tells
which database backend to use, is patched so, devstack installs both
postgresql and cockroachdb. Then we put switching function to tell a
service use the one or other.

Vagrantfile

@@ -4,14 +4,13 @@
 # Vagrantfile Deploying 2 nodes in order to compare pgsql and cockroachdb backends
 #
 DEBUG = false
-G5K_USER = "acarat"
 
 Vagrant.configure(2) do |config|
 
   # Configuration for VirtualBox
   config.vm.provider :virtualbox do |vb, override|
-    vb.cpus = 4
-    vb.memory = 4096
+    vb.cpus = 6
+    vb.memory = 8192
     override.vm.synced_folder "./", "/vagrant_data",
                               owner: "vagrant",
                               group: "vagrant"
@@ -58,8 +57,8 @@ Vagrant.configure(2) do |config|
 
     ## VM size customization default values are
     g5k.resources = {
-      :cpu => 6,
-      :mem => 8192
+      :cpu => 4,
+      :mem => 4096
     }
   end #g5k
 
@@ -72,22 +71,6 @@ Vagrant.configure(2) do |config|
       ansible.playbook = "provision.yml"
       # ansible.verbose = "-vvvv"
       ansible.extra_vars = {
-        :backend => "cockroachdb",
-        :debug => DEBUG
-      }
-    end
-  end
-
-  # VM for PostreSQL backend
-  config.vm.define "psql" do |psql|
-    psql.vm.box = "debian/contrib-jessie64"
-    psql.vm.provision :ansible_local do |ansible|
-      ansible.install_mode = "pip"
-      ansible.version = "2.3.1.0"
-      ansible.playbook = "provision.yml"
-      # ansible.verbose = "-vvvv"
-      ansible.extra_vars = {
-        :backend => "psql",
         :debug => DEBUG
       }
     end

lib/subunit_trace.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 Samsung Electronics
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+import subunit
+import testtools
+import json
+
+import six
+import ast
+
+OSTREAM = sys.stdout
+
+def split_lines(content):
+    """Split log into lines"""
+    res = []
+
+    if isinstance(content, six.string_types):
+        res = content.splitlines()
+    elif isinstance(content, testtools.content.Content):
+        res = content.as_text().splitlines()
+
+    return res
+
+def filter_sql_query(log):
+    """Extract SQL statements from log"""
+    sqls = []
+
+    for q in log:
+        if 'sqlalchemy.engine.base.Engine' in q:
+            # Remove '2017-11-22 15:17:14,810 INFO [sqlalchemy.engine.base.Engine] '
+            q = q[61:].strip()
+            sqls.append(q)
+        elif sqls and not q.startswith('2017-'):
+            # Rest of previous SQL query: append to previous sql
+            sqls[-1] = sqls[-1] + ' ' + q.strip()
+
+    return sqls
+
+def expand_sql_template(log):
+    """Expand SQL template.
+
+    Expand
+      > SELECT a FROM b WHERE a.c = ? AND a.d = ?
+      > (1,2)
+    Into
+      > SELECT a FROM b WHERE a.c = 1 AND a.d = 2
+    """
+
+    # Expand template
+    log.reverse()
+    for i, l in enumerate(log):
+        if l.startswith('('):
+            values = ast.literal_eval(l) # Template values
+            sql = log[i+1]               # Templated query
+
+            # Expand
+            for v in values:
+                sql = sql.replace('?', str(v), 1)
+
+            log[i+1] = sql
+
+    # Remove template values
+    log.reverse()
+    log = [ sql for sql in log if not sql.startswith('(') ]
+
+    return log
+
+def formatsql(log):
+    log = split_lines(log)
+    log = filter_sql_query(log)
+    log = expand_sql_template(log)
+
+    return log
+
+def handle_test(test):
+    name = test['id']
+    status = test['status'] is 'success'
+    sql = formatsql(test['details'].get("pythonlogging:''", ''))
+
+    if status and sql:
+        res = {
+            'name': name,
+            'sql': sql
+        }
+        OSTREAM.write(json.dumps(res))
+        OSTREAM.write(',\n')
+
+def trace():
+    case = subunit.ByteStreamToStreamResult(sys.stdin, non_subunit_name='stdout')
+    result = testtools.StreamToDict(handle_test) # Call handle_test
+                                                 # when test completes
+    OSTREAM.write("[\n")
+    result.startTestRun()
+    case.run(result)
+    result.stopTestRun()
+    OSTREAM.write("]\n")
+
+
+exit(trace())