backported doc test setup from master

 - fixed port assignment bug in setup for doc tests
 - improved performance of itest run

docs/src/crate/ports.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
 import socket
+from threading import Lock
+
 
 def public_ipv4():
     """
@@ -14,10 +16,43 @@ def public_ipv4():
             return addrinfo[4][0]
 
 
-def random_available_port():
-    """return some available port reported by the kernel"""
-    sock = socket.socket()
-    sock.bind(('', 0))
-    port = sock.getsockname()[1]
-    sock.close()
-    return port
+class PortPool(object):
+    """
+    Pool that returns a unique available port
+    reported by the kernel.
+    """
+
+    MAX_RETRIES = 10
+
+    def __init__(self):
+        self.ports = set()
+        self.lock = Lock()
+
+    def random_available_port(self, addr):
+        sock = socket.socket()
+        sock.bind((addr, 0))
+        port = sock.getsockname()[1]
+        try:
+            sock.shutdown(socket.SHUT_RDWR)
+        except:
+            # ok, at least we know that the socket is not connected
+            pass
+        sock.close()
+        return port
+
+    def get(self, addr='127.0.0.1'):
+        retries = 0
+        port = self.random_available_port(addr)
+
+        with self.lock:
+            while port in self.ports:
+                port = self.random_available_port(addr)
+                retries += 1
+                if retries > self.MAX_RETRIES:
+                    raise OSError("Could not get free port. Max retries exceeded.")
+            self.ports.add(port)
+        return port
+
+
+GLOBAL_PORT_POOL = PortPool()
+

docs/src/crate/process_test.py

@@ -7,12 +7,22 @@
 from crate.testing.layer import CrateLayer
 from crate.client.http import Client
 from .paths import crate_path
-from .ports import public_ipv4, random_available_port
+from .ports import GLOBAL_PORT_POOL
 from lovely.testlayers.layer import CascadedLayer
 
 
 class GracefulStopCrateLayer(CrateLayer):
 
+    MAX_RETRIES = 3
+
+    def start(self, retry=0):
+        if retry >= self.MAX_RETRIES:
+            raise SystemError('Could not start Crate server. Max retries exceeded!')
+        try:
+            super(GracefulStopCrateLayer, self).start()
+        except Exception as e:
+            self.start(retry=retry+1)
+
     def stop(self):
         """do not care if process already died"""
         try:
@@ -37,9 +47,9 @@ def __init__(self, *args, **kwargs):
         for i in range(num_servers):
             layer = GracefulStopCrateLayer(self.node_name(i),
                            crate_path(),
-                           host=public_ipv4(),
-                           port=random_available_port(),
-                           transport_port=random_available_port(),
+                           host='0.0.0.0',
+                           port = GLOBAL_PORT_POOL.get(),
+                           transport_port = GLOBAL_PORT_POOL.get(),
                            multicast=True,
                            cluster_name=self.__class__.__name__)
             client = Client(layer.crate_servers)
@@ -106,20 +116,21 @@ class TestGracefulStopPrimaries(GracefulStopTest):
 
     NUM_SERVERS = 2
 
+    def setUp(self):
+        super(TestGracefulStopPrimaries, self).setUp()
+        client = self.clients[0]
+        client.sql("create table t1 (id int, name string) "
+                   "clustered into 4 shards "
+                   "with (number_of_replicas=0)")
+        client.sql("insert into t1 (id, name) values (?, ?), (?, ?)",
+                   (1, "Ford", 2, "Trillian"))
+        client.sql("refresh table t1")
+
     def test_graceful_stop_primaries(self):
         """
         test min_availability: primaries
         """
-
-        client1 = self.clients[0]
         client2 = self.clients[1]
-
-        client1.sql("create table t1 (id int, name string) "
-                    "clustered into 4 shards "
-                    "with (number_of_replicas=0)")
-        client1.sql("insert into t1 (id, name) values (?, ?), (?, ?)",
-                    (1, "Ford", 2, "Trillian"))
-        client1.sql("refresh table t1")
         self.settings({
             "cluster.graceful_stop.min_availability": "primaries",
             "cluster.routing.allocation.enable": "new_primaries"
@@ -131,24 +142,31 @@ def test_graceful_stop_primaries(self):
         # assert that all shards are assigned
         self.assertEqual(response.get("rowcount", -1), 0)
 
+    def tearDown(self):
+        client = self.clients[1]
+        client.sql("drop table t1")
+
 
 class TestGracefulStopFull(GracefulStopTest):
 
     NUM_SERVERS = 3
 
+    def setUp(self):
+        super(TestGracefulStopFull, self).setUp()
+        client = self.clients[0]
+        client.sql("create table t1 (id int, name string) "
+                    "clustered into 4 shards "
+                    "with (number_of_replicas=1)")
+        client.sql("insert into t1 (id, name) values (?, ?), (?, ?)",
+                    (1, "Ford", 2, "Trillian"))
+        client.sql("refresh table t1")
+
     def test_graceful_stop_full(self):
         """
         min_availability: full moves all shards
         """
-        crate1, crate2, crate3 = self.crates[0], self.crates[1], self.crates[2]
-        client1, client2, client3 = self.clients[0], self.clients[1], self.clients[2]
-
-        client1.sql("create table t1 (id int, name string) "
-                    "clustered into 4 shards "
-                    "with (number_of_replicas=1)")
-        client1.sql("insert into t1 (id, name) values (?, ?), (?, ?)",
-                    (1, "Ford", 2, "Trillian"))
-        client1.sql("refresh table t1")
+        crate1, crate2, crate3 = self.crates
+        client1, client2, client3 = self.clients
         self.settings({
             "cluster.graceful_stop.min_availability": "full",
             "cluster.routing.allocation.enable": "new_primaries"
@@ -160,29 +178,36 @@ def test_graceful_stop_full(self):
         # assert that all shards are assigned
         self.assertEqual(response.get("rowcount", -1), 0)
 
+    def tearDown(self):
+        client = self.clients[2]
+        client.sql("drop table t1")
+
 
 class TestGracefulStopNone(GracefulStopTest):
 
     NUM_SERVERS = 2
 
+    def setUp(self):
+        super(TestGracefulStopNone, self).setUp()
+        client = self.clients[0]
+
+        client.sql("create table t1 (id int, name string) "
+                   "clustered into 8 shards "
+                   "with (number_of_replicas=0)")
+        client.sql("refresh table t1")
+        names = ("Ford", "Trillian", "Zaphod", "Jeltz")
+        for i in range(16):
+            client.sql("insert into t1 (id, name) "
+                       "values (?, ?)",
+                       (i, random.choice(names)))
+        client.sql("refresh table t1")
+
     def test_graceful_stop_none(self):
         """
         test min_availability: none
         """
-
-        client1 = self.clients[0]
         client2 = self.clients[1]
 
-        client1.sql("create table t1 (id int, name string) "
-                    "clustered into 8 shards "
-                    "with (number_of_replicas=0)")
-        client1.sql("refresh table t1")
-        names = ("Ford", "Trillian", "Zaphod", "Jeltz")
-        for i in range(16):
-            client1.sql("insert into t1 (id, name) "
-                        "values (?, ?)",
-                        (i, random.choice(names)))
-        client1.sql("refresh table t1")
         self.settings({
             "cluster.graceful_stop.min_availability": "none",
             "cluster.routing.allocation.enable": "none"
@@ -200,3 +225,8 @@ def test_graceful_stop_none(self):
             unassigned_shards > 0,
             "{0} unassigned shards, expected more than 0".format(unassigned_shards)
         )
+
+    def tearDown(self):
+        client = self.clients[1]
+        client.sql("drop table t1")
+

docs/src/crate/tests.py

@@ -7,18 +7,22 @@
 import re
 import process_test
 from .paths import crate_path, project_path
-from .ports import random_available_port
+from .ports import GLOBAL_PORT_POOL
 from crate.crash.command import CrateCmd
 from crate.crash.printer import PrintWrapper, ColorPrinter
 from crate.client import connect
 
 
+CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get()
+CRATE_TRANSPORT_PORT = GLOBAL_PORT_POOL.get()
+
+
 class CrateTestCmd(CrateCmd):
 
     def __init__(self, **kwargs):
         super(CrateTestCmd, self).__init__(**kwargs)
         doctest_print = PrintWrapper()
-        self.logger = ColorPrinter(False, stream=doctest_print, line_end='')
+        self.logger = ColorPrinter(False, stream=doctest_print, line_end='\n')
 
     def stmt(self, stmt):
         stmt = stmt.replace('\n', ' ')
@@ -30,10 +34,6 @@ def stmt(self, stmt):
 cmd = CrateTestCmd(is_tty=False)
 
 
-CRATE_HTTP_PORT = random_available_port()
-CRATE_TRANSPORT_PORT = random_available_port()
-
-
 def wait_for_schema_update(schema, table, column):
     conn = connect('localhost:' + str(CRATE_HTTP_PORT))
     c = conn.cursor()
@@ -121,11 +121,13 @@ def setUpLocations(test):
     cmd.stmt("""copy locations from '{0}'""".format(locations_file))
     cmd.stmt("""refresh table locations""")
 
+def tearDownLocations(test):
+    cmd.stmt("""drop table locations""")
 
 def setUpUserVisits(test):
     test.globs['cmd'] = cmd
     cmd.stmt("""
-        create table uservisits(
+        create table uservisits (
           id integer primary key,
           name string,
           visits integer,
@@ -136,6 +138,8 @@ def setUpUserVisits(test):
     cmd.stmt("""copy uservisits from '{0}'""".format(uservisits_file))
     cmd.stmt("""refresh table uservisits""")
 
+def tearDownUserVisits(test):
+    cmd.stmt("""drop table uservisits""")
 
 def setUpQuotes(test):
     test.globs['cmd'] = cmd
@@ -151,16 +155,24 @@ def setUpQuotes(test):
     shutil.copy(project_path('sql/src/test/resources/essetup/data/copy', 'test_copy_from.json'),
                 os.path.join(import_dir, "quotes.json"))
 
+def tearDownQuotes(test):
+    cmd.stmt("""drop table quotes""")
 
 def setUpLocationsAndQuotes(test):
     setUpLocations(test)
     setUpQuotes(test)
 
+def tearDownLocationsAndQuotes(test):
+    tearDownLocations(test)
+    tearDownQuotes(test)
 
 def setUpLocationsQuotesAndUserVisits(test):
     setUpLocationsAndQuotes(test)
     setUpUserVisits(test)
 
+def tearDownLocationsQuotesAndUserVisits(test):
+    tearDownLocationsAndQuotes(test)
+    tearDownUserVisits(test)
 
 def setUpTutorials(test):
     setUp(test)
@@ -175,36 +187,36 @@ def setUpTutorials(test):
     shutil.copy(project_path(source_dir, 'data_import_1408312800.json'),
                 os.path.join(import_dir, "users_1408312800.json"))
 
-
 def setUp(test):
     test.globs['cmd'] = cmd
     test.globs['wait_for_schema_update'] = wait_for_schema_update
 
 
-def tearDownDropQuotes(test):
-    cmd.stmt("drop table quotes")
-
-
 def test_suite():
     suite = unittest.TestSuite()
-    processSuite = unittest.TestLoader().loadTestsFromModule(process_test)
-    suite.addTest(processSuite)
+
+    # Graceful stop tests
+    process_suite = unittest.TestLoader().loadTestsFromModule(process_test)
+    suite.addTest(process_suite)
+
+    # Documentation tests
+    docs_suite = unittest.TestSuite()
     s = doctest.DocFileSuite('../../blob.txt',
                              parser=bash_parser,
                              setUp=setUp,
-                             tearDown=tearDownDropQuotes,
                              optionflags=doctest.NORMALIZE_WHITESPACE |
                              doctest.ELLIPSIS)
     s.layer = empty_layer
-    suite.addTest(s)
+    docs_suite.addTest(s)
     for fn in ('sql/rest.txt',):
         s = doctest.DocFileSuite('../../' + fn,
                                  parser=bash_parser,
                                  setUp=setUpLocations,
+                                 tearDown=tearDownLocations,
                                  optionflags=doctest.NORMALIZE_WHITESPACE |
                                  doctest.ELLIPSIS)
         s.layer = empty_layer
-        suite.addTest(s)
+        docs_suite.addTest(s)
     for fn in ('sql/ddl.txt',
                'sql/dql.txt',
                'sql/refresh.txt',
@@ -221,31 +233,34 @@ def test_suite():
                'hello.txt'):
         s = doctest.DocFileSuite('../../' + fn, parser=crash_parser,
                                  setUp=setUpLocationsAndQuotes,
+                                 tearDown=tearDownLocationsAndQuotes,
                                  optionflags=doctest.NORMALIZE_WHITESPACE |
                                  doctest.ELLIPSIS)
         s.layer = empty_layer
-        suite.addTest(s)
+        docs_suite.addTest(s)
     for fn in ('sql/dml.txt',):
         s = doctest.DocFileSuite('../../' + fn, parser=crash_parser,
                                  setUp=setUpLocationsQuotesAndUserVisits,
+                                 tearDown=tearDownLocationsQuotesAndUserVisits,
                                  optionflags=doctest.NORMALIZE_WHITESPACE |
                                  doctest.ELLIPSIS)
         s.layer = empty_layer
-        suite.addTest(s)
+        docs_suite.addTest(s)
     for fn in ('best_practice/migrating_from_mongodb.txt',):
         path = os.path.join('..', '..', fn)
         s = doctest.DocFileSuite(path, parser=crash_parser,
                                  setUp=setUp,
                                  optionflags=doctest.NORMALIZE_WHITESPACE |
                                  doctest.ELLIPSIS)
         s.layer = empty_layer
-        suite.addTest(s)
+        docs_suite.addTest(s)
     for fn in ('data_import.txt', 'cluster_upgrade.txt'):
         path = os.path.join('..', '..', 'best_practice', fn)
         s = doctest.DocFileSuite(path, parser=crash_parser,
                                  setUp=setUpTutorials,
                                  optionflags=doctest.NORMALIZE_WHITESPACE |
                                  doctest.ELLIPSIS)
         s.layer = empty_layer
-        suite.addTest(s)
+        docs_suite.addTest(s)
+    suite.addTests(docs_suite)
     return suite