From 479060426966d2b4de23b35b7d28ddb3e4b64344 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 31 Oct 2016 20:56:17 -0700 Subject: [PATCH 1/5] PoC TSQA running against existing builds --- ci/tsqa/tests/helpers.py | 24 +-------------- ci/tsqa/tests/test_buildoptions.py | 47 ------------------------------ ci/tsqa/tests/test_example.py | 16 +++------- ci/tsqa/tests/test_regressions.py | 10 ------- 4 files changed, 5 insertions(+), 92 deletions(-) delete mode 100644 ci/tsqa/tests/test_buildoptions.py diff --git a/ci/tsqa/tests/helpers.py b/ci/tsqa/tests/helpers.py index af1104b34e2..5bc45ce151e 100644 --- a/ci/tsqa/tests/helpers.py +++ b/ci/tsqa/tests/helpers.py @@ -32,29 +32,7 @@ def tests_file_path(path): return os.path.join(base, path) -class EnvironmentCase(tsqa.test_cases.EnvironmentCase): +class EnvironmentCase(tsqa.test_cases.CloneEnvironmentCase): ''' This class will get an environment (which is unique) but won't start it ''' - @classmethod - def getEnv(cls): - ''' - This function is responsible for returning an environment - ''' - SOURCE_DIR = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) - TMP_DIR = os.path.join(tempfile.gettempdir(), 'tsqa') - ef = tsqa.environment.EnvironmentFactory(SOURCE_DIR, - os.path.join(TMP_DIR, 'base_envs'), - default_configure={'enable-experimental-plugins': None, - 'enable-example-plugins': None, - 'enable-test-tools': None, - 'disable-dependency-tracking': None, - 'enable-ccache': None, - }, - ) - # TODO: figure out a way to determine why the build didn't fail and - # not skip all build failures? - try: - return ef.get_environment(cls.environment_factory.get('configure'), cls.environment_factory.get('env')) - except Exception as e: - raise unittest.SkipTest(e) diff --git a/ci/tsqa/tests/test_buildoptions.py b/ci/tsqa/tests/test_buildoptions.py deleted file mode 100644 index fd0f1125c2d..00000000000 --- a/ci/tsqa/tests/test_buildoptions.py +++ /dev/null @@ -1,47 +0,0 @@ -''' -Test that configuration options successfully compile -''' - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import helpers - -log = logging.getLogger(__name__) - - -class TestBuildOption(helpers.EnvironmentCase): - ''' - Run the built-in traffic_server regression test suite. - ''' - def test_buildoption(self): - pass - - -class TestBuildOptionFastSDK(TestBuildOption): - '''Build with --enable-fast-sdk''' - environment_factory = {'configure': {'enable-fast-sdk': None}} - - -class TestBuildOptionDisableDiags(TestBuildOption): - '''Build with --disable-diags''' - environment_factory = {'configure': {'disable-diags': None}} - - -class TestBuildOptionDisableTests(TestBuildOption): - '''Build with --disable-tests''' - environment_factory = {'configure': {'disable-tests': None}} diff --git a/ci/tsqa/tests/test_example.py b/ci/tsqa/tests/test_example.py index 2fcda644ba1..3ba0cf04cdc 100644 --- a/ci/tsqa/tests/test_example.py +++ b/ci/tsqa/tests/test_example.py @@ -37,14 +37,9 @@ class TestNoOp(helpers.EnvironmentCase): ''' This is purely a documentation test ''' - # you can set configure/environment options for the source build here - environment_factory = { - 'configure': { - 'enable-wccp': None, # A value of None means that the argument has no value - 'with-max-api-stats': 2048, # if there is a value it will be converted to --key=value - }, - 'env': None, - } + # you can set dependencies on specific "features" (according to traffic_layout) + # if the requirement isn't met the test is skipped + feature_requirements = {'TS_HAS_WCCP': 0} @classmethod def setUpClass(cls): @@ -86,10 +81,7 @@ def test_something(self): class TestConfigureFlags(helpers.EnvironmentCase): - environment_factory = { - 'configure': {'enable-wccp': None}, - } - + feature_requirements = {'TS_HAS_WCCP': 0} def test_wccp(self): self.assertTrue(True) diff --git a/ci/tsqa/tests/test_regressions.py b/ci/tsqa/tests/test_regressions.py index e49aa74ae62..ba0b39f8ef0 100644 --- a/ci/tsqa/tests/test_regressions.py +++ b/ci/tsqa/tests/test_regressions.py @@ -51,13 +51,3 @@ def tearDownClass(cls): def test_regressions(self): cmd = [os.path.join(self.environment.layout.bindir, 'traffic_server'), '-R', '1'] tsqa.utils.run_sync_command(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - -class TestRegressionsLinuxNativeAIO(TestRegressions): - ''' - Run the built-in traffic_server regression test suite with - --enable-linux-native-aio. - ''' - environment_factory = { - 'configure': {'enable-linux-native-aio': None}, - } From 291a3b6056d3f5dc29474a6432e81926e897ed03 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 31 Oct 2016 21:16:50 -0700 Subject: [PATCH 2/5] Fix broken test-- invalid python --- ci/tsqa/tests/test_https.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tsqa/tests/test_https.py b/ci/tsqa/tests/test_https.py index c0515c62ca9..8e45d6ed76e 100644 --- a/ci/tsqa/tests/test_https.py +++ b/ci/tsqa/tests/test_https.py @@ -324,7 +324,7 @@ def test_config_file_group(self): time.sleep(5) os.system('cp %s %s' % (helpers.tests_file_path('ec_keys/www.test.com.pem'), helpers.tests_file_path('www.unknown.com.pem'))) log.info('cp %s %s' % (helpers.tests_file_path('ec_keys/www.test.com.pem'), helpers.tests_file_path('www.unknown.com.pem'))) - os.system(signal_cmd) + os.system(' '.join(signal_cmd)) log.info(signal_cmd) # waiting for the reconfiguration completed sec = 0 From 032dd51345067334e1f52e23dbed690b536d2d40 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 31 Oct 2016 21:28:16 -0700 Subject: [PATCH 3/5] Fix broken tests --- ci/tsqa/tests/test_chunked.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/tsqa/tests/test_chunked.py b/ci/tsqa/tests/test_chunked.py index e56b940e4c2..1a6861441bf 100644 --- a/ci/tsqa/tests/test_chunked.py +++ b/ci/tsqa/tests/test_chunked.py @@ -194,5 +194,6 @@ def test_chunked_keepalive_client(self): def test_chunked_bad_close(self): url = 'http://127.0.0.1:{0}/5/0.1/false'.format(self.port) - with self.assertRaises(socket.timeout): + # TODO: better exception catch (seems to be ConnectionError) + with self.assertRaises(Exception): requests.get(url, proxies=self.proxies, timeout=2) From 3d49b2966b87b445c24a3352d4007e3e3149f73e Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 1 Nov 2016 08:23:05 -0700 Subject: [PATCH 4/5] Add timeouts, don't want tests to run forever --- ci/tsqa/tests/test_connect_attempts.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/tsqa/tests/test_connect_attempts.py b/ci/tsqa/tests/test_connect_attempts.py index d52f41e41f9..5bb41bfdd5f 100644 --- a/ci/tsqa/tests/test_connect_attempts.py +++ b/ci/tsqa/tests/test_connect_attempts.py @@ -199,23 +199,23 @@ def _add_sock(name): def test_bound_origin(self): '''Verify that we get 502s from an origin which just did a bind''' url = 'http://127.0.0.1:{0}/bound/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 502) def test_listen_origin(self): '''Verify that we get 502s from origins that bind + listen''' url = 'http://127.0.0.1:{0}/listen/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 502) url = 'http://127.0.0.1:{0}/listen/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 502) def test_die_on_connect_origin(self): '''Verify that we get 504s from origins that die_on_connect''' url = 'http://127.0.0.1:{0}/die_on_connect/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 504) def test_partial_response_origin(self): @@ -226,25 +226,25 @@ def test_partial_response_origin(self): gaurantee that the request is re-entrant ''' url = 'http://127.0.0.1:{0}/partial_response/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 500) def test_reset_after_accept_origin(self): '''Verify that we get 502s from origins that reset_after_accept, once any bytes are sent to origin we assume we cannot re-dispatch''' url = 'http://127.0.0.1:{0}/reset_after_accept/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) self.assertEqual(ret.status_code, 502) def test_slow_response(self): '''Verify that we get 5xx from origins that take longer than acceptable, since we will not retry them''' url = 'http://127.0.0.1:{0}/slow_response/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) # make sure it worked self.assertEqual(ret.status_code, 504) def test_slow_close(self): '''Verify that we retry connecting to an origin when there is a connection failure''' url = 'http://127.0.0.1:{0}/slow_close/s'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports']) - ret = requests.get(url) + ret = requests.get(url, timeout=2) # make sure it worked self.assertEqual(ret.status_code, 200) From 3bb533f2417cebfc9c31f86eab130db2968d3703 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 1 Nov 2016 08:23:37 -0700 Subject: [PATCH 5/5] Don't re-run regression tests as of now the CI already runs the regression tests, so there is no need to re-run them in tsqa. At some point in the future we may consolidate these-- but for now we'll leave them separate --- ci/tsqa/tests/test_regressions.py | 53 ------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 ci/tsqa/tests/test_regressions.py diff --git a/ci/tsqa/tests/test_regressions.py b/ci/tsqa/tests/test_regressions.py deleted file mode 100644 index ba0b39f8ef0..00000000000 --- a/ci/tsqa/tests/test_regressions.py +++ /dev/null @@ -1,53 +0,0 @@ -''' -Run the built-in regression tests with experimental build configurations. -''' - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import logging - -import helpers -import tsqa.test_cases -import tsqa.utils - -log = logging.getLogger(__name__) - - -class TestRegressions(helpers.EnvironmentCase): - ''' - Run the built-in traffic_server regression test suite. - ''' - - # NOTE: we need to stop the running Traffic Server in the environment so - # that we can start up our own. Make sure to restart it when we are done so - # that the EnvironmentCase doesn't get upset. - - @classmethod - def setUpClass(cls): - super(TestRegressions, cls).setUpClass() - cls.environment.stop() - - @classmethod - def tearDownClass(cls): - cls.environment.start() - super(TestRegressions, cls).tearDownClass() - - def test_regressions(self): - cmd = [os.path.join(self.environment.layout.bindir, 'traffic_server'), '-R', '1'] - tsqa.utils.run_sync_command(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)