From 974f1822b6d496fe9af269e3bedb6521f5ec27f0 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Wed, 4 Sep 2019 12:00:19 -0400 Subject: [PATCH 01/19] more stuff --- requirements/dev.txt | 1 + salt/pillar/environments.py | 5 ++ tests/unit/test_master.py | 107 +++++++++++++++++++++++------------- 3 files changed, 75 insertions(+), 38 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index b367c2c76615..58ce2e84e555 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,6 +5,7 @@ SaltPyLint>=v2017.3.6 pytest>=3.5.0 git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 +bloomberg.hostinfo # httpretty Needs to be here for now even though it's a dependency of boto. # A pip install on a fresh system will decide to target httpretty 0.8.10 to diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index 6a695a063e54..83da3b592100 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -161,6 +161,11 @@ def ext_pillar(minion_id, pillar): :return: a dictionary which is included by the salt master in the pillars returned to the minion """ + # hostinfo resolving a node that is None will throw an error + # Instead lets just return empty array + if not minion_id: + return [] + node = resolve_node(minion_id) if node is None: diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index b7394ffa01d7..6cda67ca8249 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -15,11 +15,42 @@ ) +class AESFuncsTestCase(TestCase): + ''' + TestCase for salt.master.AESFuncs class + ''' + def test__file_envs_no_matching_node(self): + # Default master opts + opts = salt.config.master_config(None) + opts['ext_pillar'] = { + 'environments': [ + 'word' + ] + } + + self.aes_funcs = salt.master.AESFuncs(opts) + res = self.aes_funcs._file_envs({ + "id": "pytest_minion_1" + }) + self.assertEqual(res, []) + + def test__file_envs_load_is_none(self): + # Default master opts + opts = salt.config.master_config(None) + opts['ext_pillar'] = { + 'environments': [ + 'word' + ] + } + + self.aes_funcs = salt.master.AESFuncs(opts) + res = self.aes_funcs._file_envs() + self.assertEqual(res, []) + class ClearFuncsTestCase(TestCase): ''' TestCase for salt.master.ClearFuncs class ''' - def setUp(self): opts = salt.config.master_config(None) self.clear_funcs = salt.master.ClearFuncs(opts, {}) @@ -31,7 +62,7 @@ def test_runner_token_not_authenticated(self): Asserts that a TokenAuthenticationError is returned when the token can't authenticate. ''' mock_ret = {'error': {'name': 'TokenAuthenticationError', - 'message': 'Authentication failure of type "token" occurred.'}} + 'message': 'Authentication failure of type "token" occurred.'}} ret = self.clear_funcs.runner({'token': 'asdfasdfasdfasdf'}) self.assertDictEqual(mock_ret, ret) @@ -44,11 +75,11 @@ def test_runner_token_authorization_error(self): clear_load = {'token': token, 'fun': 'test.arg'} mock_token = {'token': token, 'eauth': 'foo', 'name': 'test'} mock_ret = {'error': {'name': 'TokenAuthenticationError', - 'message': 'Authentication failure of type "token" occurred ' - 'for user test.'}} + 'message': 'Authentication failure of type "token" occurred ' + 'for user test.'}} with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): ret = self.clear_funcs.runner(clear_load) self.assertDictEqual(mock_ret, ret) @@ -62,10 +93,10 @@ def test_runner_token_salt_invocation_error(self): clear_load = {'token': token, 'fun': 'badtestarg'} mock_token = {'token': token, 'eauth': 'foo', 'name': 'test'} mock_ret = {'error': {'name': 'SaltInvocationError', - 'message': 'A command invocation error occurred: Check syntax.'}} + 'message': 'A command invocation error occurred: Check syntax.'}} with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): ret = self.clear_funcs.runner(clear_load) self.assertDictEqual(mock_ret, ret) @@ -75,8 +106,8 @@ def test_runner_eauth_not_authenticated(self): Asserts that an EauthAuthenticationError is returned when the user can't authenticate. ''' mock_ret = {'error': {'name': 'EauthAuthenticationError', - 'message': 'Authentication failure of type "eauth" occurred for ' - 'user UNKNOWN.'}} + 'message': 'Authentication failure of type "eauth" occurred for ' + 'user UNKNOWN.'}} ret = self.clear_funcs.runner({'eauth': 'foo'}) self.assertDictEqual(mock_ret, ret) @@ -87,10 +118,10 @@ def test_runner_eauth_authorization_error(self): ''' clear_load = {'eauth': 'foo', 'username': 'test', 'fun': 'test.arg'} mock_ret = {'error': {'name': 'EauthAuthenticationError', - 'message': 'Authentication failure of type "eauth" occurred for ' - 'user test.'}} + 'message': 'Authentication failure of type "eauth" occurred for ' + 'user test.'}} with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): ret = self.clear_funcs.runner(clear_load) self.assertDictEqual(mock_ret, ret) @@ -102,9 +133,9 @@ def test_runner_eauth_salt_invocation_error(self): ''' clear_load = {'eauth': 'foo', 'username': 'test', 'fun': 'bad.test.arg.func'} mock_ret = {'error': {'name': 'SaltInvocationError', - 'message': 'A command invocation error occurred: Check syntax.'}} + 'message': 'A command invocation error occurred: Check syntax.'}} with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): ret = self.clear_funcs.runner(clear_load) self.assertDictEqual(mock_ret, ret) @@ -114,7 +145,7 @@ def test_runner_user_not_authenticated(self): Asserts that an UserAuthenticationError is returned when the user can't authenticate. ''' mock_ret = {'error': {'name': 'UserAuthenticationError', - 'message': 'Authentication failure of type "user" occurred'}} + 'message': 'Authentication failure of type "user" occurred'}} ret = self.clear_funcs.runner({}) self.assertDictEqual(mock_ret, ret) @@ -125,7 +156,7 @@ def test_wheel_token_not_authenticated(self): Asserts that a TokenAuthenticationError is returned when the token can't authenticate. ''' mock_ret = {'error': {'name': 'TokenAuthenticationError', - 'message': 'Authentication failure of type "token" occurred.'}} + 'message': 'Authentication failure of type "token" occurred.'}} ret = self.clear_funcs.wheel({'token': 'asdfasdfasdfasdf'}) self.assertDictEqual(mock_ret, ret) @@ -138,11 +169,11 @@ def test_wheel_token_authorization_error(self): clear_load = {'token': token, 'fun': 'test.arg'} mock_token = {'token': token, 'eauth': 'foo', 'name': 'test'} mock_ret = {'error': {'name': 'TokenAuthenticationError', - 'message': 'Authentication failure of type "token" occurred ' - 'for user test.'}} + 'message': 'Authentication failure of type "token" occurred ' + 'for user test.'}} with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): ret = self.clear_funcs.wheel(clear_load) self.assertDictEqual(mock_ret, ret) @@ -156,10 +187,10 @@ def test_wheel_token_salt_invocation_error(self): clear_load = {'token': token, 'fun': 'badtestarg'} mock_token = {'token': token, 'eauth': 'foo', 'name': 'test'} mock_ret = {'error': {'name': 'SaltInvocationError', - 'message': 'A command invocation error occurred: Check syntax.'}} + 'message': 'A command invocation error occurred: Check syntax.'}} with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): ret = self.clear_funcs.wheel(clear_load) self.assertDictEqual(mock_ret, ret) @@ -169,8 +200,8 @@ def test_wheel_eauth_not_authenticated(self): Asserts that an EauthAuthenticationError is returned when the user can't authenticate. ''' mock_ret = {'error': {'name': 'EauthAuthenticationError', - 'message': 'Authentication failure of type "eauth" occurred for ' - 'user UNKNOWN.'}} + 'message': 'Authentication failure of type "eauth" occurred for ' + 'user UNKNOWN.'}} ret = self.clear_funcs.wheel({'eauth': 'foo'}) self.assertDictEqual(mock_ret, ret) @@ -181,10 +212,10 @@ def test_wheel_eauth_authorization_error(self): ''' clear_load = {'eauth': 'foo', 'username': 'test', 'fun': 'test.arg'} mock_ret = {'error': {'name': 'EauthAuthenticationError', - 'message': 'Authentication failure of type "eauth" occurred for ' - 'user test.'}} + 'message': 'Authentication failure of type "eauth" occurred for ' + 'user test.'}} with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): ret = self.clear_funcs.wheel(clear_load) self.assertDictEqual(mock_ret, ret) @@ -196,9 +227,9 @@ def test_wheel_eauth_salt_invocation_error(self): ''' clear_load = {'eauth': 'foo', 'username': 'test', 'fun': 'bad.test.arg.func'} mock_ret = {'error': {'name': 'SaltInvocationError', - 'message': 'A command invocation error occurred: Check syntax.'}} + 'message': 'A command invocation error occurred: Check syntax.'}} with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ - patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=['testing'])): ret = self.clear_funcs.wheel(clear_load) self.assertDictEqual(mock_ret, ret) @@ -208,7 +239,7 @@ def test_wheel_user_not_authenticated(self): Asserts that an UserAuthenticationError is returned when the user can't authenticate. ''' mock_ret = {'error': {'name': 'UserAuthenticationError', - 'message': 'Authentication failure of type "user" occurred'}} + 'message': 'Authentication failure of type "user" occurred'}} ret = self.clear_funcs.wheel({}) self.assertDictEqual(mock_ret, ret) @@ -219,7 +250,7 @@ def test_publish_user_is_blacklisted(self): Asserts that an AuthorizationError is returned when the user has been blacklisted. ''' mock_ret = {'error': {'name': 'AuthorizationError', - 'message': 'Authorization error occurred.'}} + 'message': 'Authorization error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=True)): self.assertEqual(mock_ret, self.clear_funcs.publish({'user': 'foo', 'fun': 'test.arg'})) @@ -228,7 +259,7 @@ def test_publish_cmd_blacklisted(self): Asserts that an AuthorizationError is returned when the command has been blacklisted. ''' mock_ret = {'error': {'name': 'AuthorizationError', - 'message': 'Authorization error occurred.'}} + 'message': 'Authorization error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=True)): self.assertEqual(mock_ret, self.clear_funcs.publish({'user': 'foo', 'fun': 'test.arg'})) @@ -238,7 +269,7 @@ def test_publish_token_not_authenticated(self): Asserts that an AuthenticationError is returned when the token can't authenticate. ''' mock_ret = {'error': {'name': 'AuthenticationError', - 'message': 'Authentication error occurred.'}} + 'message': 'Authentication error occurred.'}} load = {'user': 'foo', 'fun': 'test.arg', 'tgt': 'test_minion', 'kwargs': {'token': 'asdfasdfasdfasdf'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ @@ -255,7 +286,7 @@ def test_publish_token_authorization_error(self): 'arg': 'bar', 'kwargs': {'token': token}} mock_token = {'token': token, 'eauth': 'foo', 'name': 'test'} mock_ret = {'error': {'name': 'AuthorizationError', - 'message': 'Authorization error occurred.'}} + 'message': 'Authorization error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ @@ -270,7 +301,7 @@ def test_publish_eauth_not_authenticated(self): load = {'user': 'test', 'fun': 'test.arg', 'tgt': 'test_minion', 'kwargs': {'eauth': 'foo'}} mock_ret = {'error': {'name': 'AuthenticationError', - 'message': 'Authentication error occurred.'}} + 'message': 'Authentication error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): self.assertEqual(mock_ret, self.clear_funcs.publish(load)) @@ -283,7 +314,7 @@ def test_publish_eauth_authorization_error(self): load = {'user': 'test', 'fun': 'test.arg', 'tgt': 'test_minion', 'kwargs': {'eauth': 'foo'}, 'arg': 'bar'} mock_ret = {'error': {'name': 'AuthorizationError', - 'message': 'Authorization error occurred.'}} + 'message': 'Authorization error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ @@ -296,7 +327,7 @@ def test_publish_user_not_authenticated(self): ''' load = {'user': 'test', 'fun': 'test.arg', 'tgt': 'test_minion'} mock_ret = {'error': {'name': 'AuthenticationError', - 'message': 'Authentication error occurred.'}} + 'message': 'Authentication error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): self.assertEqual(mock_ret, self.clear_funcs.publish(load)) @@ -309,7 +340,7 @@ def test_publish_user_authenticated_missing_auth_list(self): load = {'user': 'test', 'fun': 'test.arg', 'tgt': 'test_minion', 'kwargs': {'user': 'test'}, 'arg': 'foo'} mock_ret = {'error': {'name': 'AuthenticationError', - 'message': 'Authentication error occurred.'}} + 'message': 'Authentication error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ @@ -324,7 +355,7 @@ def test_publish_user_authorization_error(self): load = {'user': 'test', 'fun': 'test.arg', 'tgt': 'test_minion', 'kwargs': {'user': 'test'}, 'arg': 'foo'} mock_ret = {'error': {'name': 'AuthorizationError', - 'message': 'Authorization error occurred.'}} + 'message': 'Authorization error occurred.'}} with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ From a2951255040172c3901f58ed70ea0c00538dc346 Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Wed, 4 Sep 2019 11:36:25 -0500 Subject: [PATCH 02/19] compose --- docker-compose.yml | 7 +++++++ tests/bbcpu/bbcpu.alias | 16 ++++++++++++++++ tests/bbcpu/bbcpu.lst | 15 +++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 docker-compose.yml create mode 100644 tests/bbcpu/bbcpu.alias create mode 100644 tests/bbcpu/bbcpu.lst diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000000..fdf1610abe1d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.4' +services: + salt-proper-tests: + image: "artprod.dev.bloomberg.com/bb-inf/salt-minion:2018.3.3" + volumes: + - .:/srv/sysca-salt + - ./tests/bbcpu/:/bb/bin/ diff --git a/tests/bbcpu/bbcpu.alias b/tests/bbcpu/bbcpu.alias new file mode 100644 index 000000000000..039c6375816d --- /dev/null +++ b/tests/bbcpu/bbcpu.alias @@ -0,0 +1,16 @@ +a1a thishost 100 +a1b thathost 101 +j13o sltdm-ob-127 19036 +g0mx sltdm-ob-189 24391 +o13j sltdm-ob-618 19161 +g0mz sltdm-rr-005 24393 +o13k sltdm-rr-129 19162 +j13p sltdm-rr-889 19037 +a1c bluehost 102 +a1d redhost 103 +g10c sltpm-ob-035 16918 +n0nl sltpm-ob-394 29137 +n0nk sltpm-ob-749 29136 +n0os sltpm-rr-533 29170 +o10r sltpm-rr-730 17141 +n0or sltpm-rr-941 29169 diff --git a/tests/bbcpu/bbcpu.lst b/tests/bbcpu/bbcpu.lst new file mode 100644 index 000000000000..5cdeb3a2dce7 --- /dev/null +++ b/tests/bbcpu/bbcpu.lst @@ -0,0 +1,15 @@ +thishost 100 lnxdev sn2 sn2e brdev bbcpu ridge bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clsrt me +thathost 101 linux lnxdev sn2 orange sn2b west brdev bbcpu bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clstr me +sltdm-ob-127 19036 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4p nohost obtoff salt bccvm sltdm vkrudysz +sltdm-ob-189 24391 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4k nohost obtoff salt sltdm vkrudysz +sltdm-ob-618 19161 linux orange pacld s2e west s2 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz +sltdm-rr-005 24393 linux pacld s4n s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt sltdm vkrudysz +sltdm-rr-129 19162 linux pacld s2c s2 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz +sltdm-rr-889 19037 linux pacld s4m s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz +bluehost 102 lnxdev sn2 sn2e brdev bbcpu ridge bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clsrt me +redhost 103 linux lnxdev sn2 orange sn2b west brdev bbcpu bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clstr me +sltpm-ob-394 29137 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4k nohost obtoff salt bccvm sltpm apapp +sltpm-ob-749 29136 linux orange pacld s3e west s3 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp +sltpm-rr-533 29170 linux pacld s4m s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp +sltpm-rr-730 17141 linux pacld s2a s2 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt sltpm apapp +sltpm-rr-941 29169 linux pacld s3h s3 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp From 599ce93169189a22ca41fa35b076cb6d1cf80e21 Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Wed, 4 Sep 2019 14:01:12 -0400 Subject: [PATCH 03/19] more stuff --- salt/pillar/environments.py | 2 ++ tests/unit/test_master.py | 60 ++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index 83da3b592100..149bc1092a63 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -168,6 +168,8 @@ def ext_pillar(minion_id, pillar): node = resolve_node(minion_id) + import pdb; pdb.set_trace() + if node is None: return [] diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index 6cda67ca8249..e59e794fcf62 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -15,6 +15,8 @@ ) +# These tests require a working /bb/bin/bbcpu.lst/alias. +# I ran them inside a docker container using the docker-compose.yml in root class AESFuncsTestCase(TestCase): ''' TestCase for salt.master.AESFuncs class @@ -26,12 +28,10 @@ def test__file_envs_no_matching_node(self): 'environments': [ 'word' ] - } + } self.aes_funcs = salt.master.AESFuncs(opts) - res = self.aes_funcs._file_envs({ - "id": "pytest_minion_1" - }) + res = self.aes_funcs._file_envs({"id": "pytest_minion_1"}) self.assertEqual(res, []) def test__file_envs_load_is_none(self): @@ -47,6 +47,57 @@ def test__file_envs_load_is_none(self): res = self.aes_funcs._file_envs() self.assertEqual(res, []) + def test__file_envs_node_is_found(self): + # Default master opts + opts = salt.config.master_config(None) + opts['ext_pillar'] = { + 'environments': [ + 'word' + ] + } + opts['evaporator'] = {} + opts['evaporator']['tenancies'] = [ + {"environment": "sltdm", "global": False}, + {"environment": "salt-native", "global": True}, + {"environment": "salt-water", "global": False}, + ] + + self.aes_funcs = salt.master.AESFuncs(opts) + res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"}) + self.assertEqual(res, {u'environments': ['salt-native', "sltdm"]}) + + def test__file_envs_node_no_environment(self): + # Default master opts + opts = salt.config.master_config(None) + opts['evaporator'] = {} + opts['evaporator']['tenancies'] = [ + {"environment": "sltdm", "global": False}, + {"environment": "salt-native", "global": True}, + {"environment": "salt-water", "global": False}, + ] + + self.aes_funcs = salt.master.AESFuncs(opts) + import pdb; pdb.set_trace() + res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"}) + self.assertEqual(res, {u'environments': ['salt-native', "sltdm"]}) + + def test_master_opts(self): + opts = salt.config.master_config(None) + opts['ext_pillar'] = { + 'environments': [ + 'word' + ] + } + opts['evaporator'] = {} + opts['evaporator']['tenancies'] = [ + {"environment": "sltdm", "global": False}, + {"environment": "salt-native", "global": True}, + {"environment": "salt-water", "global": False}, + ] + + self.aes_funcs = salt.master.AESFuncs(opts) + res = self.aes_funcs._master_opts() + class ClearFuncsTestCase(TestCase): ''' TestCase for salt.master.ClearFuncs class @@ -362,3 +413,4 @@ def test_publish_user_authorization_error(self): patch('salt.utils.master.get_values_of_matching_keys', MagicMock(return_value=['test'])), \ patch('salt.utils.minions.CkMinions.auth_check', MagicMock(return_value=False)): self.assertEqual(mock_ret, self.clear_funcs.publish(load)) + From 16f57946edd3a6ca72509a4a2fd898ec4f0a690b Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Wed, 4 Sep 2019 14:15:16 -0400 Subject: [PATCH 04/19] morre stuff --- salt/fileserver/__init__.py | 1 + tests/unit/test_master.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 9c99fd2a2fbb..ea4b93a26b4f 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -497,6 +497,7 @@ def envs(self, back=None, sources=False): ''' Return the environments for the named backend or all backends ''' + import pdb; pdb.set_trace() back = self.backends(back) ret = set() if sources: diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index e59e794fcf62..c56e73bf50c7 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -77,9 +77,8 @@ def test__file_envs_node_no_environment(self): ] self.aes_funcs = salt.master.AESFuncs(opts) - import pdb; pdb.set_trace() res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"}) - self.assertEqual(res, {u'environments': ['salt-native', "sltdm"]}) + self.assertEqual(res, ["base"]) def test_master_opts(self): opts = salt.config.master_config(None) From 180a21a421c36454e479baf0bb5defb71dc6b38e Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Wed, 4 Sep 2019 14:54:03 -0400 Subject: [PATCH 05/19] more tests --- tests/unit/test_master.py | 40 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index c56e73bf50c7..e1e8776aaba5 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -24,11 +24,9 @@ class AESFuncsTestCase(TestCase): def test__file_envs_no_matching_node(self): # Default master opts opts = salt.config.master_config(None) - opts['ext_pillar'] = { - 'environments': [ - 'word' - ] - } + opts['ext_pillar'] = [ + {'environments': ['word']} + ] self.aes_funcs = salt.master.AESFuncs(opts) res = self.aes_funcs._file_envs({"id": "pytest_minion_1"}) @@ -37,11 +35,9 @@ def test__file_envs_no_matching_node(self): def test__file_envs_load_is_none(self): # Default master opts opts = salt.config.master_config(None) - opts['ext_pillar'] = { - 'environments': [ - 'word' - ] - } + opts['ext_pillar'] = [ + {'environments': ['word']} + ] self.aes_funcs = salt.master.AESFuncs(opts) res = self.aes_funcs._file_envs() @@ -50,11 +46,9 @@ def test__file_envs_load_is_none(self): def test__file_envs_node_is_found(self): # Default master opts opts = salt.config.master_config(None) - opts['ext_pillar'] = { - 'environments': [ - 'word' - ] - } + opts['ext_pillar'] = [ + {'environments': ['word']} + ] opts['evaporator'] = {} opts['evaporator']['tenancies'] = [ {"environment": "sltdm", "global": False}, @@ -82,11 +76,9 @@ def test__file_envs_node_no_environment(self): def test_master_opts(self): opts = salt.config.master_config(None) - opts['ext_pillar'] = { - 'environments': [ - 'word' - ] - } + opts['ext_pillar'] = [ + {'environments': ['word']} + ] opts['evaporator'] = {} opts['evaporator']['tenancies'] = [ {"environment": "sltdm", "global": False}, @@ -95,7 +87,12 @@ def test_master_opts(self): ] self.aes_funcs = salt.master.AESFuncs(opts) - res = self.aes_funcs._master_opts() + + res = self.aes_funcs._master_opts({ + "id": "sltdm-rr-005", + "env_only": True, + }) + self.assertEqual(res, {u'default_top': u'base', u'env_order': [], u'ext_pillar': [{'environments': ['word']}], u'top_file_merging_strategy': u'merge', u'file_roots': {}}) class ClearFuncsTestCase(TestCase): ''' @@ -413,3 +410,4 @@ def test_publish_user_authorization_error(self): patch('salt.utils.minions.CkMinions.auth_check', MagicMock(return_value=False)): self.assertEqual(mock_ret, self.clear_funcs.publish(load)) + From d83c95e067ea9b2a0c55a8122569b8375449d0f9 Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Wed, 4 Sep 2019 15:00:21 -0500 Subject: [PATCH 06/19] more stuff --- salt/fileserver/__init__.py | 3 +-- salt/master.py | 2 +- salt/pillar/environments.py | 2 -- tests/unit/test_master.py | 4 ++-- tests/unit/test_pillar.py | 18 ++++++++++++++++++ tests/unit/utils/test_state.py | 8 ++++++++ 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index ea4b93a26b4f..c7049e2a4855 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -493,11 +493,10 @@ def update_intervals(self, back=None): ret[fsb] = self.servers[fstr]() return ret - def envs(self, back=None, sources=False): + def envs(self, back=None, sources=False, **kwargs): ''' Return the environments for the named backend or all backends ''' - import pdb; pdb.set_trace() back = self.backends(back) ret = set() if sources: diff --git a/salt/master.py b/salt/master.py index 21b6b6f10d9b..7f19cc3d40ae 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1201,7 +1201,7 @@ def _file_envs(self, load=None): if any('environments' in ext for ext in self.opts['ext_pillar']): return self.pillars['environments'](load.get('id'), {}) else: - return self.fs_.envs() + return self.fs_.envs(**load) def __verify_minion(self, id_, token): ''' diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index 149bc1092a63..83da3b592100 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -168,8 +168,6 @@ def ext_pillar(minion_id, pillar): node = resolve_node(minion_id) - import pdb; pdb.set_trace() - if node is None: return [] diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index e1e8776aaba5..83168d2fa545 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -74,7 +74,7 @@ def test__file_envs_node_no_environment(self): res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"}) self.assertEqual(res, ["base"]) - def test_master_opts(self): + def test_master_opts_ext_pillar_environments(self): opts = salt.config.master_config(None) opts['ext_pillar'] = [ {'environments': ['word']} @@ -92,7 +92,7 @@ def test_master_opts(self): "id": "sltdm-rr-005", "env_only": True, }) - self.assertEqual(res, {u'default_top': u'base', u'env_order': [], u'ext_pillar': [{'environments': ['word']}], u'top_file_merging_strategy': u'merge', u'file_roots': {}}) + self.assertEqual(res, {u'default_top': u'base', u'env_order': [], u'ext_pillar': [{'environments': ['word']}], u'top_file_merging_strategy': u'merge', u'file_roots': {u'environments': []},}) class ClearFuncsTestCase(TestCase): ''' diff --git a/tests/unit/test_pillar.py b/tests/unit/test_pillar.py index c878caadb95c..d0796a2e8372 100644 --- a/tests/unit/test_pillar.py +++ b/tests/unit/test_pillar.py @@ -827,3 +827,21 @@ def test_pillar_send_extra_minion_data_from_config(self): 'pillar_override': {}, 'extra_minion_data': {'path_to_add': 'fake_data'}}, dictkey='pillar') + +class Pillar(TestCase): + def test__get_envs(self): + opts = salt.config.master_config(None) + opts['ext_pillar'] = [ + {'environments': ['word']} + ] + + opts['evaporator'] = {} + opts['evaporator']['tenancies'] = [ + {"environment": "sltdm", "global": False}, + {"environment": "salt-native", "global": True}, + {"environment": "salt-water", "global": False}, + ] + + pillar = salt.pillar.Pillar(opts, {}, 'sltdm-rr-005', 'base') + res = pillar._get_envs() + self.assertEqual(res, {u'environments': ['salt-native', 'sltdm']}) diff --git a/tests/unit/utils/test_state.py b/tests/unit/utils/test_state.py index d076e7d00436..d4677a3f1ac5 100644 --- a/tests/unit/utils/test_state.py +++ b/tests/unit/utils/test_state.py @@ -686,3 +686,11 @@ def test_merge_empty_comments(self): res = salt.utils.state.merge_subreturn(m, s) self.assertEqual(res['comment'], [sub_comment_1, sub_comment_2]) self.assertEqual('\n'.join(res['comment']), final_comment) + +class UtilStateGetSlsOptsTestcase(TestCase): + ''' + Test cases for salt.utils.state.merge_subreturn function. + ''' + def test_get_sls_opts(self): + opts = salt.utils.state.get_sls_opts({'orig':'inal'}, {"environments": ['base', 'scuba-diving']}) + self.assertEqual(opts, {u'pillarenv': [u'base', u'scuba-diving'], u'saltenv': [u'base', u'scuba-diving'], u'orig': u'inal'}) From c9ab89311774f903b613d51db0013d4933c61a07 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Wed, 4 Sep 2019 17:18:25 -0400 Subject: [PATCH 07/19] Address comments --- docker-compose.yml | 1 + requirements/dev.txt | 1 - requirements/dev_bloomberg.txt | 15 +++++++++++++++ salt/pillar/environments.py | 6 ++++-- tests/bbcpu/bbcpu.alias | 16 ---------------- tests/bbcpu/bbcpu.lst | 15 --------------- 6 files changed, 20 insertions(+), 34 deletions(-) create mode 100644 requirements/dev_bloomberg.txt delete mode 100644 tests/bbcpu/bbcpu.alias delete mode 100644 tests/bbcpu/bbcpu.lst diff --git a/docker-compose.yml b/docker-compose.yml index fdf1610abe1d..cba5839c7cba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +# This is only for bloomberg, we can make Jenkins run the unit tests inside of a container and remove this later version: '2.4' services: salt-proper-tests: diff --git a/requirements/dev.txt b/requirements/dev.txt index 58ce2e84e555..b367c2c76615 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,7 +5,6 @@ SaltPyLint>=v2017.3.6 pytest>=3.5.0 git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 -bloomberg.hostinfo # httpretty Needs to be here for now even though it's a dependency of boto. # A pip install on a fresh system will decide to target httpretty 0.8.10 to diff --git a/requirements/dev_bloomberg.txt b/requirements/dev_bloomberg.txt new file mode 100644 index 000000000000..58ce2e84e555 --- /dev/null +++ b/requirements/dev_bloomberg.txt @@ -0,0 +1,15 @@ +-r base.txt + +mock>=2.0.0 +SaltPyLint>=v2017.3.6 +pytest>=3.5.0 +git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt +testinfra>=1.7.0 +bloomberg.hostinfo + +# httpretty Needs to be here for now even though it's a dependency of boto. +# A pip install on a fresh system will decide to target httpretty 0.8.10 to +# satisfy other requirements, and httpretty 0.8.10 has bugs in setup.py that +# prevent it from being successfully installed (at least on Python 3.4). +httpretty; python_version >= '3.4' +pylint==1.6.5 diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index 83da3b592100..a0c49afa6ed5 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -168,11 +168,13 @@ def ext_pillar(minion_id, pillar): node = resolve_node(minion_id) + global_tenancy_groups = global_tenancy_groups_set() + if node is None: - return [] + return {'environments': list(global_tenancy_groups)} # any matching tenancy_group is a 1 to 1 association with environment # we use an IndexedSet to ensure global roots are always highest priority - environments = IndexedSet(global_tenancy_groups_set() | ( node.groups_set() & tenancy_groups_set())) + environments = IndexedSet(global_tenancy_groups | ( node.groups_set() & tenancy_groups_set())) return {'environments': list(environments)} diff --git a/tests/bbcpu/bbcpu.alias b/tests/bbcpu/bbcpu.alias deleted file mode 100644 index 039c6375816d..000000000000 --- a/tests/bbcpu/bbcpu.alias +++ /dev/null @@ -1,16 +0,0 @@ -a1a thishost 100 -a1b thathost 101 -j13o sltdm-ob-127 19036 -g0mx sltdm-ob-189 24391 -o13j sltdm-ob-618 19161 -g0mz sltdm-rr-005 24393 -o13k sltdm-rr-129 19162 -j13p sltdm-rr-889 19037 -a1c bluehost 102 -a1d redhost 103 -g10c sltpm-ob-035 16918 -n0nl sltpm-ob-394 29137 -n0nk sltpm-ob-749 29136 -n0os sltpm-rr-533 29170 -o10r sltpm-rr-730 17141 -n0or sltpm-rr-941 29169 diff --git a/tests/bbcpu/bbcpu.lst b/tests/bbcpu/bbcpu.lst deleted file mode 100644 index 5cdeb3a2dce7..000000000000 --- a/tests/bbcpu/bbcpu.lst +++ /dev/null @@ -1,15 +0,0 @@ -thishost 100 lnxdev sn2 sn2e brdev bbcpu ridge bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clsrt me -thathost 101 linux lnxdev sn2 orange sn2b west brdev bbcpu bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clstr me -sltdm-ob-127 19036 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4p nohost obtoff salt bccvm sltdm vkrudysz -sltdm-ob-189 24391 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4k nohost obtoff salt sltdm vkrudysz -sltdm-ob-618 19161 linux orange pacld s2e west s2 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz -sltdm-rr-005 24393 linux pacld s4n s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt sltdm vkrudysz -sltdm-rr-129 19162 linux pacld s2c s2 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz -sltdm-rr-889 19037 linux pacld s4m s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltdm vkrudysz -bluehost 102 lnxdev sn2 sn2e brdev bbcpu ridge bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clsrt me -redhost 103 linux lnxdev sn2 orange sn2b west brdev bbcpu bass dvmn mini nano lnxsa bpkg rtcpux pdcld zjobdv prcchk zjob notrst bbrc ctm20 ctmna nohost obtoff bccvm utc prnt clstr me -sltpm-ob-394 29137 linux orange pacld west s4 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc s4k nohost obtoff salt bccvm sltpm apapp -sltpm-ob-749 29136 linux orange pacld s3e west s3 bbcpu bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp -sltpm-rr-533 29170 linux pacld s4m s4 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp -sltpm-rr-730 17141 linux pacld s2a s2 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt sltpm apapp -sltpm-rr-941 29169 linux pacld s3h s3 bbcpu ridge bass mini nano lnxsa bpkg rtcpux prcchk notrst bbrc nohost obtoff salt bccvm sltpm apapp From f6526567b340ad396cdc7c95179cef35315e4d23 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Thu, 5 Sep 2019 17:05:37 -0400 Subject: [PATCH 08/19] This should be code change.. tests coming in hot --- salt/pillar/environments.py | 45 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index a0c49afa6ed5..7660fd403505 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -104,13 +104,11 @@ def __get_by_node(self, node): def tenancy_groups_set(): groups = IndexedSet() - # assume config is a sane schema - # scalar = tag, dict = 'tag' key is tag for tenancy in __opts__['evaporator']['tenancies']: - if isinstance(tenancy, dict): - groups.add(tenancy['environment']) - else: - groups.add(tenancy) + try: + groups.add(tenancy['environment']['groups']) + except KeyError: + pass return groups @@ -118,17 +116,19 @@ def tenancy_groups_set(): def global_tenancy_groups_set(): groups = IndexedSet() - # assume config is a sane schema - # scalar = tag, dict = 'tag' key is tag for tenancy in __opts__['evaporator']['tenancies']: - if isinstance(tenancy, dict) and tenancy.get('global'): - groups.add(tenancy['environment']) + try: + if tenancy['global']: + groups.add(tenancy['environment']['groups']) + except KeyError: + pass return groups # first try node-id if it exists in grains, then try the minion_id def resolve_node(minion_id): + if __grains__.get('bb', {}).get('node-id'): node_id = __grains__['bb']['node-id'] try: @@ -145,6 +145,18 @@ def resolve_node(minion_id): # if we've gotten this far its an unknown node return None +def stage_envs(stage, envs) + """ + Takes in an iterable of env names and prepends a stage to the beginning. + Example: + env = [salt-core, natm] + stage = 'sn2' + + >> {'environments': ['salt-core-sn2', 'natm-sn2']} + """ + staged_envs = ['{}-{}'.format(env, stage) for env in envs] + return {'environments': staged_envs} + # the goal here is to # 1. if a local top for the given repo exists, include it @@ -161,20 +173,19 @@ def ext_pillar(minion_id, pillar): :return: a dictionary which is included by the salt master in the pillars returned to the minion """ + global_tenancy_groups = global_tenancy_groups_set() + # hostinfo resolving a node that is None will throw an error - # Instead lets just return empty array if not minion_id: - return [] + return stage_envs('nostage', global_tenancy_groups) node = resolve_node(minion_id) - - global_tenancy_groups = global_tenancy_groups_set() - + if node is None: - return {'environments': list(global_tenancy_groups)} + return stage_envs('nostage', global_tenancy_groups) # any matching tenancy_group is a 1 to 1 association with environment # we use an IndexedSet to ensure global roots are always highest priority environments = IndexedSet(global_tenancy_groups | ( node.groups_set() & tenancy_groups_set())) - return {'environments': list(environments)} + return stage_envs(node.stage, environments) From b84aff91c29ab735f7d4aadff7fcf19302372a67 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 11:23:27 -0400 Subject: [PATCH 09/19] run upstream unit tests --- Jenkinsfile | 44 +++++++++++++------------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 150e920a5538..d81b3f9e480a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,49 +5,31 @@ pipeline { PYPI_CREDENTIAL = credentials('salt_jenkins_ad_user_pass_escaped') } stages { - stage('build') { - when { - changeRequest() - } + stage('Build') { + when {changeRequest()} steps { - sh 'echo ========================' - sh 'echo running Build Stage' - sh 'whoami' - sh 'python --version' - sh 'hostname' - sh 'pwd' - sh 'printenv' sh 'bash ./build/build.sh -b $CHANGE_ID' } } - stage('test') { - when { - changeRequest() - } + stage('Run Upstream Salt Unit Tests') { + when {changeRequest()} steps { - sh 'echo ========================' - sh 'echo running Test Stage' + sh ''' + tox -e pylint-tests --notest + source .tox/pylint-tests/bin/activate + ./tests/runtests.py -n unit + ''' } } - stage('deploy to dev pypi') { - when { - changeRequest() - } + stage('Deploy to dev pypi') { + when {changeRequest()} steps { - sh 'echo =========================' - sh 'echo running Deploy to dev Stage' sh 'bash ./build/build.sh -b $CHANGE_ID -k -s -u' } } - stage('deploy to ose pypi') { - when { - anyOf { - branch 'v2018.3.3-ca' - } - } + stage('Deploy to ose pypi') { + when {anyOf {branch 'v2018.3.3-ca'}} steps { - sh 'echo =========================' - sh 'echo running Deploy to ose pypi Stage' sh 'bash ./build/build.sh -u -p -t $BBGH_TOKEN_PSW' } } From d5858953fc2f3b12046172812ab1cebae4324959 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 11:30:14 -0400 Subject: [PATCH 10/19] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d81b3f9e480a..fe2b52350a2a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { when {changeRequest()} steps { sh ''' - tox -e pylint-tests --notest + /usr/local/bin/tox -e pylint-tests --notest source .tox/pylint-tests/bin/activate ./tests/runtests.py -n unit ''' From 27b2653c41defa0b93127cf938beb75988a081fa Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 11:37:17 -0400 Subject: [PATCH 11/19] run upstream unit tests --- Jenkinsfile | 44 +++++++++++++------------------------------- tox.ini | 1 + 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 150e920a5538..d81b3f9e480a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,49 +5,31 @@ pipeline { PYPI_CREDENTIAL = credentials('salt_jenkins_ad_user_pass_escaped') } stages { - stage('build') { - when { - changeRequest() - } + stage('Build') { + when {changeRequest()} steps { - sh 'echo ========================' - sh 'echo running Build Stage' - sh 'whoami' - sh 'python --version' - sh 'hostname' - sh 'pwd' - sh 'printenv' sh 'bash ./build/build.sh -b $CHANGE_ID' } } - stage('test') { - when { - changeRequest() - } + stage('Run Upstream Salt Unit Tests') { + when {changeRequest()} steps { - sh 'echo ========================' - sh 'echo running Test Stage' + sh ''' + tox -e pylint-tests --notest + source .tox/pylint-tests/bin/activate + ./tests/runtests.py -n unit + ''' } } - stage('deploy to dev pypi') { - when { - changeRequest() - } + stage('Deploy to dev pypi') { + when {changeRequest()} steps { - sh 'echo =========================' - sh 'echo running Deploy to dev Stage' sh 'bash ./build/build.sh -b $CHANGE_ID -k -s -u' } } - stage('deploy to ose pypi') { - when { - anyOf { - branch 'v2018.3.3-ca' - } - } + stage('Deploy to ose pypi') { + when {anyOf {branch 'v2018.3.3-ca'}} steps { - sh 'echo =========================' - sh 'echo running Deploy to ose pypi Stage' sh 'bash ./build/build.sh -u -p -t $BBGH_TOKEN_PSW' } } diff --git a/tox.ini b/tox.ini index ea50d3d28452..9d4a079963ff 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ skip_missing_interpreters = True skipsdist = True [testenv] +install_command = pip install --ignore-installed --no-cache-dir --trusted-host artifactory.inf.bloomberg.com --index-url=http://artifactory.inf.bloomberg.com/artifactory/api/pypi/bloomberg-pypi-ose/simple/ {opts} {packages} deps = -Ur{toxinidir}/requirements/tests.txt commands = pytest --rootdir {toxinidir} {posargs} passenv = LANG HOME From b552ca2921742885226636715e2467909403c743 Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Fri, 6 Sep 2019 12:12:17 -0400 Subject: [PATCH 12/19] Only edit bb reqs --- requirements/dev.txt | 2 +- requirements/dev_bloomberg.txt | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index c18618b03f1c..b367c2c76615 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,7 +3,7 @@ mock>=2.0.0 SaltPyLint>=v2017.3.6 pytest>=3.5.0 -git+https://bbgithub.dev.bloomberg.com/saltstack/pytest-salt.git@master#egg=pytest-salt +git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 # httpretty Needs to be here for now even though it's a dependency of boto. diff --git a/requirements/dev_bloomberg.txt b/requirements/dev_bloomberg.txt index 58ce2e84e555..1a057711cf25 100644 --- a/requirements/dev_bloomberg.txt +++ b/requirements/dev_bloomberg.txt @@ -3,7 +3,7 @@ mock>=2.0.0 SaltPyLint>=v2017.3.6 pytest>=3.5.0 -git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt +git+https://e904693b4b9f451885e6a05696ab0a60c3be959c:bbgithub.dev.bloomberg.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 bloomberg.hostinfo diff --git a/tox.ini b/tox.ini index 9d4a079963ff..d3ae74ba9d66 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skipsdist = True [testenv] install_command = pip install --ignore-installed --no-cache-dir --trusted-host artifactory.inf.bloomberg.com --index-url=http://artifactory.inf.bloomberg.com/artifactory/api/pypi/bloomberg-pypi-ose/simple/ {opts} {packages} -deps = -Ur{toxinidir}/requirements/tests.txt +deps = -Ur{toxinidir}/requirements/dev_bloomberg.txt commands = pytest --rootdir {toxinidir} {posargs} passenv = LANG HOME sitepackages = True From 2145473681f976f623f034999d5af65e93cff1bc Mon Sep 17 00:00:00 2001 From: nsmith269 Date: Fri, 6 Sep 2019 11:18:26 -0500 Subject: [PATCH 13/19] run custom unit tests in jenkins --- Jenkinsfile | 4 +++- requirements/dev_bloomberg.txt | 2 +- tox.ini | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4717299dbbcd..03b8327d1d91 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,9 @@ pipeline { sh ''' /usr/local/bin/tox -e pylint-tests --notest source .tox/pylint-tests/bin/activate - python tests/runtests.py --unit + ./tests/runtests.py -n unit.test_master.AESFuncsTestCase + ./tests/runtests.py -n unit.test_pillar.Pillar + ./tests/runtests.py -n unit.test_state.UtilStateGetSlsOptsTestcase ''' } } diff --git a/requirements/dev_bloomberg.txt b/requirements/dev_bloomberg.txt index 1a057711cf25..6382c9b2eda0 100644 --- a/requirements/dev_bloomberg.txt +++ b/requirements/dev_bloomberg.txt @@ -3,7 +3,7 @@ mock>=2.0.0 SaltPyLint>=v2017.3.6 pytest>=3.5.0 -git+https://e904693b4b9f451885e6a05696ab0a60c3be959c:bbgithub.dev.bloomberg.com/saltstack/pytest-salt.git@master#egg=pytest-salt +git+https://e904693b4b9f451885e6a05696ab0a60c3be959c:@bbgithub.dev.bloomberg.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 bloomberg.hostinfo diff --git a/tox.ini b/tox.ini index d3ae74ba9d66..5ac7bc01b084 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ sitepackages = False [testenv:pylint-tests] basepython = python2.7 -deps = -r{toxinidir}/requirements/dev.txt +deps = -r{toxinidir}/requirements/dev_bloomberg.txt commands = pylint --version pylint --rcfile=.testing.pylintrc --disable=I,W0232,E1002,W1307,C0411,C0413,W8410,str-format-in-logging {posargs:tests/} From 22e2a26499e65607bd30dc143e40e3928ce5b950 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 12:34:38 -0400 Subject: [PATCH 14/19] run the unit tests inside a container --- Jenkinsfile | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 03b8327d1d91..a58c027b0d41 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,20 @@ +def unique_container_name = "unit-tests-${env.BUILD_ID}${env.JOB_NAME.replace("/", "-")}" +def image_name = "artprod.dev.bloomberg.com/bb-inf/salt-minion:2018.3.3" + pipeline { agent { label 'syscore-salt'} environment { BBGH_TOKEN = credentials('bbgithub_token') PYPI_CREDENTIAL = credentials('salt_jenkins_ad_user_pass_escaped') } + options { + ansiColor('xterm') + // Currently builds take an hour and 30 minutes. + // If every executor is used, this will give enough time for the queue to pass and this build to run + // timeout(time: 3, unit: 'HOURS') + // Keep up to 10 builds (artifacts/console output) from master and each branch retains up to 5 builds of that specific branch + buildDiscarder(logRotator(numToKeepStr: env.BRANCH_NAME == 'master' ? '10' : '5')) + } stages { stage('Build') { when {changeRequest()} @@ -14,13 +25,33 @@ pipeline { stage('Run Upstream Salt Unit Tests') { when {changeRequest()} steps { - sh ''' - /usr/local/bin/tox -e pylint-tests --notest - source .tox/pylint-tests/bin/activate - ./tests/runtests.py -n unit.test_master.AESFuncsTestCase - ./tests/runtests.py -n unit.test_pillar.Pillar - ./tests/runtests.py -n unit.test_state.UtilStateGetSlsOptsTestcase - ''' + script { + docker.withRegistry('https://artprod.dev.bloomberg.com', 'syscore_jenkins_docker_jwt_tuple') { + sh "docker pull ${image_name}" + } + // Jenkins docker integration is confusing wrapper and doesn't seem to work as expected + sh "docker run --name ${unique_container_name} -d -v `pwd`:`pwd` -w `pwd` ${image_name}" + sh "docker exec ${unique_container_name} tox -e pylint-tests --notest" + sh "docker exec ${unique_container_name} source .tox/pylint-tests/bin/activate && \ + ./tests/runtests.py -n unit.test_master.AESFuncsTestCase && \ + ./tests/runtests.py -n unit.test_pillar.Pillar && \ + ./tests/runtests.py -n unit.test_state.UtilStateGetSlsOptsTestcase" + } + } + post { + cleanup { + node("syscore-salt") { + script { + deleteDir() /* clean up our workspace */ + + try { + sh "docker stop ${unique_container_name}" + } catch(Exception e) { + // continue + } + } + } + } } } stage('Deploy to dev pypi') { From 1b425f3c4f104690802a95a873278b40e7569dee Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 13:10:24 -0400 Subject: [PATCH 15/19] Only run the custom subset for now --- Jenkinsfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a58c027b0d41..0a22bb5bfddb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,16 +26,14 @@ pipeline { when {changeRequest()} steps { script { + // We are running inside a container so we can have the bbcpu.lst/alias docker.withRegistry('https://artprod.dev.bloomberg.com', 'syscore_jenkins_docker_jwt_tuple') { sh "docker pull ${image_name}" } // Jenkins docker integration is confusing wrapper and doesn't seem to work as expected sh "docker run --name ${unique_container_name} -d -v `pwd`:`pwd` -w `pwd` ${image_name}" - sh "docker exec ${unique_container_name} tox -e pylint-tests --notest" - sh "docker exec ${unique_container_name} source .tox/pylint-tests/bin/activate && \ - ./tests/runtests.py -n unit.test_master.AESFuncsTestCase && \ - ./tests/runtests.py -n unit.test_pillar.Pillar && \ - ./tests/runtests.py -n unit.test_state.UtilStateGetSlsOptsTestcase" + sh "docker exec ${unique_container_name} docker exec test pip install -r requirements/dev_bloomberg.txt" + sh "docker exec ${unique_container_name} ./tests/runtests.py -n unit.test_master.AESFuncsTestCase -n unit.test_pillar.Pillar -n unit.utils.test_state.UtilStateGetSlsOptsTestcase" } } post { From 8abdd2ac3932ca368a2bf714d4d77bb249f1dfef Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Fri, 6 Sep 2019 13:28:45 -0400 Subject: [PATCH 16/19] this will run the tests i made altho now virtual is breaking... --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0a22bb5bfddb..08c9d2d53f20 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { } // Jenkins docker integration is confusing wrapper and doesn't seem to work as expected sh "docker run --name ${unique_container_name} -d -v `pwd`:`pwd` -w `pwd` ${image_name}" - sh "docker exec ${unique_container_name} docker exec test pip install -r requirements/dev_bloomberg.txt" + sh "docker exec ${unique_container_name} pip install -r requirements/dev_bloomberg.txt" sh "docker exec ${unique_container_name} ./tests/runtests.py -n unit.test_master.AESFuncsTestCase -n unit.test_pillar.Pillar -n unit.utils.test_state.UtilStateGetSlsOptsTestcase" } } From bec375b0916c07fa219cd8afa33d9b287c96580e Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Mon, 9 Sep 2019 15:13:14 -0400 Subject: [PATCH 17/19] verbose mode for the win --- salt/pillar/environments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pillar/environments.py b/salt/pillar/environments.py index 7660fd403505..8591b56f1194 100644 --- a/salt/pillar/environments.py +++ b/salt/pillar/environments.py @@ -145,7 +145,7 @@ def resolve_node(minion_id): # if we've gotten this far its an unknown node return None -def stage_envs(stage, envs) +def stage_envs(stage, envs): """ Takes in an iterable of env names and prepends a stage to the beginning. Example: From 627a561a76545d0eae9e82b1ba37d8cfde96c2fe Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Mon, 9 Sep 2019 15:31:34 -0400 Subject: [PATCH 18/19] switching branch quick --- tests/unit/test_master.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index 83168d2fa545..6cc010cb05b7 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -35,6 +35,7 @@ def test__file_envs_no_matching_node(self): def test__file_envs_load_is_none(self): # Default master opts opts = salt.config.master_config(None) + opts['evaporator'] = {'tenancies']: [{'name': 'salt-core', 'groups': ['salt'], 'global': True}]} opts['ext_pillar'] = [ {'environments': ['word']} ] @@ -46,6 +47,7 @@ def test__file_envs_load_is_none(self): def test__file_envs_node_is_found(self): # Default master opts opts = salt.config.master_config(None) + opts['evaporator'] = {'tenancies']: [{'name': 'salt-core', 'groups': ['salt'], 'global': True}]} opts['ext_pillar'] = [ {'environments': ['word']} ] From 082fcebc4b93f63efe11e0c204946948149c72d4 Mon Sep 17 00:00:00 2001 From: Nick Smith Date: Mon, 9 Sep 2019 15:57:58 -0400 Subject: [PATCH 19/19] add correct evap options --- tests/unit/test_master.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index 6cc010cb05b7..0c6067aebbab 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -35,7 +35,7 @@ def test__file_envs_no_matching_node(self): def test__file_envs_load_is_none(self): # Default master opts opts = salt.config.master_config(None) - opts['evaporator'] = {'tenancies']: [{'name': 'salt-core', 'groups': ['salt'], 'global': True}]} + opts['evaporator'] = {'tenancies': [{'name': 'salt-core', 'groups': ['salt'], 'global': True}]} opts['ext_pillar'] = [ {'environments': ['word']} ] @@ -47,16 +47,16 @@ def test__file_envs_load_is_none(self): def test__file_envs_node_is_found(self): # Default master opts opts = salt.config.master_config(None) - opts['evaporator'] = {'tenancies']: [{'name': 'salt-core', 'groups': ['salt'], 'global': True}]} opts['ext_pillar'] = [ {'environments': ['word']} ] - opts['evaporator'] = {} - opts['evaporator']['tenancies'] = [ - {"environment": "sltdm", "global": False}, - {"environment": "salt-native", "global": True}, - {"environment": "salt-water", "global": False}, - ] + opts['evaporator'] = { + 'tenancies': [ + {"name": "sltdm", "groups": ["salt"], "global": False}, + {"name": "salt-native", "groups": ["salt"], "global": True}, + {"name": "salt-water", "groups": ["salt"], "global": False}, + ] + } self.aes_funcs = salt.master.AESFuncs(opts) res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"}) @@ -65,12 +65,13 @@ def test__file_envs_node_is_found(self): def test__file_envs_node_no_environment(self): # Default master opts opts = salt.config.master_config(None) - opts['evaporator'] = {} - opts['evaporator']['tenancies'] = [ - {"environment": "sltdm", "global": False}, - {"environment": "salt-native", "global": True}, - {"environment": "salt-water", "global": False}, - ] + opts['evaporator'] = { + 'tenancies': [ + {"name": "sltdm", "groups": ["salt"], "global": False}, + {"name": "salt-native", "groups": ["salt"], "global": True}, + {"name": "salt-water", "groups": ["salt"], "global": False}, + ] + } self.aes_funcs = salt.master.AESFuncs(opts) res = self.aes_funcs._file_envs({"id": "sltdm-rr-005"})