Merge branch 'v2018.3.3-ca' into RDTISALT-3200-tests

sbrennan4 · Sep 10, 2019 · 1d4db20 · 1d4db20
2 parents 082fceb + 097d93a
commit 1d4db20
Show file tree

Hide file tree

Showing 73 changed files with 901 additions and 646 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -31,15 +31,21 @@ pipeline {
                         sh "docker pull ${image_name}"
                     }
                     // Jenkins docker integration is confusing wrapper and doesn't seem to work as expected
-                    sh "docker run --name ${unique_container_name} -d -v `pwd`:`pwd` -w `pwd` ${image_name}"
+                    sh "docker run --name ${unique_container_name} -d -v `pwd`:`pwd`:ro -w `pwd` ${image_name}"
                     sh "docker exec ${unique_container_name} pip install -r requirements/dev_bloomberg.txt"
-                    sh "docker exec ${unique_container_name} ./tests/runtests.py -n unit.test_master.AESFuncsTestCase -n unit.test_pillar.Pillar -n unit.utils.test_state.UtilStateGetSlsOptsTestcase"
+
+                    // Run whatever tests you want here for now. All tests takes like an hour.
+                    // If you write custom tests, add them here so we are sure they continue passing
+                    sh "docker exec ${unique_container_name} ./tests/runtests.py --unit -v"
+
+                    // Whatever is failing we can skip with
+                    // @expectedFailure #bb test was failing when ran in Jenkins
                 }
             } 
             post {
                 cleanup {
                     node("syscore-salt") {
-                        script {
+                        script {                            
                             deleteDir() /* clean up our workspace */
 
                             try {

diff --git a/doc/man/salt.7 b/doc/man/salt.7
@@ -379790,7 +379790,7 @@ from __future__ import absolute_import
 
 # Import Salt Testing Libs
 from tests.support.mixins import LoaderModuleMockMixin
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch,
     MagicMock,
@@ -379860,7 +379860,7 @@ a separate implementation which has additional functionality.
 .sp
 .nf
 .ft C
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch
     mock_open,
@@ -379919,7 +379919,7 @@ those cases, you can pass \fBread_data\fP as a dictionary:
 .ft C
 import textwrap
 
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch
     mock_open,
@@ -380012,7 +380012,7 @@ Instead of a string, an exception can also be used as the \fBread_data\fP:
 .ft C
 import errno
 
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch
     mock_open,
@@ -380054,7 +380054,7 @@ and produce a mocked filehandle with the specified contents. For example:
 import errno
 import textwrap
 
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch
     mock_open,
@@ -380173,7 +380173,7 @@ available via its \fBkwargs\fP attribute.
 .sp
 .nf
 .ft C
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 from tests.support.mock import (
     patch
     mock_open,
@@ -380336,7 +380336,7 @@ Most commonly, the following imports are necessary to create a unit test:
 .sp
 .nf
 .ft C
-from tests.support.unit import TestCase, skipIf
+from tests.support.unit import TestCase, skipIf, expectedFailure
 .ft P
 .fi
 .UNINDENT
@@ -380590,7 +380590,7 @@ import salt.modules.linux_sysictl as linux_sysctl
 
 # Import Salt Testing Libs
 from tests.support.mixins import LoaderModuleMockMixin
-from tests.support.unit import skipIf, TestCase
+from tests.support.unit import skipIf, TestCase, expectedFailure
 from tests.support.mock import (
     MagicMock,
     patch,
@@ -380699,7 +380699,7 @@ from salt.exceptions import CommandExecutionError
 
 # Import Salt Testing Libs
 from tests.support.mixins import LoaderModuleMockMixin
-from tests.support.unit import skipIf, TestCase
+from tests.support.unit import skipIf, TestCase, expectedFailure
 from tests.support.mock import (
     MagicMock,
     patch,

diff --git a/doc/topics/development/tests/unit.rst b/doc/topics/development/tests/unit.rst
@@ -132,7 +132,7 @@ Consider this more extensive example from
 
    # Import Salt Testing Libs
    from tests.support.mixins import LoaderModuleMockMixin
-   from tests.support.unit import TestCase, skipIf
+   from tests.support.unit import TestCase, skipIf, expectedFailure
    from tests.support.mock import (
        patch,
        MagicMock,
@@ -196,7 +196,7 @@ a separate implementation which has additional functionality.
 
 .. code-block:: python
 
-    from tests.support.unit import TestCase, skipIf
+    from tests.support.unit import TestCase, skipIf, expectedFailure
     from tests.support.mock import (
         patch
         mock_open,
@@ -251,7 +251,7 @@ those cases, you can pass ``read_data`` as a dictionary:
 
     import textwrap
 
-    from tests.support.unit import TestCase, skipIf
+    from tests.support.unit import TestCase, skipIf, expectedFailure
     from tests.support.mock import (
         patch
         mock_open,
@@ -324,7 +324,7 @@ Instead of a string, an exception can also be used as the ``read_data``:
 
     import errno
 
-    from tests.support.unit import TestCase, skipIf
+    from tests.support.unit import TestCase, skipIf, expectedFailure
     from tests.support.mock import (
         patch
         mock_open,
@@ -362,7 +362,7 @@ and produce a mocked filehandle with the specified contents. For example:
     import errno
     import textwrap
 
-    from tests.support.unit import TestCase, skipIf
+    from tests.support.unit import TestCase, skipIf, expectedFailure
     from tests.support.mock import (
         patch
         mock_open,
@@ -463,7 +463,7 @@ several useful attributes:
 
   .. code-block:: python
 
-      from tests.support.unit import TestCase, skipIf
+      from tests.support.unit import TestCase, skipIf, expectedFailure
       from tests.support.mock import (
           patch
           mock_open,
@@ -607,7 +607,7 @@ Most commonly, the following imports are necessary to create a unit test:
 
 .. code-block:: python
 
-    from tests.support.unit import TestCase, skipIf
+    from tests.support.unit import TestCase, skipIf, expectedFailure
 
 If you need mock support to your tests, please also import:
 
@@ -824,7 +824,7 @@ will also redefine the ``__salt__`` dictionary such that it only contains
 
     # Import Salt Testing Libs
     from tests.support.mixins import LoaderModuleMockMixin
-    from tests.support.unit import skipIf, TestCase
+    from tests.support.unit import skipIf, TestCase, expectedFailure
     from tests.support.mock import (
         MagicMock,
         patch,
@@ -926,7 +926,7 @@ with.
 
     # Import Salt Testing Libs
     from tests.support.mixins import LoaderModuleMockMixin
-    from tests.support.unit import skipIf, TestCase
+    from tests.support.unit import skipIf, TestCase, expectedFailure
     from tests.support.mock import (
         MagicMock,
         patch,

diff --git a/requirements/dev_bloomberg.txt b/requirements/dev_bloomberg.txt
@@ -1,5 +1,4 @@
 -r base.txt
-
 mock>=2.0.0
 SaltPyLint>=v2017.3.6
 pytest>=3.5.0

diff --git a/salt/cache/__init__.py b/salt/cache/__init__.py
@@ -308,6 +308,14 @@ def query(self, *args, **kwargs):
         query = '{0}.query'.format(self.driver)
         return self.modules[query](*args, **kwargs)
 
+    def maintenance(self, *args, **kwargs):
+        '''
+        if backend is configured for such, trigger maintenance
+        '''
+        maintenance = '{0}.maintenance'.format(self.driver)
+        if maintenance in self.modules:
+            return self.modules[maintenance](*args, **kwargs)
+
 class MemCache(Cache):
     '''
     Short-lived in-memory cache store keeping values on time and/or size (count)

diff --git a/salt/cache/pgjsonb.py b/salt/cache/pgjsonb.py
@@ -168,8 +168,7 @@ def store(bank, key, data, expires=None):
         with _exec_pg(autocommit=False) as cur:
             cur.execute(store_sql, params)
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not store cache with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not store cache with postgres cache: {}'.format(err))
 
 
 def flush(bank, key=None):
@@ -188,8 +187,7 @@ def flush(bank, key=None):
         with _exec_pg(autocommit=False) as cur:
             cur.execute(del_sql, params)
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not flush cache with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not flush cache with postgres cache: {}'.format(err))
 
 
 def fetch(bank, key):
@@ -205,8 +203,7 @@ def fetch(bank, key):
                 return data[0]
             return {}
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not fetch cache with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not fetch cache with postgres cache: {}'.format(err))
 
 
 def list(bank):
@@ -222,8 +219,7 @@ def list(bank):
             tuples = cur.fetchall()
             return [x[0] for x in tuples]
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not list cache with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not list cache with postgres cache: {}'.format(err))
 
 
 def contains(bank, key):
@@ -243,8 +239,7 @@ def contains(bank, key):
                 return False
             return False
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not run contains with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not run contains with postgres cache: {}'.format(err))
 
 
 def updated(bank, key):
@@ -261,8 +256,7 @@ def updated(bank, key):
                 return time.mktime(data[0].timetuple())
             return None
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not run updated with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not run updated with postgres cache: {}'.format(err))
 
 
 def clean_expired(bank):
@@ -276,10 +270,20 @@ def clean_expired(bank):
         with _exec_pg(autocommit=False) as cur:
             cur.execute(expire_sql, (bank,))
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not clean up expired tokens with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not clean up expired tokens with postgres cache: {}'.format(err))
 
 
+def maintenance():
+    '''
+    if any maintenance is required, do it here
+    for pg, this means refresh the materialized view
+    '''
+    try:
+        with _exec_pg(autocommit=True) as cur:
+            return cur.execute('REFRESH MATERIALIZED VIEW CONCURRENTLY "cache_grains_ipv4_view"')
+    except salt.exceptions.SaltMasterError as err:
+        raise salt.exceptions.SaltCacheError('Could not execute cache with postgres cache: {}'.format(err))
+
 def query(sql, bind=None, autocommit=True):
     '''
     execute a sql/bind and return results
@@ -290,5 +294,4 @@ def query(sql, bind=None, autocommit=True):
             cur.execute(sql, bind)
             return cur.fetchall()
     except salt.exceptions.SaltMasterError as err:
-        raise salt.exceptions.SaltCacheError(
-            'Could not fetch cache with postgres cache: {}'.format(err))
+        raise salt.exceptions.SaltCacheError('Could not fetch cache with postgres cache: {}'.format(err))
diff --git a/salt/client/mixins.py b/salt/client/mixins.py
@@ -469,7 +469,7 @@ class AsyncClientMixin(object):
     client = None
     tag_prefix = None
 
-    def _proc_function(self, fun, low, user, tag, jid, daemonize=True):
+    def _proc_function(self, fun, low, user, tag, jid, daemonize=True, print_event=True):
         '''
         Run this method in a multiprocess target to execute the function in a
         multiprocess and fire the return data on the event bus
@@ -488,7 +488,7 @@ def _proc_function(self, fun, low, user, tag, jid, daemonize=True):
         low['__user__'] = user
         low['__tag__'] = tag
 
-        return self.low(fun, low, full_return=False)
+        return self.low(fun, low, full_return=False, print_event=print_event)
 
     def cmd_async(self, low):
         '''