From 1eef0f2a003ebc76b6070f3d23585aede1203270 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 18 Sep 2019 16:44:40 -0700 Subject: [PATCH 1/4] Porting PR #49837 to 2019.2.1 --- salt/states/file.py | 27 ++++++++++++++++--- salt/utils/stringutils.py | 40 ++++++++++++++++------------ tests/unit/utils/test_stringutils.py | 15 +++++++++++ 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 4e451c58f812..11f4411ae423 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3679,8 +3679,8 @@ def recurse(name, :ref:`backup_mode documentation ` for more details. include_pat - When copying, include only this pattern from the source. Default - is glob match; if prefixed with 'E@', then regexp match. + When copying, include only this pattern, or list of patterns, from the + source. Default is glob match; if prefixed with 'E@', then regexp match. Example: .. code-block:: text @@ -3690,9 +3690,19 @@ def recurse(name, - include_pat: E@hello :: regexp matches 'otherhello', 'hello01' ... + .. versionchanged:: Neon + + List patterns are now supported + + .. code-block:: text + + - include_pat: + - hello01 + - hello02 + exclude_pat - Exclude this pattern from the source when copying. If both - `include_pat` and `exclude_pat` are supplied, then it will apply + Exclude this pattern, or list of patterns, from the source when copying. + If both `include_pat` and `exclude_pat` are supplied, then it will apply conditions cumulatively. i.e. first select based on include_pat, and then within that result apply exclude_pat. @@ -3707,6 +3717,15 @@ def recurse(name, - exclude_pat: E@(APPDATA)|(TEMPDATA) :: regexp matches APPDATA or TEMPDATA for exclusion + .. versionchanged:: Neon + List patterns are now supported + + .. code-block:: text + + - exclude_pat: + - APPDATA.01 + - APPDATA.02 + maxdepth When copying, only copy paths which are of depth `maxdepth` from the source path. diff --git a/salt/utils/stringutils.py b/salt/utils/stringutils.py index 50f6bd4f3229..81029db19c2d 100644 --- a/salt/utils/stringutils.py +++ b/salt/utils/stringutils.py @@ -452,31 +452,37 @@ def check_include_exclude(path_str, include_pat=None, exclude_pat=None): - If both include_pat and exclude_pat are supplied: return 'True' if include_pat matches AND exclude_pat does not match ''' - ret = True # -- default true - # Before pattern match, check if it is regexp (E@'') or glob(default) - if include_pat: - if re.match('E@', include_pat): - retchk_include = True if re.search( - include_pat[2:], + def _pat_check(path_str, check_pat): + if re.match('E@', check_pat): + return True if re.search( + check_pat[2:], path_str ) else False else: - retchk_include = True if fnmatch.fnmatch( + return True if fnmatch.fnmatch( path_str, - include_pat + check_pat ) else False + ret = True # -- default true + # Before pattern match, check if it is regexp (E@'') or glob(default) + if include_pat: + if isinstance(include_pat, list): + for include_line in include_pat: + retchk_include = _pat_check(path_str, include_line) + if retchk_include: + break + else: + retchk_include = _pat_check(path_str, include_pat) + if exclude_pat: - if re.match('E@', exclude_pat): - retchk_exclude = False if re.search( - exclude_pat[2:], - path_str - ) else True + if isinstance(exclude_pat, list): + for exclude_line in exclude_pat: + retchk_exclude = not _pat_check(path_str, exclude_line) + if not retchk_exclude: + break else: - retchk_exclude = False if fnmatch.fnmatch( - path_str, - exclude_pat - ) else True + retchk_exclude = not _pat_check(path_str, exclude_pat) # Now apply include/exclude conditions if include_pat and not exclude_pat: diff --git a/tests/unit/utils/test_stringutils.py b/tests/unit/utils/test_stringutils.py index 65ebda655949..122f50707a9f 100644 --- a/tests/unit/utils/test_stringutils.py +++ b/tests/unit/utils/test_stringutils.py @@ -555,3 +555,18 @@ def test_check_whitelist_blacklist(self): salt.utils.stringutils.check_whitelist_blacklist, 'foo', blacklist=123 ) + + def test_check_include_exclude_empty(self): + self.assertTrue(salt.utils.stringutils.check_include_exclude("/some/test")) + + def test_check_include_exclude_exclude(self): + self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, "*test*")) + + def test_check_include_exclude_exclude_list(self): + self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, ["*test"])) + + def test_check_include_exclude_exclude_include(self): + self.assertTrue(salt.utils.stringutils.check_include_exclude("/some/test", "*test*", "/some/")) + + def test_check_include_exclude_regex(self): + self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, "E@/some/(test|other)")) From 097572fbafcd36976400582b727d075806944ae0 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sun, 26 Apr 2020 16:54:09 -0700 Subject: [PATCH 2/4] Fixing versionchanged Fixing versionchanged --- salt/states/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 9013eccc459d..415c6ef3ea2e 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -4108,7 +4108,7 @@ def recurse( - include_pat: E@hello :: regexp matches 'otherhello', 'hello01' ... - .. versionchanged:: Neon + .. versionchanged:: Sodium List patterns are now supported @@ -4135,7 +4135,7 @@ def recurse( - exclude_pat: E@(APPDATA)|(TEMPDATA) :: regexp matches APPDATA or TEMPDATA for exclusion - .. versionchanged:: Neon + .. versionchanged:: Sodium List patterns are now supported .. code-block:: text From 99650def26a5b11c93bd7a638d46c33bd118420f Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 27 Apr 2020 10:33:40 -0700 Subject: [PATCH 3/4] Fixing black issues and docstring issues. --- salt/states/file.py | 1 + salt/utils/stringutils.py | 11 +++-------- tests/unit/utils/test_stringutils.py | 20 ++++++++++++++++---- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 415c6ef3ea2e..a7fb82c0350c 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -4188,6 +4188,7 @@ def recurse( True to inherit permissions from parent, otherwise False .. versionadded:: 2017.7.7 + """ if "env" in kwargs: # "env" is not supported; Use "saltenv". diff --git a/salt/utils/stringutils.py b/salt/utils/stringutils.py index 30e1a81593ac..de5d9ee35f7c 100644 --- a/salt/utils/stringutils.py +++ b/salt/utils/stringutils.py @@ -466,17 +466,12 @@ def check_include_exclude(path_str, include_pat=None, exclude_pat=None): - If both include_pat and exclude_pat are supplied: return 'True' if include_pat matches AND exclude_pat does not match """ + def _pat_check(path_str, check_pat): if re.match("E@", check_pat): - return True if re.search( - check_pat[2:], - path_str - ) else False + return True if re.search(check_pat[2:], path_str) else False else: - return True if fnmatch.fnmatch( - path_str, - check_pat - ) else False + return True if fnmatch.fnmatch(path_str, check_pat) else False ret = True # -- default true # Before pattern match, check if it is regexp (E@'') or glob(default) diff --git a/tests/unit/utils/test_stringutils.py b/tests/unit/utils/test_stringutils.py index 0861d760f39f..fcf5afad68bd 100644 --- a/tests/unit/utils/test_stringutils.py +++ b/tests/unit/utils/test_stringutils.py @@ -540,13 +540,25 @@ def test_check_include_exclude_empty(self): self.assertTrue(salt.utils.stringutils.check_include_exclude("/some/test")) def test_check_include_exclude_exclude(self): - self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, "*test*")) + self.assertFalse( + salt.utils.stringutils.check_include_exclude("/some/test", None, "*test*") + ) def test_check_include_exclude_exclude_list(self): - self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, ["*test"])) + self.assertFalse( + salt.utils.stringutils.check_include_exclude("/some/test", None, ["*test"]) + ) def test_check_include_exclude_exclude_include(self): - self.assertTrue(salt.utils.stringutils.check_include_exclude("/some/test", "*test*", "/some/")) + self.assertTrue( + salt.utils.stringutils.check_include_exclude( + "/some/test", "*test*", "/some/" + ) + ) def test_check_include_exclude_regex(self): - self.assertFalse(salt.utils.stringutils.check_include_exclude("/some/test", None, "E@/some/(test|other)")) + self.assertFalse( + salt.utils.stringutils.check_include_exclude( + "/some/test", None, "E@/some/(test|other)" + ) + ) From 2cd8521a167511cd8816633c49dee894c8fd1218 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 27 Apr 2020 11:26:15 -0700 Subject: [PATCH 4/4] Another doc fix. --- salt/states/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index a7fb82c0350c..59585eec0bd6 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -4120,7 +4120,7 @@ def recurse( exclude_pat Exclude this pattern, or list of patterns, from the source when copying. - If both `include_pat` and `exclude_pat` are supplied, then it will apply + If both `include_pat` and `exclude_pat` are supplied, then it will apply conditions cumulatively. i.e. first select based on include_pat, and then within that result apply exclude_pat. @@ -4136,6 +4136,7 @@ def recurse( or TEMPDATA for exclusion .. versionchanged:: Sodium + List patterns are now supported .. code-block:: text