Add SCRAM-SHA-256 support to postgres states

changelog/51271.added

@@ -0,0 +1,2 @@
+SCRAM-SHA-256 support for PostgreSQL passwords.
+Pass encrypted=scram-sha-256 to the postgres_user.present (or postgres_group.present) state.

changelog/59034.fixed

@@ -0,0 +1,3 @@
+Correct comment when updating postrges users and groups.
+Errors reported when removing postgres groups.
+Partial group membership changes in postgres groups.

doc/topics/releases/3003.rst

@@ -6,6 +6,18 @@ Salt 3003 Release Notes - Codename Aluminium
 
 Salt 3003 is an *unreleased* upcoming feature release.
 
+New Features
+============
+
+SCRAM-SHA-256 support for PostgreSQL passwords
+----------------------------------------------
+
+Support for SCRAM-SHA-256 password hashes has been added to the
+:py:func:`postgres_user.present <salt.states.postgres_user.present>`
+and :py:func:`postgres_group.present <salt.states.postgres_group.present>`
+states. This allows migration away from the insecure and deprecated
+previous storage methods.
+
 Execution Module Changes
 ========================
 

salt/ext/saslprep.py

@@ -0,0 +1,109 @@
+# Copyright 2016-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# https://github.com/mongodb/mongo-python-driver/blob/3.11.1/pymongo/saslprep.py
+
+"""An implementation of RFC4013 SASLprep."""
+
+try:
+    import stringprep
+except ImportError:
+    HAVE_STRINGPREP = False
+
+    def saslprep(data):
+        """SASLprep dummy"""
+        if isinstance(data, str):
+            raise TypeError(
+                "The stringprep module is not available. Usernames and "
+                "passwords must be ASCII strings.")
+        return data
+else:
+    HAVE_STRINGPREP = True
+    import unicodedata
+    # RFC4013 section 2.3 prohibited output.
+    _PROHIBITED = (
+        # A strict reading of RFC 4013 requires table c12 here, but
+        # characters from it are mapped to SPACE in the Map step. Can
+        # normalization reintroduce them somehow?
+        stringprep.in_table_c12,
+        stringprep.in_table_c21_c22,
+        stringprep.in_table_c3,
+        stringprep.in_table_c4,
+        stringprep.in_table_c5,
+        stringprep.in_table_c6,
+        stringprep.in_table_c7,
+        stringprep.in_table_c8,
+        stringprep.in_table_c9)
+
+    def saslprep(data, prohibit_unassigned_code_points=True):
+        """An implementation of RFC4013 SASLprep.
+
+        :Parameters:
+          - `data`: The string to SASLprep. Unicode strings
+            (python 2.x unicode, 3.x str) are supported. Byte strings
+            (python 2.x str, 3.x bytes) are ignored.
+          - `prohibit_unassigned_code_points`: True / False. RFC 3454
+            and RFCs for various SASL mechanisms distinguish between
+            `queries` (unassigned code points allowed) and
+            `stored strings` (unassigned code points prohibited). Defaults
+            to ``True`` (unassigned code points are prohibited).
+
+        :Returns:
+        The SASLprep'ed version of `data`.
+        """
+        if not isinstance(data, str):
+            return data
+
+        if prohibit_unassigned_code_points:
+            prohibited = _PROHIBITED + (stringprep.in_table_a1,)
+        else:
+            prohibited = _PROHIBITED
+
+        # RFC3454 section 2, step 1 - Map
+        # RFC4013 section 2.1 mappings
+        # Map Non-ASCII space characters to SPACE (U+0020). Map
+        # commonly mapped to nothing characters to, well, nothing.
+        in_table_c12 = stringprep.in_table_c12
+        in_table_b1 = stringprep.in_table_b1
+        data = "".join(
+            ["\u0020" if in_table_c12(elt) else elt
+             for elt in data if not in_table_b1(elt)])
+
+        # RFC3454 section 2, step 2 - Normalize
+        # RFC4013 section 2.2 normalization
+        data = unicodedata.ucd_3_2_0.normalize('NFKC', data)
+
+        in_table_d1 = stringprep.in_table_d1
+        if in_table_d1(data[0]):
+            if not in_table_d1(data[-1]):
+                # RFC3454, Section 6, #3. If a string contains any
+                # RandALCat character, the first and last characters
+                # MUST be RandALCat characters.
+                raise ValueError("SASLprep: failed bidirectional check")
+            # RFC3454, Section 6, #2. If a string contains any RandALCat
+            # character, it MUST NOT contain any LCat character.
+            prohibited = prohibited + (stringprep.in_table_d2,)
+        else:
+            # RFC3454, Section 6, #3. Following the logic of #3, if
+            # the first character is not a RandALCat, no other character
+            # can be either.
+            prohibited = prohibited + (in_table_d1,)
+
+        # RFC3454 section 2, step 3 and 4 - Prohibit and check bidi
+        for char in data:
+            if any(in_table(char) for in_table in prohibited):
+                raise ValueError(
+                    "SASLprep: failed prohibited character check")
+
+        return data

salt/modules/postgres.py

@@ -31,8 +31,11 @@
 # pylint: disable=E8203
 
 
+import base64
 import datetime
 import hashlib
+import hmac
+import io
 import logging
 import os
 import pipes
@@ -45,8 +48,7 @@
 import salt.utils.path
 import salt.utils.stringutils
 from salt.exceptions import CommandExecutionError, SaltInvocationError
-from salt.ext.six.moves import zip  # pylint: disable=import-error,redefined-builtin
-from salt.ext.six.moves import StringIO
+from salt.ext.saslprep import saslprep
 from salt.utils.versions import LooseVersion as _LooseVersion
 
 try:
@@ -56,11 +58,16 @@
 except ImportError:
     HAS_CSV = False
 
+try:
+    from secrets import token_bytes
+except ImportError:
+    # python <3.6
+    from os import urandom as token_bytes
 
 log = logging.getLogger(__name__)
 
 
-_DEFAULT_PASSWORDS_ENCRYPTION = True
+_DEFAULT_PASSWORDS_ENCRYPTION = "md5"
 _EXTENSION_NOT_INSTALLED = "EXTENSION NOT INSTALLED"
 _EXTENSION_INSTALLED = "EXTENSION INSTALLED"
 _EXTENSION_TO_UPGRADE = "EXTENSION TO UPGRADE"
@@ -272,9 +279,7 @@ def version(
 
         salt '*' postgres.version
     """
-    query = (
-        "SELECT setting FROM pg_catalog.pg_settings " "WHERE name = 'server_version'"
-    )
+    query = "SELECT setting FROM pg_catalog.pg_settings WHERE name = 'server_version'"
     cmd = _psql_cmd(
         "-c",
         query,
@@ -459,7 +464,7 @@ def psql_query(
     if cmdret["retcode"] > 0:
         return ret
 
-    csv_file = StringIO(cmdret["stdout"])
+    csv_file = io.StringIO(cmdret["stdout"])
     header = {}
     for row in csv.reader(
         csv_file,
@@ -1151,20 +1156,89 @@ def _add_role_flag(string, test, flag, cond=None, prefix="NO", addtxt="", skip=F
 
 
 def _maybe_encrypt_password(role, password, encrypted=_DEFAULT_PASSWORDS_ENCRYPTION):
-    """
-    pgsql passwords are md5 hashes of the string: 'md5{password}{rolename}'
-    """
     if password is not None:
         password = str(password)
-    if encrypted and password and not password.startswith("md5"):
-        password = "md5{}".format(
-            hashlib.md5(
-                salt.utils.stringutils.to_bytes("{}{}".format(password, role))
-            ).hexdigest()
-        )
+    else:
+        return None
+
+    if encrypted is True:
+        encrypted = "md5"
+    if encrypted not in (False, "md5", "scram-sha-256"):
+        raise ValueError("Unknown password algorithm: " + str(encrypted))
+
+    if encrypted == "scram-sha-256" and not password.startswith("SCRAM-SHA-256"):
+        password = _scram_sha_256(password)
+    elif encrypted == "md5" and not password.startswith("md5"):
+        log.warning("The md5 password algorithm was deprecated in PostgreSQL 10")
+        password = _md5_password(role, password)
+    elif encrypted is False:
+        log.warning("Unencrypted passwords were removed in PostgreSQL 10")
+
     return password
 
 
+def _verify_password(role, password, verifier, method):
+    """
+    Test the given password against the verifier.
+
+    The given password may already be a verifier, in which case test for
+     simple equality.
+    """
+    if method == "md5" or method is True:
+        if password.startswith("md5"):
+            expected = password
+        else:
+            expected = _md5_password(role, password)
+    elif method == "scram-sha-256":
+        if password.startswith("SCRAM-SHA-256"):
+            expected = password
+        else:
+            match = re.match(r"^SCRAM-SHA-256\$(\d+):([^\$]+?)\$", verifier)
+            if match:
+                iterations = int(match.group(1))
+                salt_bytes = base64.b64decode(match.group(2))
+                expected = _scram_sha_256(
+                    password, salt_bytes=salt_bytes, iterations=iterations
+                )
+            else:
+                expected = object()
+    elif method is False:
+        expected = password
+    else:
+        expected = object()
+
+    return verifier == expected
+
+
+def _md5_password(role, password):
+    return "md5{}".format(
+        hashlib.md5(
+            salt.utils.stringutils.to_bytes("{}{}".format(password, role))
+        ).hexdigest()
+    )
+
+
+def _scram_sha_256(password, salt_bytes=None, iterations=4096):
+    """
+    Build a SCRAM-SHA-256 password verifier.
+
+    Ported from https://doxygen.postgresql.org/scram-common_8c.html
+    """
+    if salt_bytes is None:
+        salt_bytes = token_bytes(16)
+    password = salt.utils.stringutils.to_bytes(saslprep(password))
+    salted_password = hashlib.pbkdf2_hmac("sha256", password, salt_bytes, iterations)
+    stored_key = hmac.new(salted_password, b"Client Key", "sha256").digest()
+    stored_key = hashlib.sha256(stored_key).digest()
+    server_key = hmac.new(salted_password, b"Server Key", "sha256").digest()
+    return "SCRAM-SHA-256${}:{}${}:{}".format(
+        iterations,
+        base64.b64encode(salt_bytes).decode("ascii"),
+        base64.b64encode(stored_key).decode("ascii"),
+        base64.b64encode(server_key).decode("ascii"),
+    )
+
+
 def _role_cmd_args(
     name,
     sub_cmd="",
@@ -1190,7 +1264,7 @@ def _role_cmd_args(
             login = True
         if typ_ == "group":
             login = False
-    # defaults to encrypted passwords (md5{password}{rolename})
+    # defaults to encrypted passwords
     if encrypted is None:
         encrypted = _DEFAULT_PASSWORDS_ENCRYPTION
     skip_passwd = False
@@ -1232,7 +1306,7 @@ def _role_cmd_args(
             "flag": "ENCRYPTED",
             "test": (encrypted is not None and bool(rolepassword)),
             "skip": skip_passwd or isinstance(rolepassword, bool),
-            "cond": encrypted,
+            "cond": bool(encrypted),
             "prefix": "UN",
         },
         {