Configure masking algorithm default (#4336)

Signed-off-by: Terry Quigley <terry.quigley@sas.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
opensearch-project · May 15, 2024 · d19a8ba · d19a8ba
1 parent f065575
commit d19a8ba
Show file tree

Hide file tree

Showing 7 changed files with 226 additions and 55 deletions.
diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java
@@ -58,6 +58,7 @@
 import java.util.stream.Stream;
 
 import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.QueryCachingPolicy;
@@ -430,6 +431,19 @@ public List<Path> run() {
             }
         }
 
+        try {
+            String maskingAlgorithmDefault = settings.get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT);
+            if (StringUtils.isNotEmpty(maskingAlgorithmDefault)) {
+                MessageDigest.getInstance(maskingAlgorithmDefault);
+            }
+        } catch (Exception ex) {
+            throw new OpenSearchSecurityException(
+                "JVM does not support algorithm for {}",
+                ex,
+                ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT
+            );
+        }
+
         if (!client && !settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES, false)) {
             // check for demo certificates
             final List<String> files = AccessController.doPrivileged(new PrivilegedAction<List<String>>() {
@@ -1383,6 +1397,9 @@ public List<Setting<?>> getSettings() {
             settings.add(
                 Setting.boolSetting(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_TRANSPORT, true, Property.NodeScope, Property.Filtered)
             );
+            settings.add(
+                Setting.simpleString(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, Property.NodeScope, Property.Filtered)
+            );
             final List<String> disabledCategories = new ArrayList<String>(2);
             disabledCategories.add("AUTHENTICATED");
             disabledCategories.add("GRANTED_PRIVILEGES");

diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java
@@ -106,6 +106,7 @@ class DlsFlsFilterLeafReader extends SequentialStoredFieldsLeafReader {
     private final ShardId shardId;
     private final boolean maskFields;
     private final Salt salt;
+    private final String maskingAlgorithmDefault;
 
     private DlsGetEvaluator dge = null;
 
@@ -130,7 +131,8 @@ class DlsFlsFilterLeafReader extends SequentialStoredFieldsLeafReader {
         this.clusterService = clusterService;
         this.auditlog = auditlog;
         this.salt = salt;
-        this.maskedFieldsMap = MaskedFieldsMap.extractMaskedFields(maskFields, maskedFields, salt);
+        this.maskingAlgorithmDefault = clusterService.getSettings().get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT);
+        this.maskedFieldsMap = MaskedFieldsMap.extractMaskedFields(maskFields, maskedFields, salt, maskingAlgorithmDefault);
 
         this.shardId = shardId;
         flsEnabled = includesExcludes != null && !includesExcludes.isEmpty();
@@ -292,11 +294,16 @@ private MaskedFieldsMap(Map<WildcardMatcher, MaskedField> maskedFieldsMap) {
             this.maskedFieldsMap = maskedFieldsMap;
         }
 
-        public static MaskedFieldsMap extractMaskedFields(boolean maskFields, Set<String> maskedFields, final Salt salt) {
+        public static MaskedFieldsMap extractMaskedFields(
+            boolean maskFields,
+            Set<String> maskedFields,
+            final Salt salt,
+            String algorithmDefault
+        ) {
             if (maskFields) {
                 return new MaskedFieldsMap(
                     maskedFields.stream()
-                        .map(mf -> new MaskedField(mf, salt))
+                        .map(mf -> new MaskedField(mf, salt, algorithmDefault))
                         .collect(ImmutableMap.toImmutableMap(mf -> WildcardMatcher.from(mf.getName()), Function.identity()))
                 );
             } else {
@@ -1210,7 +1217,7 @@ private MaskedFieldsMap getRuntimeMaskedFieldInfo() {
         if (maskedEval != null) {
             final Set<String> mf = maskedFieldsMap.get(maskedEval);
             if (mf != null && !mf.isEmpty()) {
-                return MaskedFieldsMap.extractMaskedFields(true, mf, salt);
+                return MaskedFieldsMap.extractMaskedFields(true, mf, salt, maskingAlgorithmDefault);
             }
 
         }

diff --git a/src/main/java/org/opensearch/security/configuration/MaskedField.java b/src/main/java/org/opensearch/security/configuration/MaskedField.java
@@ -20,6 +20,7 @@
 import java.util.Objects;
 
 import com.google.common.base.Splitter;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.lucene.util.BytesRef;
 import org.bouncycastle.util.encoders.Hex;
 
@@ -31,9 +32,11 @@ public class MaskedField {
     private String algo = null;
     private List<RegexReplacement> regexReplacements;
     private final byte[] defaultSalt;
+    private final String defaultAlgorithm;
 
-    public MaskedField(final String value, final Salt salt) {
+    public MaskedField(final String value, final Salt salt, final String defaultAlgorithm) {
         this.defaultSalt = salt.getSalt16();
+        this.defaultAlgorithm = defaultAlgorithm;
         final List<String> tokens = Splitter.on("::").splitToList(Objects.requireNonNull(value));
         final int tokenCount = tokens.size();
         if (tokenCount == 1) {
@@ -57,31 +60,31 @@ public final void isValid() throws Exception {
     }
 
     public byte[] mask(byte[] value) {
-        if (isDefault()) {
-            return blake2bHash(value);
+        if (algo != null) {
+            return customHash(value, algo);
+        } else if (regexReplacements != null) {
+            String cur = new String(value, StandardCharsets.UTF_8);
+            for (RegexReplacement rr : regexReplacements) {
+                cur = cur.replaceAll(rr.getRegex(), rr.getReplacement());
+            }
+            return cur.getBytes(StandardCharsets.UTF_8);
+        } else if (StringUtils.isNotEmpty(defaultAlgorithm)) {
+            return customHash(value, defaultAlgorithm);
         } else {
-            return customHash(value);
+            return blake2bHash(value);
         }
     }
 
     public String mask(String value) {
-        if (isDefault()) {
-            return blake2bHash(value);
-        } else {
-            return customHash(value);
-        }
+        return new String(mask(value.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
     }
 
     public BytesRef mask(BytesRef value) {
         if (value == null) {
             return null;
         }
-
-        if (isDefault()) {
-            return blake2bHash(value);
-        } else {
-            return customHash(value);
-        }
+        final BytesRef copy = BytesRef.deepCopyOf(value);
+        return new BytesRef(mask(copy.bytes));
     }
 
     public String getName() {
@@ -126,6 +129,8 @@ public String toString() {
             + regexReplacements
             + ", defaultSalt="
             + Arrays.toString(defaultSalt)
+            + ", defaultAlgorithm="
+            + defaultAlgorithm
             + ", isDefault()="
             + isDefault()
             + "]";
@@ -135,35 +140,15 @@ private boolean isDefault() {
         return regexReplacements == null && algo == null;
     }
 
-    private byte[] customHash(byte[] in) {
-        if (algo != null) {
-            try {
-                MessageDigest digest = MessageDigest.getInstance(algo);
-                return Hex.encode(digest.digest(in));
-            } catch (NoSuchAlgorithmException e) {
-                throw new IllegalArgumentException(e);
-            }
-        } else if (regexReplacements != null) {
-            String cur = new String(in, StandardCharsets.UTF_8);
-            for (RegexReplacement rr : regexReplacements) {
-                cur = cur.replaceAll(rr.getRegex(), rr.getReplacement());
-            }
-            return cur.getBytes(StandardCharsets.UTF_8);
-
-        } else {
-            throw new IllegalArgumentException();
+    private static byte[] customHash(byte[] in, final String algorithm) {
+        try {
+            MessageDigest digest = MessageDigest.getInstance(algorithm);
+            return Hex.encode(digest.digest(in));
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalArgumentException(e);
         }
     }
 
-    private BytesRef customHash(BytesRef in) {
-        final BytesRef copy = BytesRef.deepCopyOf(in);
-        return new BytesRef(customHash(copy.bytes));
-    }
-
-    private String customHash(String in) {
-        return new String(customHash(in.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
-    }
-
     private byte[] blake2bHash(byte[] in) {
         // Salt is passed incorrectly but order of parameters is retained at present to ensure full backwards compatibility
         // Tracking with https://github.com/opensearch-project/security/issues/4274
@@ -174,15 +159,6 @@ private byte[] blake2bHash(byte[] in) {
         return Hex.encode(out);
     }
 
-    private BytesRef blake2bHash(BytesRef in) {
-        final BytesRef copy = BytesRef.deepCopyOf(in);
-        return new BytesRef(blake2bHash(copy.bytes));
-    }
-
-    private String blake2bHash(String in) {
-        return new String(blake2bHash(in.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
-    }
-
     private static class RegexReplacement {
         private final String regex;
         private final String replacement;

diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java
@@ -36,6 +36,7 @@
 import org.opensearch.security.dlic.rest.validation.RequestContentValidator.DataType;
 import org.opensearch.security.dlic.rest.validation.ValidationResult;
 import org.opensearch.security.securityconf.impl.CType;
+import org.opensearch.security.support.ConfigConstants;
 import org.opensearch.threadpool.ThreadPool;
 
 import static org.opensearch.security.dlic.rest.api.RequestHandler.methodNotImplementedHandler;
@@ -91,7 +92,11 @@ private ValidationResult<JsonNode> validateMaskedFields(final JsonNode content)
 
         private Pair<String, String> validateMaskedFieldSyntax(final JsonNode maskedFieldNode) {
             try {
-                new MaskedField(maskedFieldNode.asText(), SALT).isValid();
+                new MaskedField(
+                    maskedFieldNode.asText(),
+                    SALT,
+                    validationContext.settings().get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT)
+                ).isValid();
             } catch (Exception e) {
                 return Pair.of(maskedFieldNode.asText(), e.getMessage());
             }

diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java
@@ -327,6 +327,7 @@ public enum RolesMappingResolution {
     public static final Boolean SECURITY_SYSTEM_INDICES_PERMISSIONS_DEFAULT = false;
     public static final String SECURITY_SYSTEM_INDICES_KEY = "plugins.security.system_indices.indices";
     public static final List<String> SECURITY_SYSTEM_INDICES_DEFAULT = Collections.emptyList();
+    public static final String SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT = "plugins.security.masked_fields.algorithm.default";
 
     public static final String TENANCY_PRIVATE_TENANT_NAME = "private";
     public static final String TENANCY_GLOBAL_TENANT_NAME = "global";

diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java
@@ -18,7 +18,9 @@
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.support.WriteRequest.RefreshPolicy;
 import org.opensearch.client.Client;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.security.support.ConfigConstants;
 import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse;
 
 public class CustomFieldMaskedTest extends AbstractDlsFlsTest {
@@ -272,4 +274,49 @@ public void testCustomMaskedGet() throws Exception {
         Assert.assertTrue(res.getBody().contains("***.100.1.XXX"));
         Assert.assertTrue(res.getBody().contains("123.123.1.XXX"));
     }
+
+    @Test
+    public void testCustomMaskedGetWithClusterDefaultSHA3() throws Exception {
+
+        final Settings settings = Settings.builder().put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA3-224").build();
+        setup(settings);
+
+        HttpResponse res;
+
+        Assert.assertEquals(
+            HttpStatus.SC_OK,
+            (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("admin", "admin"))).getStatusCode()
+        );
+        Assert.assertTrue(res.getBody().contains("\"found\" : true"));
+        Assert.assertTrue(res.getBody().contains("cust1"));
+        Assert.assertFalse(res.getBody().contains("cust2"));
+        Assert.assertTrue(res.getBody().contains("100.100.1.1"));
+        Assert.assertFalse(res.getBody().contains("100.100.2.2"));
+        Assert.assertFalse(
+            res.getBody()
+                .contains(
+                    "8976994d0491e35f74fcac67ede9c83334a6ad34dae07c176df32f10225f93c5077ddd302c02ddd618b2406b1e4dfe50a727cbc880cfe264c552decf2d224ffc"
+                )
+        );
+        Assert.assertFalse(res.getBody().contains("***"));
+        Assert.assertFalse(res.getBody().contains("XXX"));
+
+        Assert.assertEquals(
+            HttpStatus.SC_OK,
+            (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("user_masked_custom", "password"))).getStatusCode()
+        );
+        Assert.assertTrue(res.getBody().contains("\"found\" : true"));
+        Assert.assertFalse(res.getBody().contains("cust1"));
+        Assert.assertFalse(res.getBody().contains("cust2"));
+        Assert.assertFalse(res.getBody().contains("100.100.1.1"));
+        Assert.assertFalse(res.getBody().contains("100.100.2.2"));
+        Assert.assertTrue(
+            res.getBody()
+                .contains(
+                    "8976994d0491e35f74fcac67ede9c83334a6ad34dae07c176df32f10225f93c5077ddd302c02ddd618b2406b1e4dfe50a727cbc880cfe264c552decf2d224ffc"
+                )
+        );
+        Assert.assertTrue(res.getBody().contains("***.100.1.XXX"));
+        Assert.assertTrue(res.getBody().contains("123.123.1.XXX"));
+    }
 }