Fields with a state of DROP on normalization failure

.github/workflows/settings.xml

@@ -16,8 +16,8 @@
    <servers>
       <server>
          <id>github-datawave</id>
-         <username>${env.USER_NAME}</username>
-         <password>${env.ACCESS_TOKEN}</password>
+         <username>rdhayes68</username>
+         <password>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</password>
       </server>
    </servers>
 </settings>

warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java

@@ -978,15 +978,17 @@ protected void applyNormalizationAndAddToResults(Multimap<String,NormalizedConte
                 if (n.getError() != null) {
                     switch (getFailurePolicy(n.getIndexedFieldName())) {
                         case LEAVE:
-                            // for the leave policy, clear out the exception but add
-                            // a failed normalization field
+                            // for the leave policy, clear out the exception
                             n.setError(null);
                             results.put(n.getIndexedFieldName(), n);
                             results.put(failedNormalizationField, new NormalizedFieldAndValue(failedNormalizationField, n.getIndexedFieldName()));
                             break;
                         case DROP:
-                            // for the leave policy, only add a failed normalization
+                            // for the drop policy, add a failed normalization and clear out indexed field value
                             // field
+                            n.setError(null);
+                            n.setIndexedFieldValue(null);
+                            results.put(n.getIndexedFieldName(), n);
                             results.put(failedNormalizationField, new NormalizedFieldAndValue(failedNormalizationField, n.getIndexedFieldName()));
                             break;
                         case FAIL:

warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java

@@ -1019,50 +1019,52 @@ protected Multimap<BulkIngestKey,Value> createShardFieldIndexColumn(RawRecordCon
 
         Multimap<BulkIngestKey,Value> values = HashMultimap.create();
 
-        Text colf = new Text("fi");
-        TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
-        Text unmaskedColq = new Text(fieldValue);
-        TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-        TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
-
-        if (value == null) {
-            value = NULL_VALUE;
-        }
-
-        if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
-            if (!StringUtils.isEmpty(fieldValue)) {
-                // Put unmasked colq with original visibility
-                Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
-                BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-                values.put(bKey, value);
+        if (StringUtils.isNotEmpty(fieldValue)) {
+            Text colf = new Text("fi");
+            TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
+            Text unmaskedColq = new Text(fieldValue);
+            TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+            TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+            if (value == null) {
+                value = NULL_VALUE;
             }
 
-            // We need to use the normalized masked values
-            final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
-            if (!StringUtils.isEmpty(normalizedMaskedValue)) {
-                Text maskedColq = new Text(normalizedMaskedValue);
-                TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-                TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+            if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
+                if (!StringUtils.isEmpty(fieldValue)) {
+                    // Put unmasked colq with original visibility
+                    Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
 
-                // Put masked colq with masked visibility
-                Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                // We need to use the normalized masked values
+                final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
+                if (!StringUtils.isEmpty(normalizedMaskedValue)) {
+                    Text maskedColq = new Text(normalizedMaskedValue);
+                    TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+                    TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+                    // Put masked colq with masked visibility
+                    Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
+            } else if (!StringUtils.isEmpty(fieldValue)) {
+                /**
+                 * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
+                 * lower visibility
+                 */
+                byte[] refVisibility = visibility;
+
+                if (null != maskedFieldHelper) {
+                    refVisibility = maskedVisibility;
+                }
+
+                Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
                 BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
                 values.put(bKey, value);
             }
-        } else if (!StringUtils.isEmpty(fieldValue)) {
-            /**
-             * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
-             * lower visibility
-             */
-            byte[] refVisibility = visibility;
-
-            if (null != maskedFieldHelper) {
-                refVisibility = maskedVisibility;
-            }
-
-            Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
-            BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-            values.put(bKey, value);
         }
 
         return values;