Skip to content
Permalink
Browse files
[NO ISSUE][OTH] Redact field name
- user model changes: no
- storage format changes: no
- interface changes: no

Details:

Change-Id: I360681ff4e1460561eb5b614ce16f7d6519096b2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/16023
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Michael Blow <mblow@apache.org>
  • Loading branch information
AliSolaiman committed Apr 8, 2022
1 parent 8698498 commit a81d38112c3c8c114432c5039c0b752a1954dbdd
Showing 19 changed files with 109 additions and 58 deletions.
@@ -98,6 +98,7 @@
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.hyracks.util.LogRedactionUtil;

import com.google.common.collect.ImmutableMap;

@@ -334,7 +335,7 @@ private boolean foldRecordArgs(AbstractFunctionCallExpression expr, Void arg) th
IWarningCollector warningCollector = optContext.getWarningCollector();
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(fieldNameExpr.second.getSourceLocation(),
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName));
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName)));
}
iterator.remove();
iterator.next();
@@ -41,6 +41,7 @@
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* <pre>
@@ -116,7 +117,7 @@ private boolean transform(Mutable<ILogicalExpression> expressionRef) throws Alge
IWarningCollector warningCollector = context.getWarningCollector();
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(fieldNameExpr.getSourceLocation(),
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName));
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName)));
}
iterator.remove();
iterator.next();
@@ -136,6 +136,7 @@
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* Each visit returns a pair of an operator and a variable. The variable
@@ -939,7 +940,8 @@ private FieldBinding getFieldBinding(Projection projection, Set<String> outField
private FieldBinding generateFieldBinding(String fieldName, Expression fieldValueExpr, Set<String> outFieldNames,
SourceLocation sourceLoc) throws CompilationException {
if (!outFieldNames.add(fieldName)) {
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc, fieldName);
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc,
LogRedactionUtil.userData(fieldName));
}
return new FieldBinding(new LiteralExpr(new StringLiteral(fieldName)), fieldValueExpr);
}
@@ -32,6 +32,7 @@
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* A util that can verify if a filter field, a list of partitioning expressions,
@@ -66,7 +67,7 @@ public static void validateFilterField(ARecordType recordType, ARecordType metaT
IAType fieldType = itemType.getSubFieldType(filterField);
if (fieldType == null) {
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
RecordUtil.toFullyQualifiedName(filterField));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField)));
}
switch (fieldType.getTypeTag()) {
case TINYINT:
@@ -86,7 +87,7 @@ public static void validateFilterField(ARecordType recordType, ARecordType metaT
break;
case UNION:
throw new CompilationException(ErrorCode.COMPILATION_FILTER_CANNOT_BE_NULLABLE,
RecordUtil.toFullyQualifiedName(filterField));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField)));
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_FILTER_TYPE,
fieldType.getTypeTag().name());
@@ -128,7 +129,8 @@ public static List<IAType> validatePartitioningExpressions(ARecordType recType,
IAType fieldType = recType.getSubFieldType(fieldName);
if (fieldType == null) {
String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0);
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, unTypeField);
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
LogRedactionUtil.userData(unTypeField));
}
partitioningExprTypes.add(fieldType);
ATypeTag pkTypeTag = fieldType.getTypeTag();
@@ -144,14 +146,14 @@ public static List<IAType> validatePartitioningExpressions(ARecordType recType,
IAType fieldType = partitioningExprTypes.get(i);
if (fieldType == null) {
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
RecordUtil.toFullyQualifiedName(partitioningExpr));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
}
boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType)
.isSubFieldNullable(partitioningExpr);
if (nullable) {
// key field is nullable
throw new CompilationException(ErrorCode.COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE, sourceLoc,
RecordUtil.toFullyQualifiedName(partitioningExpr));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
}
switch (fieldType.getTypeTag()) {
case TINYINT:
@@ -171,7 +173,7 @@ public static List<IAType> validatePartitioningExpressions(ARecordType recType,
break;
case UNION:
throw new CompilationException(ErrorCode.COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE, sourceLoc,
RecordUtil.toFullyQualifiedName(partitioningExpr));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_PRIMARY_KEY_TYPE, sourceLoc,
fieldType.getTypeTag());
@@ -209,16 +211,20 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
fieldType = keyFieldTypes.get(pos);
if (keyFieldTypes.get(pos) == BuiltinType.AMISSING) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"A field with this name \"" + fieldName + "\" could not be found.");
"A field with this name \""
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "\" could not be found.");
}
} else if (openFieldCompositeIdx) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "A closed field \"" + fieldName
+ "\" could be only in a prefix part of the composite index, containing opened field.");
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"A closed field \"" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "\" could be only in a prefix part of the composite index, containing opened field.");
}
if (keyFieldTypes.get(pos) != BuiltinType.AMISSING
&& fieldType.getTypeTag() != keyFieldTypes.get(pos).getTypeTag()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"A field \"" + fieldName + "\" is already defined with the type \"" + fieldType + "\"");
"A field \"" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + "\" is "
+ "already defined with the type \"" + fieldType + "\"");
}
switch (indexType) {
case BTREE:
@@ -241,7 +247,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the BTree index.");
}
break;
@@ -257,7 +265,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the RTree index.");
}
break;
@@ -268,7 +278,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Length Partitioned N-Gram index.");
}
break;
@@ -281,7 +293,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Length Partitioned Keyword index.");
}
break;
@@ -292,7 +306,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the N-Gram index.");
}
break;
@@ -305,7 +321,9 @@ public static void validateKeyFields(ARecordType recType, ARecordType metaRecTyp
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Keyword index.");
}
break;
@@ -41,6 +41,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;

import org.apache.asterix.active.ActivityState;
import org.apache.asterix.active.EntityId;
@@ -186,6 +187,7 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
import org.apache.asterix.translator.AbstractLangTranslator;
import org.apache.asterix.translator.ClientRequest;
@@ -236,6 +238,7 @@
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -1090,19 +1093,21 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
}
if (stmtCreateIndex.isEnforced() && !fieldExpr.second.isUnknownable()) {
throw new AsterixException(ErrorCode.INDEX_ILLEGAL_ENFORCED_NON_OPTIONAL, sourceLoc,
String.valueOf(fieldExpr.first));
LogRedactionUtil.userData(String.valueOf(fieldExpr.first)));
}
// don't allow creating an enforced index on a closed-type field, fields that
// are part of schema.
// get the field type, if it's not null, then the field is closed-type
if (stmtCreateIndex.isEnforced()
&& subType.getSubFieldType(fieldExpr.first.subList(i, fieldExpr.first.size())) != null) {
throw new AsterixException(ErrorCode.INDEX_ILLEGAL_ENFORCED_ON_CLOSED_FIELD, sourceLoc,
String.valueOf(fieldExpr.first));
LogRedactionUtil.userData(String.valueOf(fieldExpr.first)));
}
if (!isOpen) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Typed index on \""
+ fieldExpr.first + "\" field could be created only for open datatype");
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Typed index on '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldExpr.first))
+ "' " + "field could be created only for open datatype");
}
if (stmtCreateIndex.hasMetaField()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
@@ -1115,16 +1120,17 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
overridesFieldTypes = true;
}
if (fieldType == null) {
throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, fieldExpr.second == null
? String.valueOf(fieldExpr.first) : String.valueOf(fieldExpr.second));
throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc,
fieldExpr.second == null ? LogRedactionUtil.userData(String.valueOf(fieldExpr.first))
: String.valueOf(fieldExpr.second));
}

// try to add the key & its source to the set of keys, if key couldn't be added,
// there is a duplicate
if (!indexKeysSet
.add(new Pair<>(fieldExpr.first, stmtCreateIndex.getFieldSourceIndicators().get(keyIndex)))) {
throw new AsterixException(ErrorCode.INDEX_ILLEGAL_REPETITIVE_FIELD, sourceLoc,
String.valueOf(fieldExpr.first));
LogRedactionUtil.userData(String.valueOf(fieldExpr.first)));
}

indexFields.add(fieldExpr.first);
@@ -1232,10 +1238,13 @@ private void doCreateIndexImpl(IHyracksClientConnection hcc, MetadataProvider me
if (existingIndex.getKeyFieldNames().equals(index.getKeyFieldNames())
&& !existingIndex.getKeyFieldTypes().equals(index.getKeyFieldTypes())
&& existingIndex.isEnforced()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot create index "
+ index.getIndexName() + " , enforced index " + existingIndex.getIndexName()
+ " on field \"" + StringUtils.join(index.getKeyFieldNames(), ',')
+ "\" is already defined with type \"" + existingIndex.getKeyFieldTypes() + "\"");
String fieldNames = index.getKeyFieldNames().stream().map(RecordUtil::toFullyQualifiedName)
.collect(Collectors.joining(","));
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Cannot create index " + index.getIndexName() + " , enforced index "
+ existingIndex.getIndexName() + " on field(s) '"
+ LogRedactionUtil.userData(fieldNames) + "' is already defined with type(s) '"
+ StringUtils.join(existingIndex.getKeyFieldTypes(), ',') + "'");
}
}
}

0 comments on commit a81d381

Please sign in to comment.