Skip to content
Permalink
Browse files
Merge branch 'gerrit/neo'
Change-Id: I7ba10b2a046866640ab3d646ea6b71aae399f436
  • Loading branch information
Dmitry Lychagin committed Apr 22, 2022
2 parents f989dd5 + 758f94d commit 039f0e36b01853443666c24ec3cf8fd877ef9831
Showing 39 changed files with 205 additions and 93 deletions.
@@ -98,6 +98,7 @@
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import org.apache.hyracks.util.LogRedactionUtil;

import com.google.common.collect.ImmutableMap;

@@ -338,7 +339,7 @@ private boolean foldRecordArgs(AbstractFunctionCallExpression expr, Void arg) th
IWarningCollector warningCollector = optContext.getWarningCollector();
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(fieldNameExpr.second.getSourceLocation(),
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName));
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName)));
}
iterator.remove();
iterator.next();
@@ -41,6 +41,7 @@
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* <pre>
@@ -116,7 +117,7 @@ private boolean transform(Mutable<ILogicalExpression> expressionRef) throws Alge
IWarningCollector warningCollector = context.getWarningCollector();
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(fieldNameExpr.getSourceLocation(),
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName));
ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName)));
}
iterator.remove();
iterator.next();
@@ -136,6 +136,7 @@
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* Each visit returns a pair of an operator and a variable. The variable
@@ -1009,7 +1010,8 @@ private FieldBinding getFieldBinding(Projection projection, Set<String> outField
private FieldBinding generateFieldBinding(String fieldName, Expression fieldValueExpr, Set<String> outFieldNames,
SourceLocation sourceLoc) throws CompilationException {
if (!outFieldNames.add(fieldName)) {
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc, fieldName);
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc,
LogRedactionUtil.userData(fieldName));
}
return new FieldBinding(new LiteralExpr(new StringLiteral(fieldName)), fieldValueExpr);
}
@@ -55,6 +55,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

public class TypeTranslator {

@@ -314,7 +315,8 @@ private static ARecordType computeRecordType(TypeSignature typeSignature, Record
int i = 0;
for (String s : names) {
if (names.indexOf(s) < i) {
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, rtd.getSourceLocation(), s);
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, rtd.getSourceLocation(),
LogRedactionUtil.userData(s));
}
fldNames[i++] = s;
}
@@ -242,7 +242,7 @@ public static void validateIndexFieldType(IndexType indexType, IAType fieldType,
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is " + "of type " + fieldType.getTypeTag()
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the BTree index.");
}
break;
@@ -102,6 +102,7 @@ public synchronized void flushed(long componentSequence, long lsn, long componen
public synchronized void masterFlush(long masterLsn, long localLsn) throws HyracksDataException {
final IndexCheckpoint latest = getLatest();
latest.getMasterNodeFlushMap().put(masterLsn, localLsn);
LOGGER.trace("index {} master flush {} -> {}", indexPath, masterLsn, localLsn);
final IndexCheckpoint next = IndexCheckpoint.next(latest, latest.getLowWatermark(),
latest.getValidComponentSequence(), latest.getLastComponentId(), null);
persist(next);
@@ -1235,13 +1235,15 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
if (stmtCreateIndex.isEnforced()) {
if (!projectTypeExpr.isUnknownable()) {
throw new CompilationException(ErrorCode.INDEX_ILLEGAL_ENFORCED_NON_OPTIONAL,
indexedElement.getSourceLocation(), String.valueOf(projectPath));
indexedElement.getSourceLocation(),
LogRedactionUtil.userData(String.valueOf(projectPath)));
}
// don't allow creating an enforced index on a closed-type field, fields that
// are part of schema get the field type, if it's not null, then the field is closed-type
if (isFieldFromSchema) {
throw new CompilationException(ErrorCode.INDEX_ILLEGAL_ENFORCED_ON_CLOSED_FIELD,
indexedElement.getSourceLocation(), String.valueOf(projectPath));
indexedElement.getSourceLocation(),
LogRedactionUtil.userData(String.valueOf(projectPath)));
}
} else {
if (indexType != IndexType.BTREE && indexType != IndexType.ARRAY) {
@@ -1272,7 +1274,7 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState

if (fieldTypePrime == null) {
throw new CompilationException(ErrorCode.UNKNOWN_TYPE, indexedElement.getSourceLocation(),
String.valueOf(projectPath));
LogRedactionUtil.userData(String.valueOf(projectPath)));
}
validateIndexFieldType(indexType, fieldTypePrime, projectPath, indexedElement.getSourceLocation());

@@ -1284,7 +1286,8 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
// Try to add the key & its source to the set of keys for duplicate detection.
if (!indexKeysSet.add(indexedElement.toIdentifier())) {
throw new AsterixException(ErrorCode.INDEX_ILLEGAL_REPETITIVE_FIELD,
indexedElement.getSourceLocation(), indexedElement.getProjectListDisplayForm());
indexedElement.getSourceLocation(),
LogRedactionUtil.userData(indexedElement.getProjectListDisplayForm()));
}

indexFieldTypes.add(fieldTypes);
@@ -44,7 +44,7 @@
"replication\.log\.buffer\.numpages" : 8,
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
"replication\.timeout" : 120,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
"storage.global.cleanup.timeout" : 600,
@@ -44,7 +44,7 @@
"replication\.log\.buffer\.numpages" : 8,
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
"replication\.timeout" : 120,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
"storage.global.cleanup.timeout" : 600,
@@ -44,7 +44,7 @@
"replication\.log\.buffer\.numpages" : 8,
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
"replication\.timeout" : 120,
"ssl\.enabled" : false,
"storage.compression.block" : "snappy",
"storage.global.cleanup.timeout" : 600,
@@ -48,7 +48,7 @@ public enum Option implements IOption {
"The size in bytes to replicate in each batch"),
REPLICATION_TIMEOUT(
LONG,
TimeUnit.SECONDS.toSeconds(30),
TimeUnit.SECONDS.toSeconds(120),
"The time in seconds to timeout waiting for master or replica to ack"),
REPLICATION_ENABLED(BOOLEAN, false, "Whether or not data replication is enabled"),
REPLICATION_FACTOR(NONNEGATIVE_INTEGER, 2, "Number of replicas (backups) to maintain per master replica"),
@@ -40,6 +40,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.util.LogRedactionUtil;

public final class JRecord extends JComplexObject<Map<String, Object>> {

@@ -89,7 +90,8 @@ public IJObject getValueByName(String fieldName) throws HyracksDataException {
// check open part
IJObject fieldValue = openFields.get(fieldName);
if (fieldValue == null) {
throw new RuntimeDataException(ErrorCode.LIBRARY_JAVA_JOBJECTS_UNKNOWN_FIELD, fieldName);
throw new RuntimeDataException(ErrorCode.LIBRARY_JAVA_JOBJECTS_UNKNOWN_FIELD,
LogRedactionUtil.userData(fieldName));
}
return fieldValue;
}
@@ -55,6 +55,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* Parser for ADM formatted data.
@@ -472,7 +473,7 @@ private void parseRecord(ARecordType recType, DataOutput out) throws IOException
fieldId = recBuilder.getFieldId(fldName);
if ((fieldId < 0) && !recType.isOpen()) {
throw new ParseException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD,
fldName);
LogRedactionUtil.userData(fldName));
} else if ((fieldId < 0) && recType.isOpen()) {
parseString(tmpTokenImage.getBuffer(), tmpTokenImage.getBegin() + 1,
tmpTokenImage.getLength() - 2, fieldNameBuffer.getDataOutput());
@@ -527,7 +528,7 @@ private void parseRecord(ARecordType recType, DataOutput out) throws IOException
final int nullableFieldId = checkOptionalConstraints(recType, nulls);
if (nullableFieldId != -1) {
throw new ParseException(ErrorCode.PARSER_ADM_DATA_PARSER_FIELD_NOT_NULL,
recType.getFieldNames()[nullableFieldId]);
LogRedactionUtil.userData(recType.getFieldNames()[nullableFieldId]));
}
}
recBuilder.write(out, true);
@@ -48,6 +48,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.ParseUtil;

import com.fasterxml.jackson.core.JsonFactory;
@@ -198,7 +199,7 @@ protected final void parseObject(ARecordType recordType, DataOutput out) throws

if (!recordType.isOpen() && fieldIndex < 0) {
throw new RuntimeDataException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD,
fieldName);
LogRedactionUtil.userData(fieldName));
}
valueBuffer.reset();
nextToken();
@@ -213,7 +214,8 @@ protected final void parseObject(ARecordType recordType, DataOutput out) throws

//fail fast if the current field is not nullable
if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) {
throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, fieldName);
throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
LogRedactionUtil.userData(fieldName));
}

nullBitMap.set(fieldIndex);
@@ -35,6 +35,7 @@
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* Abstract class for nested formats (ADM, JSON, XML ... etc)
@@ -117,7 +118,7 @@ protected void checkOptionalConstraints(ARecordType recordType, BitSet nullBitma
for (int i = 0; i < recordType.getFieldTypes().length; i++) {
if (!nullBitmap.get(i) && !isMissableType(recordType.getFieldTypes()[i])) {
throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
recordType.getFieldNames()[i]);
LogRedactionUtil.userData(recordType.getFieldNames()[i]));
}
}
}
@@ -51,6 +51,7 @@
import org.apache.hyracks.dataflow.common.data.parsers.IValueParser;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.std.file.FieldCursorForDelimitedDataParser;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.ParseUtil;

public class DelimitedDataParser extends AbstractDataParser implements IStreamDataParser, IRecordDataParser<char[]> {
@@ -108,7 +109,8 @@ public DelimitedDataParser(IHyracksTaskContext ctx, IValueParserFactory[] valueP
fldIds[i] = recBuilder.getFieldId(name);
if (fldIds[i] < 0) {
if (!recordType.isOpen()) {
throw new RuntimeDataException(ErrorCode.PARSER_DELIMITED_ILLEGAL_FIELD, name, recordType);
throw new RuntimeDataException(ErrorCode.PARSER_DELIMITED_ILLEGAL_FIELD,
LogRedactionUtil.userData(name), recordType);
} else {
nameBuffers[i] = new ArrayBackedValueStorage();
str.setValue(name);
@@ -43,6 +43,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.string.UTF8StringWriter;

import com.fasterxml.jackson.databind.JsonNode;
@@ -200,7 +201,7 @@ public void writeRecord(JsonNode obj, DataOutput out, ARecordType curRecType) th
if (obj.get(curFNames[iter1]).isNull() && !(curTypes[iter1] instanceof AUnionType)) {
if (curRecType.isClosedField(curFNames[iter1])) {
throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL,
curFNames[iter1]);
LogRedactionUtil.userData(curFNames[iter1]));
} else {
continue;
}
@@ -40,6 +40,7 @@
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

public class KeyFieldTypeUtil {

@@ -323,7 +324,7 @@ public static Triple<IAType, Boolean, Boolean> getKeyProjectType(final ARecordTy
} else {
// closed record type and we couldn't find the field -> error.
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
RecordUtil.toFullyQualifiedName(path));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(path)));
}
}
if (fieldType.getTypeTag() == ATypeTag.UNION) {
@@ -39,7 +39,7 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
@@ -59,6 +59,7 @@
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.common.IResourceFactory;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.util.LogRedactionUtil;

public class RTreeResourceFactoryProvider implements IResourceFactoryProvider {

@@ -82,7 +83,7 @@ public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset
indexDetails.getKeyFieldNames().get(0), recordType).first;
if (spatialType == null) {
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND,
StringUtils.join(indexDetails.getKeyFieldNames().get(0), '.'));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(indexDetails.getKeyFieldNames().get(0))));
}
List<List<String>> primaryKeyFields = dataset.getPrimaryKeys();
int numPrimaryKeys = primaryKeyFields.size();
@@ -38,6 +38,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.util.LogRedactionUtil;

public class ClosedRecordConstructorResultType implements IResultTypeComputer {

@@ -76,7 +77,8 @@ public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironmen
}
for (int j = 0; j < i; j++) {
if (fieldName.equals(fieldNames[j])) {
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(), fieldName);
throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(),
LogRedactionUtil.userData(fieldName));
}
}
fieldTypes[i] = e2Type;

0 comments on commit 039f0e3

Please sign in to comment.