Skip to content
Permalink
Browse files
Merge branch 'gerrit/neo'
Change-Id: I9e39d970fddd4652f1c459b50c9abb51cb2d0137
  • Loading branch information
mblow committed Mar 5, 2022
2 parents cb90519 + b10bda4 commit 2d4da74962c5b625cbcd4bddd12684d31aacfb32
Showing 19 changed files with 163 additions and 88 deletions.
@@ -36,6 +36,7 @@
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

/**
* A util that can verify if a filter field, a list of partitioning expressions,
@@ -73,7 +74,7 @@ public static void validateFilterField(ARecordType recordType, ARecordType metaT
IAType fieldType = itemType.getSubFieldType(filterField);
if (fieldType == null) {
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
RecordUtil.toFullyQualifiedName(filterField));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField)));
}
switch (fieldType.getTypeTag()) {
case TINYINT:
@@ -93,7 +94,7 @@ public static void validateFilterField(ARecordType recordType, ARecordType metaT
break;
case UNION:
throw new CompilationException(ErrorCode.COMPILATION_FILTER_CANNOT_BE_NULLABLE,
RecordUtil.toFullyQualifiedName(filterField));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField)));
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_FILTER_TYPE,
fieldType.getTypeTag().name());
@@ -143,7 +144,8 @@ private static List<IAType> validatePartitioningExpressionsImpl(ARecordType recT
IAType fieldType = recType.getSubFieldType(fieldName);
if (fieldType == null) {
String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0);
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, unTypeField);
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
LogRedactionUtil.userData(unTypeField));
}
partitioningExprTypes.add(fieldType);
ATypeTag pkTypeTag = fieldType.getTypeTag();
@@ -159,15 +161,16 @@ private static List<IAType> validatePartitioningExpressionsImpl(ARecordType recT
IAType fieldType = partitioningExprTypes.get(i);
if (fieldType == null) {
throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
RecordUtil.toFullyQualifiedName(partitioningExpr));
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
}
if (forPrimaryKey) {
boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType)
.isSubFieldNullable(partitioningExpr);
if (nullable) {
// key field is nullable
throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc,
keyKindDisplayName, RecordUtil.toFullyQualifiedName(partitioningExpr));
keyKindDisplayName,
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
}
} else {
fieldType = TypeComputeUtils.getActualType(fieldType);
@@ -190,7 +193,8 @@ private static List<IAType> validatePartitioningExpressionsImpl(ARecordType recT
break;
case UNION:
throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc,
keyKindDisplayName, RecordUtil.toFullyQualifiedName(partitioningExpr));
keyKindDisplayName,
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_KEY_TYPE, sourceLoc,
fieldType.getTypeTag(), keyKindDisplayName);
@@ -236,7 +240,9 @@ public static void validateIndexFieldType(IndexType indexType, IAType fieldType,
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is " + "of type " + fieldType.getTypeTag()
+ " cannot be indexed using the BTree index.");
}
break;
@@ -251,14 +257,17 @@ public static void validateIndexFieldType(IndexType indexType, IAType fieldType,
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the RTree index.");
}
break;
case LENGTH_PARTITIONED_NGRAM_INVIX:
if (fieldType.getTypeTag() != ATypeTag.STRING) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Length Partitioned N-Gram index.");
}
break;
@@ -270,14 +279,17 @@ public static void validateIndexFieldType(IndexType indexType, IAType fieldType,
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Length Partitioned Keyword index.");
}
break;
case SINGLE_PARTITION_NGRAM_INVIX:
if (fieldType.getTypeTag() != ATypeTag.STRING) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the N-Gram index.");
}
break;
@@ -289,7 +301,9 @@ public static void validateIndexFieldType(IndexType indexType, IAType fieldType,
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag()
"The field '"
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ "' which is of type " + fieldType.getTypeTag()
+ " cannot be indexed using the Keyword index.");
}
break;
@@ -21,6 +21,7 @@
import static org.apache.hyracks.util.ExitUtil.EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED;

import java.util.Map;
import java.util.Set;

import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
@@ -57,10 +58,12 @@ public void handle(INcApplicationContext appContext) throws HyracksDataException
PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
Map<Long, LocalResource> localResources = localResourceRepository.loadAndGetAllResources();
Set<Integer> nodePartitions = appContext.getReplicaManager().getPartitions();
for (LocalResource resource : localResources.values()) {
DatasetLocalResource lr = (DatasetLocalResource) resource.getResource();
if (MetadataIndexImmutableProperties.isMetadataDataset(lr.getDatasetId())) {
// skip metadata indexes
if (!nodePartitions.contains(lr.getPartition())
|| MetadataIndexImmutableProperties.isMetadataDataset(lr.getDatasetId())) {
// skip replica partitions and metadata indexes
continue;
}
if (!validDatasetIds.contains(lr.getDatasetId())) {
@@ -65,6 +65,7 @@
import org.apache.asterix.app.result.fields.ResultHandlePrinter;
import org.apache.asterix.app.result.fields.ResultsPrinter;
import org.apache.asterix.app.result.fields.StatusPrinter;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.IClientRequest;
import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.api.IRequestTracker;
@@ -202,6 +203,7 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.asterix.runtime.fulltext.AbstractFullTextFilterDescriptor;
import org.apache.asterix.runtime.fulltext.FullTextConfigDescriptor;
import org.apache.asterix.runtime.fulltext.IFullTextFilterDescriptor;
@@ -258,6 +260,7 @@
import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.TokenizerCategory;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.OptionalBoolean;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -829,7 +832,8 @@ protected Optional<? extends Dataset> doCreateDatasetStatement(MetadataProvider
metadataProvider, mdTxnCtx);
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx);
validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx,
appCtx);
datasetDetails = new ExternalDatasetDetails(externalDetails.getAdapter(), properties, new Date(),
TransactionState.COMMIT);
break;
@@ -1221,7 +1225,7 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
if (stmtCreateIndex.hasCastDefaultNull()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
stmtCreateIndex.getSourceLocation(),
"CAST modifier is used without specifying " + "the type of the indexed field");
"CAST modifier is used without specifying the type of the indexed field");
}
fieldTypePrime = projectTypePrime;
fieldTypeNullable = projectTypeNullable;
@@ -1248,7 +1252,10 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
// allow overriding the type of the closed-field only if CAST modifier is used
if (!stmtCreateIndex.hasCastDefaultNull()) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(), "Typed index on '" + projectPath
indexedElement.getSourceLocation(),
"Typed index on '"
+ LogRedactionUtil
.userData(RecordUtil.toFullyQualifiedName(projectPath))
+ "' field could be created only for open datatype");
}
}
@@ -1629,11 +1636,13 @@ private void doCreateIndexImpl(IHyracksClientConnection hcc, MetadataProvider me
}
if (existingIndexKeyFieldNames.equals(indexKeyFieldNames)
&& !existingIndexKeyFieldTypes.equals(indexKeyFieldTypes)) {
String fieldNames = indexKeyFieldNames.stream().map(RecordUtil::toFullyQualifiedName)
.collect(Collectors.joining(","));
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Cannot create index " + index.getIndexName() + " , enforced index "
+ existingIndex.getIndexName() + " on field '"
+ StringUtils.join(indexKeyFieldNames, ',') + "' is already defined with type '"
+ existingIndexKeyFieldTypes + "'");
+ existingIndex.getIndexName() + " on field(s) '"
+ LogRedactionUtil.userData(fieldNames) + "' is already defined with type(s) '"
+ StringUtils.join(existingIndexKeyFieldTypes, ',') + "'");
}
}
}
@@ -4773,23 +4782,23 @@ private static void ensureNotCancelled(ClientRequest clientRequest) throws Runti
}

protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx)
throws AlgebricksException, HyracksDataException {
Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx,
IApplicationContext appCtx) throws AlgebricksException, HyracksDataException {
// Validate adapter specific properties
String adapter = externalDetails.getAdapter();
Map<String, String> details = new HashMap<>(properties);
details.put(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE, adapter);
validateAdapterSpecificProperties(details, srcLoc);
validateAdapterSpecificProperties(details, srcLoc, appCtx);
}

/**
* Ensures that the external source container is present
*
* @param configuration external source properties
*/
protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
throws CompilationException {
ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc, warningCollector);
protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc,
IApplicationContext appCtx) throws CompilationException {
ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc, warningCollector, appCtx);
}

protected enum CreateResult {
@@ -530,37 +530,37 @@
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_1">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the BTree index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the BTree index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_2">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[age]' which is of type integer cannot be indexed using the RTree index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'age' which is of type integer cannot be indexed using the RTree index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_3">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_4">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_5">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="exception">
<compilation-unit name="issue_384_create_index_error_6">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error>
<expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error>
</compilation-unit>
</test-case>
</test-group>

0 comments on commit 2d4da74

Please sign in to comment.