Skip to content
Permalink
Browse files
Merge branch 'gerrit/neo'
Change-Id: I1944b0ce55e9aeaae4ec5c3c2947201a73902752
  • Loading branch information
mblow committed Jun 30, 2022
2 parents 0791928 + 5b15675 commit 3f59cf49dc89e9e44d4ab00799809faa8f439895
Showing 22 changed files with 439 additions and 45 deletions.
@@ -1295,8 +1295,23 @@ protected void doCreateIndex(MetadataProvider metadataProvider, CreateIndexState
}

if (fieldTypePrime == null) {
throw new CompilationException(ErrorCode.UNKNOWN_TYPE, indexedElement.getSourceLocation(),
LogRedactionUtil.userData(String.valueOf(projectPath)));
if (projectPath != null) {
String fieldName = LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(projectPath));
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(),
"cannot find type of field '" + fieldName + "'");
}
// projectPath == null should only be the case with array index having UNNESTs only
if (indexedElement.hasUnnest()) {
List<List<String>> unnestList = indexedElement.getUnnestList();
List<String> arrayField = unnestList.get(unnestList.size() - 1);
String fieldName = LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(arrayField));
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
indexedElement.getSourceLocation(),
"cannot find type of elements of field '" + fieldName + "'");
}
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
indexedElement.getSourceLocation(), "cannot find type of field");
}
validateIndexFieldType(indexType, fieldTypePrime, projectPath, indexedElement.getSourceLocation());

@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST open_array_f) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST open_array_f UNNEST nested_array) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST array_f UNNEST nested_array) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST array_f SELECT proj1) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST array_f UNNEST nested_array SELECT proj1) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST open_array_f SELECT proj1) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

/*
* Description: Verify that a proper error message is reported when the type of the indexed field could not be found
*/

DROP DATAVERSE testDv IF EXISTS;
CREATE DATAVERSE testDv;
USE testDv;

CREATE TYPE t0 as {a: string};
CREATE TYPE t1 as {id: int, obj_f: t0, array_f: [t0]};
CREATE DATASET ds(t1) PRIMARY KEY id;

CREATE INDEX idx1 ON ds(UNNEST open_array_f UNNEST nested_array SELECT proj1) EXCLUDE UNKNOWN KEY;
@@ -0,0 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

DROP DATAVERSE testDv IF EXISTS;
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

DROP DATAVERSE test IF EXISTS;
CREATE DATAVERSE test;

USE test;

CREATE TYPE ParquetType as {
};

CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
(
%template%,
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format" = "parquet")
);
@@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

USE test;

SELECT VALUE COUNT(*)
FROM ParquetDataset p
@@ -0,0 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

DROP DATAVERSE test IF EXISTS;
@@ -172,6 +172,14 @@
<expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/parquet/invalid-parquet-files">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">none</output-dir>
<source-location>false</source-location>
<expected-error>20-records.json. Reason: not a Parquet file</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset" check-warnings="true">
<compilation-unit name="common/parquet/parquet-types/unset-flags">
<placeholder name="adapter" value="S3" />

0 comments on commit 3f59cf4

Please sign in to comment.