Skip to content
Permalink
Browse files
Merge branch 'gerrit/neo'
Change-Id: I3c5e5c46a7b9e7991cb3bfff4b94275902b7ac7c
  • Loading branch information
mblow committed Jan 25, 2022
2 parents 017e1f5 + f4c503a commit 90c4a9389c09d1ce26bd5552e4c28c0302f4bc27
Showing 18 changed files with 786 additions and 411 deletions.
@@ -103,6 +103,7 @@
import org.apache.asterix.api.http.server.QueryServiceRequestParameters;
import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.api.Duration;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.utils.Servlets;
@@ -2570,32 +2571,12 @@ public static String stripAllComments(String statement) {
public void cleanup(String testCase, List<String> badtestcases) throws Exception {
try {
List<DataverseName> toBeDropped = new ArrayList<>();
InputStream resultStream = executeQueryService(
"select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName;",
getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON);
JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8));
for (int i = 0; i < result.size(); i++) {
JsonNode json = result.get(i);
if (json != null) {
DataverseName dvName = DataverseName.createFromCanonicalForm(json.get("DataverseName").asText());
if (!dvName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)
&& !dvName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)) {
toBeDropped.add(dvName);
}
}
}
listUserDefinedDataverses(toBeDropped);
if (!toBeDropped.isEmpty()) {
badtestcases.add(testCase);
LOGGER.info("Last test left some garbage. Dropping dataverses: " + StringUtils.join(toBeDropped, ','));
StringBuilder dropStatement = new StringBuilder();
for (DataverseName dv : toBeDropped) {
dropStatement.setLength(0);
dropStatement.append("drop dataverse ");
SqlppStatementUtil.encloseDataverseName(dropStatement, dv);
dropStatement.append(";\n");
resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE),
OutputFormat.CLEAN_JSON, UTF_8);
ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
dropDataverse(dv);
}
}
} catch (Throwable th) {
@@ -2604,6 +2585,54 @@ public void cleanup(String testCase, List<String> badtestcases) throws Exception
}
}

protected void listUserDefinedDataverses(List<DataverseName> outDataverses) throws Exception {
String query = "select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName";
InputStream resultStream =
executeQueryService(query, getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON);
JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8));
for (int i = 0; i < result.size(); i++) {
JsonNode json = result.get(i);
if (json != null) {
DataverseName dvName = DataverseName.createFromCanonicalForm(json.get("DataverseName").asText());
if (!dvName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)
&& !dvName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)) {
outDataverses.add(dvName);
}
}
}
}

protected void dropDataverse(DataverseName dv) throws Exception {
StringBuilder dropStatement = new StringBuilder();
dropStatement.append("drop dataverse ");
SqlppStatementUtil.encloseDataverseName(dropStatement, dv);
dropStatement.append(";\n");
InputStream resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE),
OutputFormat.CLEAN_JSON, UTF_8);
ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
}

protected void listDatasets(DataverseName dataverseName, List<Pair<String, DatasetConfig.DatasetType>> outDatasets)
throws Exception {
String query = "select d.DatasetName, d.DatasetType from Metadata.`Dataset` d where d.DataverseName = '"
+ dataverseName.getCanonicalForm() + "'";
InputStream resultStream = executeQueryService(query, getEndpoint(Servlets.QUERY_SERVICE),
TestCaseContext.OutputFormat.CLEAN_JSON);
JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8));
for (int i = 0; i < result.size(); i++) {
JsonNode json = result.get(i);
String datasetName = json.get("DatasetName").asText();
String datasetTypeText = json.get("DatasetType").asText();
DatasetConfig.DatasetType datasetType;
try {
datasetType = DatasetConfig.DatasetType.valueOf(datasetTypeText);
} catch (IllegalArgumentException e) {
throw new Exception("Unexpected dataset type: " + datasetTypeText);
}
outDatasets.add(new Pair<>(datasetName, datasetType));
}
}

private JsonNode extractResult(String jsonString) throws IOException {
try {
final JsonNode result = RESULT_NODE_READER.<ObjectNode> readValue(jsonString).get("results");
@@ -45,7 +45,6 @@
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>3.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -145,7 +145,6 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -171,32 +171,21 @@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
<gav>io.netty:netty-transport:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-codec:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-buffer:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-resolver:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-handler:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-common:4.1.69.Final</gav>
<gavs>
<gav>io.netty:netty-buffer:4.1.69.Final</gav>
<gav>io.netty:netty-codec:4.1.69.Final</gav>
<gav>io.netty:netty-codec-http:4.1.69.Final</gav>
<gav>io.netty:netty-common:4.1.69.Final</gav>
<gav>io.netty:netty-handler:4.1.69.Final</gav>
<gav>io.netty:netty-resolver:4.1.69.Final</gav>
<gav>io.netty:netty-transport:4.1.69.Final</gav>
</gavs>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-codec-http:4.1.69.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
<gav>io.netty:netty-tcnative-classes:2.0.46.Final</gav>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
<noticeUrl>https://raw.githubusercontent.com/netty/netty-tcnative/netty-tcnative-parent-2.0.46.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>org.reactivestreams:reactive-streams:1.0.2</gav>
@@ -249,21 +238,15 @@
</override>
<override>
<gavs>
<gav>com.azure:azure-core:1.17.0</gav>
<gav>com.azure:azure-core:1.22.0</gav>
<gav>com.azure:azure-core-http-netty:1.10.0</gav>
<gav>com.azure:azure-core-http-netty:1.11.2</gav>
<gav>com.azure:azure-identity:1.4.1</gav>
<gav>com.azure:azure-storage-blob:12.12.0</gav>
<gav>com.azure:azure-storage-blob:12.14.2</gav>
<gav>com.azure:azure-storage-common:12.12.0</gav>
<gav>com.azure:azure-storage-common:12.14.1</gav>
<gav>com.azure:azure-storage-internal-avro:12.0.5</gav>
<gav>com.azure:azure-storage-internal-avro:12.1.2</gav>
<gav>com.azure:azure-storage-file-datalake:12.7.2</gav>
<gav>com.azure:azure-core:1.24.1</gav>
<gav>com.azure:azure-core-http-netty:1.11.6</gav>
<gav>com.azure:azure-storage-blob:12.14.3</gav>
<gav>com.azure:azure-storage-common:12.14.2</gav>
<gav>com.azure:azure-storage-internal-avro:12.1.3</gav>
</gavs>
<noticeUrl>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/master/NOTICE.txt</noticeUrl>
<url>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/master/LICENSE.txt</url>
<!-- azure-storage-blob-batch_12.11.3 release is at 89a32290750a18d1b99c27c16b1b11d42f16c622 -->
<noticeUrl>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/89a32290750a18d1b99c27c16b1b11d42f16c622/NOTICE.txt</noticeUrl>
<url>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/89a32290750a18d1b99c27c16b1b11d42f16c622/LICENSE.txt</url>
</override>
<override>
<gav>com.microsoft.azure:msal4j:1.11.0</gav>
@@ -553,6 +536,13 @@
<url>PUBLIC_DOMAIN</url>
<aliasUrls>Public Domain</aliasUrls>
</license>
<license>
<url>https://creativecommons.org/publicdomain/zero/1.0/legalcode.txt</url>
<aliasUrls>https://creativecommons.org/publicdomain/zero/1.0/legalcode</aliasUrls>
<aliasUrls>https://creativecommons.org/publicdomain/zero/1.0/</aliasUrls>
<aliasUrls>http://creativecommons.org/publicdomain/zero/1.0/legalcode</aliasUrls>
<aliasUrls>http://creativecommons.org/publicdomain/zero/1.0/</aliasUrls>
</license>
<license>
<displayName>The 2-Clause BSD License</displayName>
<url>https://opensource.org/licenses/BSD-2-Clause</url>
@@ -87,7 +87,7 @@
<hadoop.version>3.3.1</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.17.1</log4j.version>
<awsjavasdk.version>2.17.79</awsjavasdk.version>
<awsjavasdk.version>2.17.116</awsjavasdk.version>
<parquet.version>1.12.0</parquet.version>
<hadoop-awsjavasdk.version>1.12.109</hadoop-awsjavasdk.version>
<azureblobjavasdk.version>12.14.2</azureblobjavasdk.version>
@@ -955,17 +955,17 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.28</version>
<version>1.7.33</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.8.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>3.8.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -1483,12 +1483,12 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.8.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
<version>3.8.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>com.esri.geometry</groupId>
@@ -1663,7 +1663,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>${azureblobjavasdk.version}</version>
<version>12.14.3</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -1734,7 +1734,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-common</artifactId>
<version>12.14.1</version>
<version>12.14.2</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>

0 comments on commit 90c4a93

Please sign in to comment.