Skip to content
Permalink
Browse files
Merge branch 'gerrit/stabilization-5e11053887'
Change-Id: I63dbfc59393105b7b1c58dcae2efee8ca88314b0
  • Loading branch information
mblow committed Mar 15, 2022
2 parents 5e11053 + 9679811 commit b1add3a0d68f3d77cf173934a823c0a9eecfacca
Showing 30 changed files with 1,079 additions and 384 deletions.
@@ -587,6 +587,16 @@
<artifactId>hadoop-hdfs</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -715,6 +725,16 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -730,10 +750,6 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-btree-test</artifactId>
@@ -812,6 +828,18 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
<groupId>org.mindrot</groupId>
<artifactId>jbcrypt</artifactId>
@@ -117,7 +117,7 @@
<compilation-unit name="common/bucket-does-not-exist">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">common/bucket-does-not-exist</output-dir>
<expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null)</expected-error>
<expected-error>External source error. The specified bucket does not exist (Service: S3, Status Code: 404, Request ID: null, Extended Request ID: null)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset" check-warnings="true">
@@ -10165,6 +10165,11 @@
<output-dir compare="Text">substring_multi_codepoint_01</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="string">
<compilation-unit name="substr-ASTERIXDB-2949">
<output-dir compare="Text">substr-ASTERIXDB-2949</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="string">
<compilation-unit name="substring-after-1">
<output-dir compare="Text">substring-after-1</output-dir>
@@ -161,7 +161,7 @@
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<artifactId>netty-codec-http</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -45,14 +45,14 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
</dependency>
</dependencies>

<profiles>
@@ -207,6 +207,9 @@
<usedDependency>com.sun.xml.bind:jaxb-impl</usedDependency>
<usedDependency>com.sun.activation:javax.activation</usedDependency>
<usedDependency>net.razorvine:serpent</usedDependency>
<usedDependency>io.netty:netty-resolver-dns</usedDependency>
<usedDependency>io.netty:netty-codec-http2</usedDependency>
<usedDependency>io.netty:netty-transport-native-unix-common</usedDependency>
</usedDependencies>
</configuration>
</plugin>
@@ -434,7 +437,11 @@
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<artifactId>netty-buffer</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -484,5 +491,17 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver-dns</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
</dependency>
</dependencies>
</project>
@@ -45,7 +45,6 @@
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>3.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -149,7 +149,6 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -171,8 +171,21 @@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
<gav>io.netty:netty-all:4.1.63.Final</gav>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.63.Final/NOTICE.txt</noticeUrl>
<gavs>
<gav>io.netty:netty-buffer:4.1.69.Final</gav>
<gav>io.netty:netty-codec:4.1.69.Final</gav>
<gav>io.netty:netty-codec-http:4.1.69.Final</gav>
<gav>io.netty:netty-common:4.1.69.Final</gav>
<gav>io.netty:netty-handler:4.1.69.Final</gav>
<gav>io.netty:netty-resolver:4.1.69.Final</gav>
<gav>io.netty:netty-transport:4.1.69.Final</gav>
</gavs>
<noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.69.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>io.netty:netty-tcnative-classes:2.0.46.Final</gav>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
<noticeUrl>https://raw.githubusercontent.com/netty/netty-tcnative/netty-tcnative-parent-2.0.46.Final/NOTICE.txt</noticeUrl>
</override>
<override>
<gav>org.reactivestreams:reactive-streams:1.0.2</gav>
@@ -205,14 +218,15 @@
</override>
<override>
<gavs>
<gav>com.azure:azure-core:1.17.0</gav>
<gav>com.azure:azure-core-http-netty:1.10.0</gav>
<gav>com.azure:azure-storage-blob:12.12.0</gav>
<gav>com.azure:azure-storage-common:12.12.0</gav>
<gav>com.azure:azure-storage-internal-avro:12.0.5</gav>
<gav>com.azure:azure-core:1.24.1</gav>
<gav>com.azure:azure-core-http-netty:1.11.6</gav>
<gav>com.azure:azure-storage-blob:12.14.3</gav>
<gav>com.azure:azure-storage-common:12.14.2</gav>
<gav>com.azure:azure-storage-internal-avro:12.1.3</gav>
</gavs>
<noticeUrl>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/master/NOTICE.txt</noticeUrl>
<url>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/master/LICENSE.txt</url>
<!-- azure-storage-blob-batch_12.11.3 release is at 89a32290750a18d1b99c27c16b1b11d42f16c622 -->
<noticeUrl>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/89a32290750a18d1b99c27c16b1b11d42f16c622/NOTICE.txt</noticeUrl>
<url>https://raw.githubusercontent.com/Azure/azure-sdk-for-java/89a32290750a18d1b99c27c16b1b11d42f16c622/LICENSE.txt</url>
</override>
<override>
<gav>org.mindrot:jbcrypt:0.4</gav>
@@ -322,6 +336,13 @@
<url>PUBLIC_DOMAIN</url>
<aliasUrls>Public Domain</aliasUrls>
</license>
<license>
<url>https://creativecommons.org/publicdomain/zero/1.0/legalcode.txt</url>
<aliasUrls>https://creativecommons.org/publicdomain/zero/1.0/legalcode</aliasUrls>
<aliasUrls>https://creativecommons.org/publicdomain/zero/1.0/</aliasUrls>
<aliasUrls>http://creativecommons.org/publicdomain/zero/1.0/legalcode</aliasUrls>
<aliasUrls>http://creativecommons.org/publicdomain/zero/1.0/</aliasUrls>
</license>
</licenses>
<templateProperties>
<packageName>Apache AsterixDB Server Install</packageName>
@@ -57,7 +57,6 @@
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -93,41 +92,10 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
private static final FilenameFilter MASK_FILES_FILTER =
(dir, name) -> name.startsWith(StorageConstants.MASK_FILE_PREFIX);
private static final int MAX_CACHED_RESOURCES = 1000;
private static final IOFileFilter METADATA_FILES_FILTER = new IOFileFilter() {
@Override
public boolean accept(File file) {
return file.getName().equals(StorageConstants.METADATA_FILE_NAME);
}

@Override
public boolean accept(File dir, String name) {
return false;
}
};

private static final IOFileFilter METADATA_MASK_FILES_FILTER = new IOFileFilter() {
@Override
public boolean accept(File file) {
return file.getName().equals(METADATA_FILE_MASK_NAME);
}

@Override
public boolean accept(File dir, String name) {
return false;
}
};

private static final IOFileFilter ALL_DIR_FILTER = new IOFileFilter() {
@Override
public boolean accept(File file) {
return true;
}

@Override
public boolean accept(File dir, String name) {
return true;
}
};
private static final FilenameFilter METADATA_FILES_FILTER =
(dir, name) -> name.equals(StorageConstants.METADATA_FILE_NAME);
private static final FilenameFilter METADATA_MASK_FILES_FILTER =
(dir, name) -> name.equals(METADATA_FILE_MASK_NAME);

// Finals
private final IIOManager ioManager;
@@ -258,7 +226,10 @@ public synchronized Map<Long, LocalResource> getResources(Predicate<LocalResourc
throws HyracksDataException {
Map<Long, LocalResource> resourcesMap = new HashMap<>();
for (Path root : storageRoots) {
final Collection<File> files = FileUtils.listFiles(root.toFile(), METADATA_FILES_FILTER, ALL_DIR_FILTER);
if (!Files.exists(root) || !Files.isDirectory(root)) {
continue;
}
final Collection<File> files = IoUtil.getMatchingFiles(root, METADATA_FILES_FILTER);
try {
for (File file : files) {
final LocalResource localResource = readLocalResource(file);
@@ -463,8 +434,7 @@ public List<ResourceStorageStats> getStorageStats() throws HyracksDataException

public void deleteCorruptedResources() throws HyracksDataException {
for (Path root : storageRoots) {
final Collection<File> metadataMaskFiles =
FileUtils.listFiles(root.toFile(), METADATA_MASK_FILES_FILTER, ALL_DIR_FILTER);
final Collection<File> metadataMaskFiles = IoUtil.getMatchingFiles(root, METADATA_MASK_FILES_FILTER);
for (File metadataMaskFile : metadataMaskFiles) {
final File resourceFile = new File(metadataMaskFile.getParent(), METADATA_FILE_NAME);
if (resourceFile.exists()) {
@@ -86,9 +86,8 @@
<hyracks.version>0.3.6-SNAPSHOT</hyracks.version>
<hadoop.version>2.8.5</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.14.1</log4j.version>
<awsjavasdk.version>2.10.83</awsjavasdk.version>
<azurejavasdk.version>12.12.0</azurejavasdk.version>
<log4j.version>2.17.1</log4j.version>
<awsjavasdk.version>2.17.116</awsjavasdk.version>
<parquet.version>1.8.2</parquet.version>

<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
@@ -945,17 +944,17 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.28</version>
<version>1.7.33</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.6.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>3.6.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -1389,7 +1388,7 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.5.4</version>
<version>8.5.6</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
@@ -1414,12 +1413,12 @@
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.6.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
<version>3.6.3</version>
<version>3.8.4</version>
</dependency>
<dependency>
<groupId>com.esri.geometry</groupId>
@@ -1523,7 +1522,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>${azurejavasdk.version}</version>
<version>12.14.3</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -1594,7 +1593,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-common</artifactId>
<version>${azurejavasdk.version}</version>
<version>12.14.2</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>

0 comments on commit b1add3a

Please sign in to comment.