Skip to content
Permalink
Browse files
merge with master
  • Loading branch information
neuyilan committed Jan 11, 2021
2 parents 0b51d8b + bdb5d3e commit 4b7f40d8a7983fb96080ebf2710794fe585c3d3a
Show file tree
Hide file tree
Showing 84 changed files with 1,715 additions and 1,004 deletions.
@@ -0,0 +1,21 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

*
!distribution
@@ -1,41 +1,10 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->

<!-- Thanks for trying to help us make Apache IoTDB be the best it can be!
Please fill out as much of the following information as is possible (where relevant, and remove it
when irrelevant) to help make the intention and scope of this PR clear in order to ease review. -->

## Description

<!-- Describe the goal of this PR, what problem are you fixing. If there is a corresponding issue,
it's not necessary to repeat the description here,
however, you may choose to keep one summary sentence. -->

<!-- Describe your patch: what did you change in code? How did you fix the problem? -->

<!-- If there are several relatively logically separate changes in this PR, create a mini-section
for each of them. For example: -->

### Content1 ...

### Content2 ...

### Content3 ...

<!--
@@ -89,5 +58,3 @@ self-review the PR. -->
<hr>

##### Key changed/added classes (or packages if there are too many classes) in this PR

<!-- Template copied and modified from Apache Druid-->
@@ -0,0 +1,52 @@
# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven

name: E2E Tests

on:
push:
branches:
- master
- 'rel/*'
- test/e2e
paths-ignore:
- 'docs/**'
pull_request:
branches:
- master
- 'rel/*'
- cluster_new
paths-ignore:
- 'docs/**'

jobs:
E2E:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
case:
- cli
steps:
- uses: actions/checkout@v2

- uses: actions/cache@v2
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2

- name: Build Distribution Zip
run: ./mvnw.sh -B -DskipTests clean package

- name: Build Docker Image
run: |
docker build . -f docker/src/main/Dockerfile -t "iotdb:$GITHUB_SHA"
docker images
- name: Run Test Case ${{ matrix.case }}
run: bash test/e2e/cases/${{ matrix.case }}/run.sh

- name: Clean Up
if: ${{ always() }}
run: bash test/e2e/cases/${{ matrix.case }}/cleanup.sh
2 NOTICE
@@ -1,5 +1,5 @@
Apache IoTDB
Copyright 2018-2020 The Apache Software Foundation.
Copyright 2018-2021 The Apache Software Foundation.

This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
@@ -1,5 +1,5 @@
Apache IoTDB
Copyright 2018-2020 The Apache Software Foundation.
Copyright 2018-2021 The Apache Software Foundation.

This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
@@ -93,7 +93,7 @@ IoTDB提供了三种安装方法,您可以参考以下建议,选择最适合

* 从二进制文件安装。推荐的方法是从官方网站下载二进制文件,您将获得一个开箱即用的二进制发布包。

* 使用Docker: dockerfile的路径是https://github.com/apache/incubat-iotdb/tree/master/docker/src/main
* 使用Docker: dockerfile的路径是https://github.com/apache/iotdb/tree/master/docker/src/main

在这篇《快速入门》中,我们简要介绍如何使用源代码安装IoTDB。如需进一步资料,请参阅《用户指南》第3章。

@@ -117,7 +117,7 @@ git checkout release/x.x.x
> mvn clean package -DskipTests
```

执行完成之后,可以在**distribution/target/apache-iotdb-{project.version}-incubating-bin.zip**找到编译完成的二进制版本(包括服务器和客户端)
执行完成之后,可以在**distribution/target/apache-iotdb-{project.version}-bin.zip**找到编译完成的二进制版本(包括服务器和客户端)

> 注意:"thrift/target/generated-sources/thrift" 和 "antlr/target/generated-sources/antlr4" 目录需要添加到源代码根中,以免在 IDE 中产生编译错误。
@@ -86,8 +86,8 @@ public class ClientServer extends TSServiceImpl {

private static final Logger logger = LoggerFactory.getLogger(ClientServer.class);
/**
* The Coordinator of the local node. Through this node ClientServer queries data and meta
* from the cluster and performs data manipulations to the cluster.
* The Coordinator of the local node. Through this node ClientServer queries data and meta from
* the cluster and performs data manipulations to the cluster.
*/
private Coordinator coordinator;

@@ -291,7 +291,7 @@ protected QueryContext genQueryContext(long queryId) {
* @throws StorageEngineException
*/
@Override
protected void releaseQueryResource(long queryId) throws StorageEngineException, IOException {
protected void releaseQueryResource(long queryId) throws StorageEngineException {
// release resources locally
super.releaseQueryResource(queryId);
// release resources remotely
@@ -307,11 +307,11 @@ protected void releaseQueryResource(long queryId) throws StorageEngineException,
try {
if (ClusterDescriptor.getInstance().getConfig().isUseAsyncServer()) {
AsyncDataClient client = coordinator.getAsyncDataClient(queriedNode,
RaftServer.getReadOperationTimeoutMS());
RaftServer.getReadOperationTimeoutMS());
client.endQuery(header, coordinator.getThisNode(), queryId, handler);
} else {
SyncDataClient syncDataClient = coordinator.getSyncDataClient(queriedNode,
RaftServer.getReadOperationTimeoutMS());
RaftServer.getReadOperationTimeoutMS());
syncDataClient.endQuery(header, coordinator.getThisNode(), queryId);
}
} catch (IOException | TException e) {
@@ -19,7 +19,6 @@

package org.apache.iotdb.cluster.integration;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -69,7 +68,7 @@ public void testInsertRecordsWithIllegalPath() throws StatementExecutionExceptio
session.insertRecords(deviceIds, timestamps, allMeasurements, allValues);
fail("Exception expected");
} catch (StatementExecutionException e) {
assertEquals("root..ln1 is not a legal path;root..ln1 is not a legal path;", e.getMessage());
assertTrue(e.getMessage().contains("root..ln1 is not a legal path"));
}

List<String> legalDevices = Arrays.asList("root.sg.ln1", "root.sg3.ln1");
@@ -17,37 +17,27 @@
# under the License.
#

# this docker file shows how to build an IoTDB image from IoTDB source codes.
FROM ubuntu:18.04
# docker build context is the root path of the repository

FROM openjdk:11-jre-slim

ADD distribution/target/apache-iotdb-*-bin.zip /

RUN apt update \
&& apt install wget unzip lsof maven thrift-compiler=0.9.1-2.1 -y \
&& wget https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz -O jdk11.tar.gz \
&& tar -xzf jdk11.tar.gz \
&& rm -rf jdk11.tar.gz \
&& export JAVA_HOME=/jdk-11.0.2/ \
&& export PATH="$JAVA_HOME/bin:$PATH" \
&& wget https://github.com/apache/iotdb/archive/master.zip \
&& unzip master.zip \
&& rm master.zip \
&& cd iotdb-master \
&& mvn package -pl server,client -am -Papache-release -DskipTests -Dthrift.download-url="http://www.apache.org/licenses/LICENSE-2.0.txt" -Dthrift.exec.absolute.path="/usr/bin/thrift" \
&& cd target/ \
&& unzip apache-iotdb-0.12.0-SNAPSHOT-bin.zip \
&& mkdir /iotdb \
&& mv apache-iotdb-0.12.0-SNAPSHOT/* /iotdb/ \
&& cd ../../ \
&& mvn clean \
&& ls -lh ~/.m2 \
&& rm -rf ~/.m2 \
&& rm -rf /iotdb-master \
&& sed -i '119d' /iotdb/conf/logback.xml \
&& apt remove wget maven unzip thrift-compiler -y \
&& apt install lsof procps unzip -y \
&& unzip /apache-iotdb-*-bin.zip -d / \
&& rm /apache-iotdb-*-bin.zip \
&& mv /apache-iotdb-* /iotdb \
&& apt remove unzip -y \
&& apt autoremove -y \
&& apt purge --auto-remove -y \
&& apt clean -y
ENV JAVA_HOME "/jdk-11.0.2"
ENV PATH "$JAVA_HOME/bin:$PATH"
&& apt clean -y

EXPOSE 6667
EXPOSE 31999
EXPOSE 5555
EXPOSE 8181
VOLUME /iotdb/data
VOLUME /iotdb/logs
#ENTRYPOINT ["/iotdb/bin/start-server.sh"]
ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}"
ENTRYPOINT ["/iotdb/sbin/start-server.sh"]
@@ -302,4 +302,5 @@
<logger level="info" name="QUERY_FREQUENCY">
<appender-ref ref="QUERY_FREQUENCY"/>
</logger>
<logger level="info" name="DETAILED_FAILURE_QUERY_TRACE"/>
</configuration>
@@ -98,6 +98,11 @@ public TsFileManagement(String storageGroupName, String storageGroupDir) {
*/
public abstract void add(TsFileResource tsFileResource, boolean sequence);

/**
* add one TsFile to list for recover
*/
public abstract void addRecover(TsFileResource tsFileResource, boolean sequence);

/**
* add some TsFiles to list
*/
@@ -81,6 +81,8 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
private final Map<Long, List<List<TsFileResource>>> unSequenceTsFileResources = new ConcurrentSkipListMap<>();
private final List<List<TsFileResource>> forkedSequenceTsFileResources = new ArrayList<>();
private final List<List<TsFileResource>> forkedUnSequenceTsFileResources = new ArrayList<>();
private final List<TsFileResource> sequenceRecoverTsFileResources = new CopyOnWriteArrayList<>();
private final List<TsFileResource> unSequenceRecoverTsFileResources = new CopyOnWriteArrayList<>();

public LevelCompactionTsFileManagement(String storageGroupName, String storageGroupDir) {
super(storageGroupName, storageGroupDir);
@@ -239,6 +241,19 @@ public void add(TsFileResource tsFileResource, boolean sequence) {
}
}

@Override
public void addRecover(TsFileResource tsFileResource, boolean sequence) {
if (sequence) {
synchronized (sequenceRecoverTsFileResources) {
sequenceRecoverTsFileResources.add(tsFileResource);
}
} else {
synchronized (unSequenceTsFileResources) {
unSequenceRecoverTsFileResources.add(tsFileResource);
}
}
}

@Override
public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) {
for (TsFileResource tsFileResource : tsFileResourceList) {
@@ -349,7 +364,7 @@ public void recover() {
}
if (fullMerge) {
// get tsfile resource from list, as they have been recovered in StorageGroupProcessor
TsFileResource targetTsFileResource = getTsFileResource(targetFile, isSeq);
TsFileResource targetTsFileResource = getRecoverTsFileResource(targetFile, isSeq);
long timePartition = targetTsFileResource.getTimePartition();
RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(target);
// if not complete compaction, resume merge
@@ -371,7 +386,7 @@ public void recover() {
deleteAllSubLevelFiles(isSeq, timePartition);
} else {
// get tsfile resource from list, as they have been recovered in StorageGroupProcessor
TsFileResource targetResource = getTsFileResource(targetFile, isSeq);
TsFileResource targetResource = getRecoverTsFileResource(targetFile, isSeq);
long timePartition = targetResource.getTimePartition();
List<TsFileResource> sourceTsFileResources = new ArrayList<>();
for (String file : sourceFileList) {
@@ -392,13 +407,26 @@ public void recover() {
.merge(targetResource, sourceTsFileResources, storageGroupName,
compactionLogger, deviceSet,
isSeq);
// complete compaction and delete source file
writeLock();
try {
int targetLevel = getMergeLevel(targetResource.getTsFile());
if (isSeq) {
sequenceTsFileResources.get(timePartition).get(targetLevel).add(targetResource);
sequenceRecoverTsFileResources.clear();
} else {
unSequenceTsFileResources.get(timePartition).get(targetLevel).add(targetResource);
unSequenceRecoverTsFileResources.clear();
}
deleteLevelFilesInList(timePartition, sourceTsFileResources, level, isSeq);
} finally {
writeUnlock();
}
deleteLevelFilesInDisk(sourceTsFileResources);
compactionLogger.close();
} else {
writer.close();
}
// complete compaction and delete source file
deleteLevelFilesInDisk(sourceTsFileResources);
deleteLevelFilesInList(timePartition, sourceTsFileResources, level, isSeq);
}
}
} catch (IOException | IllegalPathException e) {
@@ -611,6 +639,25 @@ public static int getMergeLevel(File file) {
return Integer.parseInt(mergeLevelStr);
}

private TsFileResource getRecoverTsFileResource(String filePath, boolean isSeq)
throws IOException {
if (isSeq) {
for (TsFileResource tsFileResource : sequenceRecoverTsFileResources) {
if (Files.isSameFile(tsFileResource.getTsFile().toPath(), new File(filePath).toPath())) {
return tsFileResource;
}
}
} else {
for (TsFileResource tsFileResource : unSequenceRecoverTsFileResources) {
if (Files.isSameFile(tsFileResource.getTsFile().toPath(), new File(filePath).toPath())) {
return tsFileResource;
}
}
}
logger.error("cannot get tsfile resource path: {}", filePath);
throw new IOException();
}

private TsFileResource getTsFileResource(String filePath, boolean isSeq) throws IOException {
if (isSeq) {
for (List<SortedSet<TsFileResource>> tsFileResourcesWithLevel : sequenceTsFileResources
@@ -91,6 +91,11 @@ public void add(TsFileResource tsFileResource, boolean sequence) {
}
}

@Override
public void addRecover(TsFileResource tsFileResource, boolean sequence) {
logger.info("{} do not need to recover", storageGroupName);
}

@Override
public void addAll(List<TsFileResource> tsFileResourceList, boolean sequence) {
if (sequence) {

0 comments on commit 4b7f40d

Please sign in to comment.