Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cherry-pick #1663 to branch RC3 #1669

Merged
merged 2 commits into from
Jan 19, 2020
Merged

cherry-pick #1663 to branch RC3 #1669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a49d7a
fix prefix released before rocksdb iterator (#1663)
critical27 Jan 17, 2020
7d0a12a
Add version info to action job and build docker images for release br…
yixinglu Jan 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@ jobs:
- ubuntu1604
- ubuntu1804
compiler:
- gcc
- clang
- gcc-7.5
- clang-8
exclude:
- tag: centos6
compiler: clang
compiler: clang-8
- tag: centos7
compiler: clang
compiler: clang-8
- tag: ubuntu1604
compiler: clang
compiler: clang-8
container:
image: vesoft/nebula-dev:${{ matrix.tag }}
steps:
Expand All @@ -37,14 +37,29 @@ jobs:
fetch-depth: 1
- name: Prepare environment
run: mkdir -p build
- name: CMake with gcc
if: matrix.compiler == 'gcc'
run: cd build && cmake ..
- name: CMake with clang
if: matrix.compiler == 'clang'
run: cd build && cmake -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_C_COMPILER=clang-8 -DENABLE_ASAN=on -DNEBULA_CLANG_USE_GCC_TOOLCHAIN=/opt/vesoft/toolset/gcc/7.5.0 ..
- name: CMake with gcc-7.5
if: matrix.compiler == 'gcc-7.5'
run: |
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
- name: CMake with clang-8
if: matrix.compiler == 'clang-8'
run: |
cd build
cmake \
-DCMAKE_CXX_COMPILER=clang++-8 \
-DCMAKE_C_COMPILER=clang-8 \
-DENABLE_ASAN=on \
-DNEBULA_CLANG_USE_GCC_TOOLCHAIN=/opt/vesoft/toolset/gcc/7.5.0 \
..
- name: Make
run: cd build && make -j $(nproc)
- name: CTest with single thread
timeout-minutes: 40
run: cd build && ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer ASAN_OPTIONS=fast_unwind_on_malloc=0 ctest --timeout 300 --output-on-failure
run: |
cd build
ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer \
ASAN_OPTIONS=fast_unwind_on_malloc=0 \
ctest \
--timeout 300 \
--output-on-failure
16 changes: 14 additions & 2 deletions .github/workflows/docker.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ name: docker
on:
schedule:
- cron: '0 18 * * *'
push:
branches:
- 'v[0-9]+*'
paths-ignore:
- 'docs/**'
- '**.md'

jobs:
docker:
Expand All @@ -24,9 +30,15 @@ jobs:
- uses: actions/checkout@v1
with:
fetch-depth: 1
- name: build nightly image
- name: Compute tag
id: tag
run: |
[[ "${{ github.event_name }}" == "push" ]] && \
echo "::set-output name=tag::${{ github.ref }}" || \
echo "::set-output name=tag::nightly"
- name: Build image
env:
IMAGE_NAME: ${{ secrets.DOCKER_USERNAME }}/nebula-${{ matrix.service }}:nightly
IMAGE_NAME: ${{ secrets.DOCKER_USERNAME }}/nebula-${{ matrix.service }}:${{ steps.tag.outputs.tag }}
run: |
docker build -t ${IMAGE_NAME} -f docker/Dockerfile.${{ matrix.service }} .
docker push ${IMAGE_NAME}
Expand Down
19 changes: 15 additions & 4 deletions .github/workflows/package.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ name: package
on:
schedule:
- cron: '0 18 * * *'
push:
branches:
- 'v[0-9]+*'
paths-ignore:
- 'docs/**'
- '**.md'

jobs:
package:
Expand All @@ -19,11 +25,16 @@ jobs:
image: vesoft/nebula-dev:${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- name: package
- name: Package
run: ./package/package.sh
shell: bash
- name: Upload nightly artifacts
uses: actions/upload-artifact@v1
- name: Compute version
id: version
run: |
[[ "${{ github.event_name }}" == "push" ]] && \
echo "::set-output name=version::${{ github.ref }}" || \
echo "::set-output name=version::nightly"
- uses: actions/upload-artifact@v1
with:
name: ${{ matrix.os }}-nightly
name: ${{ matrix.os }}-${{ steps.version.outputs.version }}
path: build/cpack_output
45 changes: 34 additions & 11 deletions .github/workflows/pull_request.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
types: [synchronize, reopened, labeled]
branches:
- master
- 'v[0-9]+*'

jobs:
lint:
Expand All @@ -30,11 +31,11 @@ jobs:
- centos7
- ubuntu1804
compiler:
- gcc
- clang
- gcc-7.5
- clang-8
exclude:
- tag: centos7
compiler: clang
compiler: clang-8
container:
image: vesoft/nebula-dev:${{ matrix.tag }}
env:
Expand All @@ -53,23 +54,45 @@ jobs:
run: |
[ -f "$CCACHE_DIR/ccache.conf" ] || cp ci/ccache.conf "$CCACHE_DIR"
[ -d build/ ] && rm -rf build/* || mkdir -p build
- name: CMake with gcc
if: matrix.compiler == 'gcc' && steps.ignore_docs.outputs.num_source_files != 0
run: cd build && cmake ..
- name: CMake with clang
if: matrix.compiler == 'clang' && steps.ignore_docs.outputs.num_source_files != 0
run: cd build && cmake -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_C_COMPILER=clang-8 -DENABLE_ASAN=on -DNEBULA_CLANG_USE_GCC_TOOLCHAIN=/opt/vesoft/toolset/gcc/7.5.0 ..
- name: CMake with gcc-7.5
if: matrix.compiler == 'gcc-7.5' && steps.ignore_docs.outputs.num_source_files != 0
run: |
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
- name: CMake with clang-8
if: matrix.compiler == 'clang-8' && steps.ignore_docs.outputs.num_source_files != 0
run: |
cd build
cmake \
-DCMAKE_CXX_COMPILER=clang++-8 \
-DCMAKE_C_COMPILER=clang-8 \
-DENABLE_ASAN=on \
-DNEBULA_CLANG_USE_GCC_TOOLCHAIN=/opt/vesoft/toolset/gcc/7.5.0 \
..
- name: Make
if: steps.ignore_docs.outputs.num_source_files != 0
run: cd build && make -j $(nproc)
- name: CTest with multiple threads
if: steps.ignore_docs.outputs.num_source_files != 0
timeout-minutes: 15
run: cd build && ctest -j 6 --timeout 300 --output-on-failure --rerun-failed
run: |
cd build
ctest \
-j 6 \
--timeout 300 \
--output-on-failure \
--rerun-failed
- name: CTest with single thread
if: failure() && steps.ignore_docs.outputs.num_source_files != 0
timeout-minutes: 30
run: cd build && ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer ASAN_OPTIONS=fast_unwind_on_malloc=0 ctest --timeout 300 --output-on-failure --rerun-failed
run: |
cd build
ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer \
ASAN_OPTIONS=fast_unwind_on_malloc=0 \
ctest \
--timeout 300 \
--output-on-failure \
--rerun-failed
- name: Cleanup
if: always()
run: rm -rf build
9 changes: 6 additions & 3 deletions src/kvstore/NebulaStore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,12 +413,15 @@ ResultCode NebulaStore::rangeWithPrefix(GraphSpaceID spaceId,
const std::string& start,
const std::string& prefix,
std::unique_ptr<KVIterator>* iter) {
auto ret = engine(spaceId, partId);
auto ret = part(spaceId, partId);
if (!ok(ret)) {
return error(ret);
}
auto* e = nebula::value(ret);
return e->rangeWithPrefix(start, prefix, iter);
auto part = nebula::value(ret);
if (!checkLeader(part)) {
return ResultCode::ERR_LEADER_CHANGED;
}
return part->engine()->rangeWithPrefix(start, prefix, iter);
}


Expand Down
21 changes: 17 additions & 4 deletions src/storage/BaseProcessor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,29 @@ class BaseProcessor {
void doRemoveRange(GraphSpaceID spaceId, PartitionID partId, std::string start,
std::string end);

kvstore::ResultCode doRange(GraphSpaceID spaceId, PartitionID partId, std::string start,
std::string end, std::unique_ptr<kvstore::KVIterator>* iter);
kvstore::ResultCode doRange(GraphSpaceID spaceId, PartitionID partId, const std::string& start,
const std::string& end, std::unique_ptr<kvstore::KVIterator>* iter);

kvstore::ResultCode doRange(GraphSpaceID spaceId, PartitionID partId,
std::string&& start, std::string&& end,
std::unique_ptr<kvstore::KVIterator>* iter) = delete;

kvstore::ResultCode doPrefix(GraphSpaceID spaceId, PartitionID partId,
const std::string& prefix,
std::unique_ptr<kvstore::KVIterator>* iter);

kvstore::ResultCode doPrefix(GraphSpaceID spaceId, PartitionID partId,
std::string prefix, std::unique_ptr<kvstore::KVIterator>* iter);
std::string prefix,
std::unique_ptr<kvstore::KVIterator>* iter) = delete;

kvstore::ResultCode doRangeWithPrefix(GraphSpaceID spaceId, PartitionID partId,
std::string start, std::string prefix,
const std::string& start, const std::string& prefix,
std::unique_ptr<kvstore::KVIterator>* iter);

kvstore::ResultCode doRangeWithPrefix(GraphSpaceID spaceId, PartitionID partId,
std::string&& start, std::string&& prefix,
std::unique_ptr<kvstore::KVIterator>* iter) = delete;

nebula::cpp2::ColumnDef columnDef(std::string name, nebula::cpp2::SupportedType type) {
nebula::cpp2::ColumnDef column;
column.set_name(std::move(name));
Expand Down
10 changes: 5 additions & 5 deletions src/storage/BaseProcessor.inl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,24 +86,24 @@ void BaseProcessor<RESP>::doRemoveRange(GraphSpaceID spaceId,
template<typename RESP>
kvstore::ResultCode BaseProcessor<RESP>::doRange(GraphSpaceID spaceId,
PartitionID partId,
std::string start,
std::string end,
const std::string& start,
const std::string& end,
std::unique_ptr<kvstore::KVIterator>* iter) {
return kvstore_->range(spaceId, partId, start, end, iter);
}

template<typename RESP>
kvstore::ResultCode BaseProcessor<RESP>::doPrefix(GraphSpaceID spaceId,
PartitionID partId,
std::string prefix,
const std::string& prefix,
std::unique_ptr<kvstore::KVIterator>* iter) {
return kvstore_->prefix(spaceId, partId, prefix, iter);
}

template<typename RESP>
kvstore::ResultCode BaseProcessor<RESP>::doRangeWithPrefix(
GraphSpaceID spaceId, PartitionID partId, std::string start, std::string prefix,
std::unique_ptr<kvstore::KVIterator>* iter) {
GraphSpaceID spaceId, PartitionID partId, const std::string& start,
const std::string& prefix, std::unique_ptr<kvstore::KVIterator>* iter) {
return kvstore_->rangeWithPrefix(spaceId, partId, start, prefix, iter);
}

Expand Down