Skip to content

nightly

nightly #1051

Workflow file for this run

name: nightly
on:
schedule:
- cron: "0 18 * * *"
concurrency:
group: nightly
cancel-in-progress: true
defaults:
run:
shell: bash
env:
OSS_DIR: minio/nightly-build/nebula-graph
jobs:
package:
name: build package
runs-on: [self-hosted, nebula]
strategy:
fail-fast: false
matrix:
os:
- ubuntu1604
- ubuntu1804
- ubuntu2004
# - centos6
- centos7
- centos8
container:
image: vesoft/nebula-dev:${{ matrix.os }}
steps:
- uses: webiny/action-post-run@3.0.0
with:
run: sh -c "find . -mindepth 1 -delete"
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "^1.16.7"
- name: package
run: ./package/package.sh -t Release -r ON -p OFF -s FALSE -k OFF
- name: output some vars
id: vars
env:
SHA_EXT: sha256sum.txt
run: |
find pkg-build/cpack_output -type f \( -iname \*.deb -o -iname \*.rpm -o -iname \*.tar.gz \) -exec bash -c "sha256sum {} > {}.sha256sum.txt" \;
subdir=$(date -u +%Y.%m.%d)
echo "subdir=$subdir" >> $GITHUB_OUTPUT
# - uses: actions/upload-artifact@v3
# with:
# name: ${{ matrix.os }}-nightly
# path: pkg-build/cpack_output
- uses: vesoft-inc/.github/actions/setup-minio@master
with:
minio_url: ${{ secrets.MINIO_ENDPOINT }}
access_key: ${{ secrets.MINIO_KEY }}
secret_key: ${{ secrets.MINIO_SECRET }}
- name: Copy dir to MinIO
run: mc cp -r pkg-build/cpack_output/ ${{ env.OSS_DIR }}/${{ steps.vars.outputs.subdir }}/
docker:
name: build docker image
needs: package
runs-on: [self-hosted, nebula]
steps:
- uses: webiny/action-post-run@3.0.0
with:
run: sh -c "find . -mindepth 1 -delete"
- uses: actions/checkout@v3
- uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- uses: docker/setup-qemu-action@v2
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v4
with:
context: .
file: ./docker/Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
vesoft/nebula-graphd:nightly
target: graphd
cache-to: type=local,dest=/tmp/buildx-cache,mode=max
push: true
- uses: docker/build-push-action@v4
with:
context: .
file: ./docker/Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
vesoft/nebula-storaged:nightly
target: storaged
cache-from: type=local,src=/tmp/buildx-cache
push: true
- uses: docker/build-push-action@v4
with:
context: .
file: ./docker/Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
vesoft/nebula-metad:nightly
target: metad
cache-from: type=local,src=/tmp/buildx-cache
push: true
- uses: docker/build-push-action@v4
with:
context: .
file: ./docker/Dockerfile
platforms: linux/amd64,linux/arm64
tags: |
vesoft/nebula-tools:nightly
target: tools
cache-from: type=local,src=/tmp/buildx-cache
push: true
- name: delete the cache
run: |
rm -rf /tmp/buildx-cache
coverage:
name: coverage
runs-on: [self-hosted, nebula]
strategy:
fail-fast: false
matrix:
os:
- centos7
compiler:
- gcc-9.3
container:
image: vesoft/nebula-dev:${{ matrix.os }}
env:
CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
CCACHE_MAXSIZE: 8G
volumes:
- /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
options: --cap-add=SYS_PTRACE
steps:
- uses: webiny/action-post-run@3.0.0
with:
run: sh -c "find . -mindepth 1 -delete"
- uses: actions/checkout@v3
- name: Prepare environment
run: |
[ -d build/ ] && rm -rf build/* || mkdir -p build
make init -C tests
- name: CMake
run: |
cmake \
-DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \
-DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \
-DCMAKE_BUILD_TYPE=Debug \
-DENABLE_TESTING=on \
-DENABLE_COVERAGE=on \
-GNinja \
-B build
- name: Make
run: |
ccache -z
ninja -j $(($(nproc)/2+1))
ccache -s
working-directory: build/
- name: CTest
env:
ASAN_OPTIONS: fast_unwind_on_malloc=1
run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure
working-directory: build/
timeout-minutes: 20
- name: Setup cluster
run: |
make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up
working-directory: tests/
timeout-minutes: 2
- name: Pytest
run: |
make RM_DIR=false DEBUG=false J=8 test
working-directory: tests/
timeout-minutes: 15
- name: TCK
run: |
make RM_DIR=false DEBUG=false J=8 tck
working-directory: tests/
timeout-minutes: 60
- name: Down cluster
run: |
make RM_DIR=false down
working-directory: tests/
timeout-minutes: 2
- name: coverage
run: |
~/.local/bin/fastcov -d build -l -o fastcov.info -p --exclude /usr/include /usr/lib /opt/vesoft build/ tests/ /test /mock .lex .yy
- uses: codecov/codecov-action@v3
with:
files: fastcov.info
fail_ci_if_error: false
- name: Upload logs
uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
path: ./build/server_*/logs/
test:
name: Tck test
needs: package
runs-on: [self-hosted, nebula]
strategy:
fail-fast: false
matrix:
os:
- ubuntu1804
- ubuntu2004
- centos7
- centos8
extra_config:
- "ENABLE_SSL=true CA_SIGNED=true QUERY_CONCURRENTLY=false"
- "ENABLE_SSL=false CA_SIGNED=false QUERY_CONCURRENTLY=true"
container:
image: vesoft/nebula-dev:${{ matrix.os }}
services:
elasticsearch:
image: elasticsearch:7.17.7
ports:
- 9200:9200
- 9300:9300
env:
discovery.type: single-node
options: >-
--health-cmd "curl elasticsearch:9200"
--health-interval 10s
--health-timeout 5s
--health-retries 10
steps:
- uses: webiny/action-post-run@3.0.0
with:
run: sh -c "find . -mindepth 1 -delete"
- uses: actions/checkout@v3
- name: output some vars
id: vars
env:
SHA_EXT: sha256sum.txt
run: |
subdir=$(date -u +%Y.%m.%d)
echo "subdir=$subdir" >> $GITHUB_OUTPUT
- id: oss_package
run: |
case ${{ matrix.os }} in
ubuntu1804)
echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.ubuntu1804.amd64.tar.gz" >> $GITHUB_OUTPUT
;;
ubuntu2004)
echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.ubuntu2004.amd64.tar.gz" >> $GITHUB_OUTPUT
;;
centos7)
echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.el7.x86_64.tar.gz" >> $GITHUB_OUTPUT
;;
centos8)
echo "p=nebula-graph-${{ steps.vars.outputs.subdir }}-nightly.el8.x86_64.tar.gz" >> $GITHUB_OUTPUT
;;
esac
- name: Prepare environment
id: prepare
run: |
[ -d build/ ] && rm -rf build/* || mkdir -p build
make init -C tests
- name: CMake
id: cmake
run: |
echo "j=8" >> $GITHUB_OUTPUT
- uses: vesoft-inc/.github/actions/setup-minio@master
with:
minio_url: ${{ secrets.MINIO_ENDPOINT }}
access_key: ${{ secrets.MINIO_KEY }}
secret_key: ${{ secrets.MINIO_SECRET }}
- name: Copy dir to MinIO
run: |
mc cp ${{ env.OSS_DIR }}/${{ steps.vars.outputs.subdir }}/${{ steps.oss_package.outputs.p }} build/
tar zxvf build/${{ steps.oss_package.outputs.p }} -C build
d=`echo ${{ steps.oss_package.outputs.p }} | sed 's/.tar.gz//'`
mv build/${d}/* build/.
- name: Setup cluster
run: |
make CONTAINERIZED=true ${{ matrix.extra_config }} up
working-directory: tests/
timeout-minutes: 2
- name: Pytest
run: |
make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test
working-directory: tests/
timeout-minutes: 15
- name: TCK
run: |
make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
working-directory: tests/
timeout-minutes: 60
- name: Down cluster
run: |
make RM_DIR=false down
working-directory: tests/
timeout-minutes: 2
- name: Upload logs
uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
path: ./build/server_*/logs*/
standalone:
name: standalone-build-tck
runs-on: [self-hosted, nebula]
strategy:
fail-fast: false
matrix:
os:
- centos7
compiler:
- gcc-9.3
env:
CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
CCACHE_MAXSIZE: 8G
container:
image: vesoft/nebula-dev:${{ matrix.os }}
volumes:
- /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
options: --cap-add=SYS_PTRACE
steps:
- uses: webiny/action-post-run@3.0.0
with:
run: sh -c "find . -mindepth 1 -delete"
- uses: actions/checkout@v3
- name: Prepare environment
id: prepare
run: |
[ -d build/ ] && rm -rf build/* || mkdir -p build
make init -C tests
- name: CMake
id: cmake
run: |
case ${{ matrix.compiler }} in
gcc-*)
case ${{ matrix.os }} in
centos7)
# build with Release type
cmake \
-DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \
-DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_TESTING=on \
-DENABLE_STANDALONE_VERSION=on \
-GNinja \
-B build
echo "::set-output name=j::8"
;;
esac
;;
esac
- name: Make
run: |
ccache -z
ninja -j $(nproc)
ccache -s
working-directory: build/
- name: CTest
env:
ASAN_OPTIONS: fast_unwind_on_malloc=1
run: |
ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -LE segment_id_test
ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure -L segment_id_test
working-directory: build/
timeout-minutes: 20
- name: Setup Cluster
run: |
make standalone-up
working-directory: tests/
timeout-minutes: 4
- name: TCK
run: |
make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} standalone-tck
working-directory: tests/
timeout-minutes: 60
- name: LDBC
run: |
make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc
working-directory: tests/
timeout-minutes: 60
- name: Down cluster
run: |
make RM_DIR=false down
working-directory: tests/
timeout-minutes: 2
- name: Upload logs
uses: actions/upload-artifact@v2
if: ${{ failure() }}
with:
name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
path: ./build/server_*/logs/