diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml index c8501ad3..e49d7f0a 100644 --- a/.github/workflows/3.0-non-x64.yml +++ b/.github/workflows/3.0-non-x64.yml @@ -133,7 +133,7 @@ jobs: apt install -y cmake build-essential git libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config libssl-dev > /dev/null echo "clone TDengine 3.0 on ${{ steps.setup.outputs.uname }}" - git clone --branch chore/sangshuduo/TD-19590-sanitizer-align --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 + git clone --branch 3.0 --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 echo "build TDengine 3.0 on ${{ steps.setup.outputs.uname }}" cd TDengine || exit 1 diff --git a/.github/workflows/3.0-taosBenchmark-debug.yml b/.github/workflows/3.0-taosBenchmark-debug.yml new file mode 100644 index 00000000..9ef6eee4 --- /dev/null +++ b/.github/workflows/3.0-taosBenchmark-debug.yml @@ -0,0 +1,217 @@ +name: Ubuntu (3.0 taosbenchmark debug) + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + +env: + TOOLS_BUILD_TYPE: Debug + PR_NUMBER: ${{ github.event.number }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Step that prints name of pull request's base branch + run: | + echo "Pull request's base branch is: ${BASE_BRANCH}" + echo "Pull request's branch is: ${GITHUB_REF##*/}" + echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" + env: + BASE_BRANCH: ${{ github.base_ref }} + if: github.event_name == 'pull_request' + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v23.2 + + - name: List all changed files + run: | + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do + echo "$file was changed" + done + + - name: Get specific changed files + id: changed-files-specific + uses: tj-actions/changed-files@v23.2 + with: + files: | + src/bench* + src/CMakeLists.txt + inc/* + tests/taosbenchmark/* + tests/taosbenchmark/ws/* + packaging/tools/* + .github/workflows/3.0-taosBenchmark-debug.yml + + - name: Run step if any of the listed files above change + if: steps.changed-files-specific.outputs.any_changed == 'true' + run: | + echo "One or more files listed above has changed." + + - name: Checkout TDengine 3.0 + uses: actions/checkout@v2 + with: + repository: 'taosdata/TDengine' + path: 'TDengine' + ref: '3.0' + + - name: Change time zone + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + sudo timedatectl set-timezone Asia/Shanghai + sudo date + + - name: Set up Go + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + uses: actions/setup-go@v3 + with: + go-version: 1.14 + + - name: Set up Rust + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - uses: actions/cache@v3 + id: cache-rust + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ steps.setup-rust.outputs.rustc_hash }} + + - name: Build & Install TDengine + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] + then + echo "TDengine build failure" + exit 1 + fi + + - name: Install packages for build taos-tools + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + sudo apt update > /dev/null + sudo apt install lzma-dev liblzma-dev libjansson-dev libz-dev libsnappy-dev pkg-config expect -y > /dev/null + + - name: Checkout taos-tools to PR number + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + run: | + git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge + git checkout -qf FETCH_HEAD + + - name: Build taos-tools + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + mkdir debug ||: + cd debug + cmake .. -DTOOLS_BUILD_TYPE=Debug -DTOOLS_SANITIZE=true -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] + then + echo "taos-tools build failure" + exit 1 + fi + + - name: Install python packages + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + pip3 install numpy fabric2 psutil pandas faker toml > /dev/null 2>&1 + pip3 install git+https://github.com/taosdata/taos-connector-python > /dev/null + + - name: Test + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd tests + + if find taosbenchmark -name "*.py"|grep -q .; + then + for i in `find taosbenchmark -name "*.py"|grep -Ev "websocket|taosdemo|kill-slow-query"|sort`; do printf $i && python3 ./test.py -f $i > /dev/null && echo -e " \033[32m taosbenchmark-release-test/$i success! \033[0m"|| echo -e " \033[31m taosbenchmark-release-test/$i failed! \033[0m" | tee -a ~/taosbenchmark-release-failed.txt; done + fi + + - name: Test WebSocket + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + #cd tests + + #if find taosbenchmark/ws -name "*.py"|grep -q .; + #then + # for i in `find taosbenchmark/ws -name "*.py"`; do printf $i && python3 ./test.py -f $i > /dev/null && echo -e " \033[32m benchmark-ws-test:$i success! \033[0m"|| echo -e " \033[31m benchmark-ws-test:$i failed! \033[0m" | tee -a ~/benchmark-ws-failed.txt; done + #fi + + - name: Test taosdemo query + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd tests + + if find taosbenchmark -name "taosdemo*.py"|grep -q .; + then + for i in `find taosbenchmark -name "taosdemo*.py"`; do printf $i && python3 ./test.py -f $i && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosbenchmark-release-failed.txt; done + fi + + - name: 'Upload taosd logs' + uses: actions/upload-artifact@v3 + with: + name: log-v3-taosd + path: /var/log/taos/ + retention-days: 2 + + - name: Check Test Result + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + if [ -f ~/taosbenchmark-release-failed.txt ];then + cat ~/taosbenchmark-release-failed.txt + exit 1 + fi + + if [ -f ~/benchmark-ws-failed.txt ];then + cat ~/benchmark-ws-failed.txt + exit 1 + fi diff --git a/.github/workflows/3.0-taosBenchmark-release.yml b/.github/workflows/3.0-taosBenchmark-release.yml index 63213e41..0d5053bf 100644 --- a/.github/workflows/3.0-taosBenchmark-release.yml +++ b/.github/workflows/3.0-taosBenchmark-release.yml @@ -44,7 +44,9 @@ jobs: with: files: | src/bench* + src/CMakeLists.txt inc/* + tests/taosbenchmark/* tests/taosbenchmark/ws/* packaging/tools/* .github/workflows/3.0-taosBenchmark-release.yml diff --git a/.github/workflows/3.0-taosdump-debug.yml b/.github/workflows/3.0-taosdump-debug.yml new file mode 100644 index 00000000..ffd4d563 --- /dev/null +++ b/.github/workflows/3.0-taosdump-debug.yml @@ -0,0 +1,188 @@ +name: Ubuntu (3.0 taosdump native debug) + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + +env: + TOOLS_BUILD_TYPE: Debug + PR_NUMBER: ${{ github.event.number }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Step that prints name of pull request's base branch + run: | + echo "Pull request's base branch is: ${BASE_BRANCH}" + echo "Pull request's branch is: ${GITHUB_REF##*/}" + echo "Pull request's head ref is: ${GITHUB_HEAD_REF}" + env: + BASE_BRANCH: ${{ github.base_ref }} + if: github.event_name == 'pull_request' + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v23.2 + + - name: List all changed files + run: | + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do + echo "$file was changed" + done + + - name: Get specific changed files + id: changed-files-specific + uses: tj-actions/changed-files@v23.2 + with: + files: | + src/CMakeLists.txt + src/taosdump* + tests/taosdump/native/* + inc/* + deps/CMakeLists.txt + packaging/tools/* + .github/workflows/3.0-taosdump-debug.yml + + - name: Run step if any of the listed files above change + if: steps.changed-files-specific.outputs.any_changed == 'true' + run: | + echo "One or more files listed above has changed." + + - name: Checkout tdengine + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + uses: actions/checkout@v2 + with: + repository: 'taosdata/TDengine' + path: 'TDengine' + ref: '3.0' + + - name: Change time zone + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + sudo timedatectl set-timezone Asia/Shanghai + sudo date + + - name: Set up Go + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + uses: actions/setup-go@v3 + with: + go-version: 1.14 + + - name: first build TDengine + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd TDengine + mkdir debug && cd debug + cmake .. -DBUILD_HTTP=false > /dev/null + make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] + then + echo "TDengine build failure" + exit 1 + fi + + - name: Install packages for build taos-tools + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: sudo apt-get install lzma-dev liblzma-dev libjansson-dev libz-dev libsnappy-dev pkg-config lcov expect -y > /dev/null + + - name: Checkout taos-tools to PR number + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + run: | + git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge + git checkout -qf FETCH_HEAD + + - name: Build taos-tools + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + mkdir debug ||: + cd debug + cmake .. -DTOOLS_BUILD_TYPE=Debug -DTOOLS_SANITIZE=true > /dev/null + make -j8 > /dev/null && sudo make install > /dev/null + if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] + then + echo "taos-tools build failure" + exit 1 + fi + + - name: Install python packages + if: | + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + pip3 install numpy fabric2 psutil pandas faker toml > /dev/null + pip3 install git+https://github.com/taosdata/taos-connector-python > /dev/null + + - name: taosdump old Test + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd tests + sudo pkill -9 taosadapter || : + sudo pkill -9 taosd || : + + echo "TEST: old cases" + python3 ./test.py -f taosdump/old/taosdumpTest.py > /dev/null && echo -e "\033[32m taosdumpTest.py success! \033[0m"|| echo -e "\033[31m taosdumpTest.py failed! \033[0m" | tee -a ~/taosdump-failed.txt + python3 ./test.py -f taosdump/old/taosdumpTest2.py > /dev/null && echo -e "\033[32m taosdumpTest2.py success! \033[0m"|| echo -e "\033[31m taosdumpTest2.py failed! \033[0m" | tee -a ~/taosdump-failed.txt + python3 ./test.py -f taosdump/old/taosdumpTestNanoSupport.py > /dev/null && echo -e "\033[32m taosdumpTestNanoSupport.py success! \033[0m"|| echo -e "\033[31m taosdumpTestNanoSupport.py failed! \033[0m" | tee -a ~/taosdump-failed.txt + + + - name: taosdump Develop Test + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + cd tests + sudo pkill -9 taosadapter || : + sudo pkill -9 taosd || : + sleep 5 + + echo "TEST: develop test cases" + if find taosdump/native -name "*.py"|grep -q .; + then + for i in `find taosdump/native -name "*.py" | grep -Ev "Float|Double"`; do python3 ./test.py -f $i > /dev/null && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt ;done + fi + + for i in `find taosdump/native -name "*.py" | grep "Float|Double"`; do python3 ./test.py -f $i > /dev/null && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" ;done + + - name: Check Test Result + if: + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + run: | + if [ -f ~/taosdump-failed.txt ];then + cat ~/taosdump-failed.txt; + exit 1; + fi diff --git a/.github/workflows/3.0-taosdump-release.yml b/.github/workflows/3.0-taosdump-release.yml index 6ef643ae..7fe331cb 100644 --- a/.github/workflows/3.0-taosdump-release.yml +++ b/.github/workflows/3.0-taosdump-release.yml @@ -49,7 +49,7 @@ jobs: inc/* deps/CMakeLists.txt packaging/tools/* - .github/workflows/*taosdump* + .github/workflows/3.0-taosdump-release.yml - name: Run step if any of the listed files above change if: steps.changed-files-specific.outputs.any_changed == 'true' diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c7f5419a..e58e676f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -401,8 +401,8 @@ ELSE () IF (${TOOLS_BUILD_TYPE} MATCHES "Release") #SET(CMAKE_C_FLAGS "/w /D_WIN32 /DWIN32 /Zi /D NDEBUG /MTd") - # SET(CMAKE_C_FLAGS "/permissive- /GS /GL /Gy /Zc:wchar_t /Zi /Gm- /O2 /Zc:inline /fp:precise /D \"RELEASE\" /D \"NDEBUG\" /D \"_CONSOLE\" /D \"_UNICODE\" /D \"UNICODE\" /errorReport:prompt /Zc:forScope /Gd /Oi /MTd /FC /EHsc /nologo /diagnostics:column") - # SET(CMAKE_CXX_FLAGS "/permissive- /GS /GL /Gy /Zc:wchar_t /Zi /Gm- /O2 /sdl /Zc:inline /fp:precise /D \"NDEBUG\" /D \"_CONSOLE\" /D \"_UNICODE\" /D \"UNICODE\" /errorReport:prompt /Zc:forScope /Gd /Oi /MTd /FC /EHsc /nologo /diagnostics:column") + # SET(CMAKE_C_FLAGS "/permissive- /GS /GL /Gy /Zc:wchar_t /Zi /Gm- /O2 /Zc:inline /fp:precise /D \"RELEASE\" /D \"NDEBUG\" /D \"_CONSOLE\" /D \"_UNICODE\" /D \"UNICODE\" /errorReport:prompt /Zc:forScope /Gd /Oi /MTd /FC /EHsc /nologo /diagnostics:column") + # SET(CMAKE_CXX_FLAGS "/permissive- /GS /GL /Gy /Zc:wchar_t /Zi /Gm- /O2 /sdl /Zc:inline /fp:precise /D \"NDEBUG\" /D \"_CONSOLE\" /D \"_UNICODE\" /D \"UNICODE\" /errorReport:prompt /Zc:forScope /Gd /Oi /MTd /FC /EHsc /nologo /diagnostics:column") #TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy pthread libargp.lib zlibstatic ${WEBSOCKET_LINK_FLAGS} msvcrt.lib ucrtd.lib) TARGET_LINK_LIBRARIES(taosdump taos avro jansson snappy pthread libargp.lib zlibstatic ${WEBSOCKET_LINK_FLAGS}) ELSE ()