Merge branch 'main' into postgres_kv_backend

.github/actions/build-macos-artifacts/action.yml

@@ -62,12 +62,13 @@ runs:
     # Get proper backtraces in mac Sonoma. Currently there's an issue with the new
     # linker that prevents backtraces from getting printed correctly.
     #
-    # <https://github.com/rust-lang/rust/issues/113783> 
+    # <https://github.com/rust-lang/rust/issues/113783>
     - name: Run integration tests
       if: ${{ inputs.disable-run-tests == 'false' }}
       shell: bash
-      env: 
+      env:
         CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic"
+        SQLNESS_OPTS: "--preserve-state"
       run: |
         make test sqlness-test
 
@@ -81,7 +82,7 @@ runs:
 
     - name: Build greptime binary
       shell: bash
-      env: 
+      env:
         CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic"
       run: |
         make build \

.github/actions/build-windows-artifacts/action.yml

@@ -40,7 +40,7 @@ runs:
     - name: Install Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: "3.10"
 
     - name: Install PyArrow Package
       shell: pwsh
@@ -62,13 +62,14 @@ runs:
       env:
         RUSTUP_WINDOWS_PATH_ADD_BIN: 1 # Workaround for https://github.com/nextest-rs/nextest/issues/1493
         RUST_BACKTRACE: 1
+        SQLNESS_OPTS: "--preserve-state"
 
     - name: Upload sqlness logs
       if: ${{ failure() }} # Only upload logs when the integration tests failed.
       uses: actions/upload-artifact@v4
       with:
         name: sqlness-logs
-        path: /tmp/greptime-*.log
+        path: C:\tmp\greptime-*.log
         retention-days: 3
 
     - name: Build greptime binary

.github/actions/setup-greptimedb-cluster/with-disk.yaml

@@ -1,18 +1,13 @@
 meta:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
 datanode:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
+    compact_rt_size = 2
 frontend:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4

.github/actions/setup-greptimedb-cluster/with-minio-and-cache.yaml

@@ -1,29 +1,24 @@
 meta:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
-    
+    global_rt_size = 4
+
     [datanode]
     [datanode.client]
     timeout = "60s"
 datanode:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
+    compact_rt_size = 2
     
     [storage]
     cache_path = "/data/greptimedb/s3cache"
     cache_capacity = "256MB"
 frontend:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
 
     [meta_client]
     ddl_timeout = "60s"

.github/actions/setup-greptimedb-cluster/with-minio.yaml

@@ -1,25 +1,20 @@
 meta:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
     
     [datanode]
     [datanode.client]
     timeout = "60s"
 datanode:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
+    compact_rt_size = 2
 frontend:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
 
     [meta_client]
     ddl_timeout = "60s"

.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml

@@ -1,9 +1,7 @@
 meta:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
     
     [wal]
     provider = "kafka"
@@ -17,9 +15,8 @@ meta:
 datanode:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
+    compact_rt_size = 2
 
     [wal]
     provider = "kafka"
@@ -28,9 +25,7 @@ datanode:
 frontend:
   config: |-
     [runtime]
-    read_rt_size = 8
-    write_rt_size = 8
-    bg_rt_size = 8
+    global_rt_size = 4
 
     [meta_client]
     ddl_timeout = "60s"
@@ -43,3 +38,8 @@ objectStorage:
   credentials:
     accessKeyId: rootuser
     secretAccessKey: rootpass123
+remoteWal:
+   enabled: true
+   kafka:
+     brokerEndpoints: 
+      - "kafka.kafka-cluster.svc.cluster.local:9092"

.github/workflows/develop.yml

@@ -141,6 +141,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
+      fail-fast: false
       matrix:
         target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
     steps:

.github/workflows/nightly-ci.yml

@@ -51,13 +51,15 @@ jobs:
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
       - name: Run sqlness
-        run: cargo sqlness
+        run: make sqlness-test
+        env:
+          SQLNESS_OPTS: "--preserve-state"
       - name: Upload sqlness logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: sqlness-logs
-          path: /tmp/greptime-*.log
+          path: C:\tmp\greptime-*.log
           retention-days: 3
 
   test-on-windows:
@@ -109,11 +111,7 @@ jobs:
 
   check-status:
     name: Check status
-    needs: [
-      sqlness-test,
-      sqlness-windows,
-      test-on-windows,
-    ]
+    needs: [sqlness-test, sqlness-windows, test-on-windows]
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
     runs-on: ubuntu-20.04
     outputs:
@@ -127,9 +125,7 @@ jobs:
   notification:
     if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
     name: Send notification to Greptime team
-    needs: [
-      check-status
-    ]
+    needs: [check-status]
     runs-on: ubuntu-20.04
     env:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}

CONTRIBUTING.md

@@ -14,7 +14,7 @@ Follow our [README](https://github.com/GreptimeTeam/greptimedb#readme) to get th
 
 It can feel intimidating to contribute to a complex project, but it can also be exciting and fun. These general notes will help everyone participate in this communal activity.
 
-- Follow the [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md)
+- Follow the [Code of Conduct](https://github.com/GreptimeTeam/.github/blob/main/.github/CODE_OF_CONDUCT.md)
 - Small changes make huge differences. We will happily accept a PR making a single character change if it helps move forward. Don't wait to have everything working.
 - Check the closed issues before opening your issue.
 - Try to follow the existing style of the code.
@@ -30,7 +30,7 @@ Pull requests are great, but we accept all kinds of other help if you like. Such
 
 ## Code of Conduct
 
-Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/greptimedb/blob/main/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
+Also, there are things that we are not looking for because they don't match the goals of the product or benefit the community. Please read [Code of Conduct](https://github.com/GreptimeTeam/.github/blob/main/.github/CODE_OF_CONDUCT.md); we hope everyone can keep good manners and become an honored member.
 
 ## License
 

Cargo.toml

@@ -119,12 +119,12 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "5c801650435d464891114502539b701c77a1b914" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "c437b55725b7f5224fe9d46db21072b4a682ee4b" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "80b72716dcde47ec4161478416a5c6c21343364d" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "049171eb16cb4249d8099751a0c46750d1fe88e7" }
 mockall = "0.11.4"
 moka = "0.12"
 notify = "6.1"
@@ -184,7 +184,7 @@ auth = { path = "src/auth" }
 cache = { path = "src/cache" }
 catalog = { path = "src/catalog" }
 client = { path = "src/client" }
-cmd = { path = "src/cmd" }
+cmd = { path = "src/cmd", default-features = false }
 common-base = { path = "src/common/base" }
 common-catalog = { path = "src/common/catalog" }
 common-config = { path = "src/common/config" }
@@ -214,7 +214,7 @@ datanode = { path = "src/datanode" }
 datatypes = { path = "src/datatypes" }
 file-engine = { path = "src/file-engine" }
 flow = { path = "src/flow" }
-frontend = { path = "src/frontend" }
+frontend = { path = "src/frontend", default-features = false }
 index = { path = "src/index" }
 log-store = { path = "src/log-store" }
 meta-client = { path = "src/meta-client" }
@@ -239,7 +239,7 @@ table = { path = "src/table" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "80b72716dcde47ec4161478416a5c6c21343364d"
+rev = "049171eb16cb4249d8099751a0c46750d1fe88e7"
 
 [profile.release]
 debug = 1

Makefile

@@ -15,6 +15,7 @@ RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2
 CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry
 ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
 OUTPUT_DIR := $(shell if [ "$(RELEASE)" = "true" ]; then echo "release"; elif [ ! -z "$(CARGO_PROFILE)" ]; then echo "$(CARGO_PROFILE)" ; else echo "debug"; fi)
+SQLNESS_OPTS ?=
 
 # The arguments for running integration tests.
 ETCD_VERSION ?= v3.5.9
@@ -161,7 +162,7 @@ nextest: ## Install nextest tools.
 
 .PHONY: sqlness-test
 sqlness-test: ## Run sqlness test.
-	cargo sqlness
+	cargo sqlness ${SQLNESS_OPTS}
 
 # Run fuzz test ${FUZZ_TARGET}.
 RUNS ?= 1
@@ -172,7 +173,7 @@ fuzz:
 
 .PHONY: fuzz-ls
 fuzz-ls:
-	cargo fuzz list --fuzz-dir tests-fuzz 
+	cargo fuzz list --fuzz-dir tests-fuzz
 
 .PHONY: check
 check: ## Cargo check all the targets.