feat: implement postgres kvbackend

.github/actions/setup-postgres-cluster/action.yml

@@ -0,0 +1,29 @@
+name: Setup PostgreSQL
+description: Deploy PostgreSQL on Kubernetes
+inputs:
+  postgres-replicas:
+    default: 1
+    description: "Number of PostgreSQL replicas"
+  namespace:
+    default: "postgres-namespace"
+  postgres-version:
+    default: "14.2"
+    description: "PostgreSQL version"
+  storage-size:
+    default: "1Gi"
+    description: "Storage size for PostgreSQL"
+
+runs:
+  using: composite
+  steps:
+  - name: Install PostgreSQL
+    shell: bash
+    run: |
+      helm upgrade \
+        --install postgresql bitnami/postgresql \
+        --set replicaCount=${{ inputs.postgres-replicas }} \
+        --set image.tag=${{ inputs.postgres-version }} \
+        --set persistence.size=${{ inputs.storage-size }} \
+        --set postgresql.username=greptimedb \
+        --create-namespace \
+        -n ${{ inputs.namespace }}

.github/workflows/develop.yml

@@ -313,6 +313,8 @@ jobs:
         uses: ./.github/actions/setup-kafka-cluster
       - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
+      - name: Setup Postgres cluser
+        uses: ./.github/actions/setup-postgres-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
         with:
@@ -439,6 +441,8 @@ jobs:
         uses: ./.github/actions/setup-kafka-cluster
       - name: Setup Etcd cluser
         uses: ./.github/actions/setup-etcd-cluster
+      - name: Setup Postgres cluser
+        uses: ./.github/actions/setup-postgres-cluster
       # Prepares for fuzz tests
       - uses: arduino/setup-protoc@v3
         with:
@@ -665,6 +669,8 @@ jobs:
       - name: Setup minio
         working-directory: tests-integration/fixtures/minio
         run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Setup postgres server
+        working-directory: tests-integration/fixtures/postgres
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
         env:
@@ -681,6 +687,7 @@ jobs:
           GT_MINIO_REGION: us-west-2
           GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
           GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_POSTGRES_ENDPOINTS: postgres://greptimedb@127.0.0.1:5432/postgres
           GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
           UNITTEST_LOG_DIR: "__unittest_logs"
       - name: Codecov upload

Cargo.toml

@@ -169,6 +169,7 @@ sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "5
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.36", features = ["full"] }
+tokio-postgres = "0.7.11"
 tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"

src/cmd/src/metasrv.rs

@@ -22,6 +22,7 @@ use common_telemetry::info;
 use common_telemetry::logging::TracingOptions;
 use common_version::{short_version, version};
 use meta_srv::bootstrap::MetasrvInstance;
+use meta_srv::metasrv::BackendImpl;
 use snafu::ResultExt;
 use tracing_appender::non_blocking::WorkerGuard;
 
@@ -137,6 +138,9 @@ struct StartCommand {
     /// The max operations per txn
     #[clap(long)]
     max_txn_ops: Option<usize>,
+    /// The database backend.
+    #[clap(long)]
+    backend: Option<String>,
 }
 
 impl StartCommand {
@@ -219,6 +223,20 @@ impl StartCommand {
             opts.max_txn_ops = max_txn_ops;
         }
 
+        if let Some(s) = &self.backend {
+            match s.to_lowercase().as_str() {
+                "etcdstore" => opts.backend = BackendImpl::EtcdStore,
+                "memorystore" => opts.backend = BackendImpl::MemoryStore,
+                "postgresstore" => opts.backend = BackendImpl::PostgresStore,
+                _ => {
+                    return error::IllegalConfigSnafu {
+                        msg: format!("Invalid backend input: {s}",),
+                    }
+                    .fail()
+                }
+            }
+        }
+
         // Disable dashboard in metasrv.
         opts.http.disable_dashboard = true;
 

src/common/meta/Cargo.toml

@@ -56,6 +56,7 @@ store-api.workspace = true
 strum.workspace = true
 table.workspace = true
 tokio.workspace = true
+tokio-postgres.workspace = true
 tonic.workspace = true
 typetag = "0.2"
 

src/common/meta/src/error.rs

@@ -644,6 +644,22 @@ pub enum Error {
 
     #[snafu(display("Failed to get cache"))]
     GetCache { source: Arc<Error> },
+
+    #[snafu(display("Failed to execute via Postgres"))]
+    PostgresFailed {
+        #[snafu(source)]
+        error: tokio_postgres::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to connect to Postgres"))]
+    ConnectPostgres {
+        #[snafu(source)]
+        error: tokio_postgres::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -655,10 +671,12 @@ impl ErrorExt for Error {
             IllegalServerState { .. }
             | EtcdTxnOpResponse { .. }
             | EtcdFailed { .. }
+            | PostgresFailed { .. }
             | EtcdTxnFailed { .. }
             | ConnectEtcd { .. }
             | MoveValues { .. }
-            | GetCache { .. } => StatusCode::Internal,
+            | GetCache { .. }
+            | ConnectPostgres { .. } => StatusCode::Internal,
 
             ValueNotExist { .. } => StatusCode::Unexpected,
 

src/common/meta/src/kv_backend.rs

@@ -31,6 +31,7 @@ use crate::rpc::KeyValue;
 pub mod chroot;
 pub mod etcd;
 pub mod memory;
+pub mod postgres;
 pub mod test;
 pub mod txn;