feat: Compression (#2235)

Closes #2082
infinyon · Mar 11, 2022 · 2975c47 · 2975c47
1 parent 3f06772
commit 2975c47
Show file tree

Hide file tree

Showing 42 changed files with 1,076 additions and 119 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -309,12 +309,14 @@ jobs:
         os: [ubuntu-latest]
         rust: [stable]
         rust-target: [x86_64-unknown-linux-gnu]
+        compression: [none, gzip, snappy, lz4]
     env:
       RUST_BACKTRACE: full
       RUSTV: ${{ matrix.rust }}
       TARGET: ${{ matrix.rust-target }}
       RELEASE: true
       RELEASE_NAME: release
+      FLV_CLIENT_DEFAULT_COMPRESSION_CODEC: ${{ matrix.compression }}
     steps:
       - uses: actions/checkout@v2
       - name: Install Rust ${{ matrix.rust }}
@@ -444,6 +446,7 @@ jobs:
       FLUVIO_BIN: ~/bin/fluvio
       TEST_BIN: ~/bin/fluvio-test
       SERVER_LOG: fluvio=debug
+      FLV_CLIENT_DEFAULT_COMPRESSION_CODEC: ${{ matrix.compression }}
     strategy:
       fail-fast: false
       matrix:
@@ -461,7 +464,7 @@ jobs:
             batch-failure,
             batch,
           ]
-
+        compression: [none, gzip, snappy, lz4]
     steps:
       - uses: actions/checkout@v2
       - name: Download artifact - fluvio
@@ -604,6 +607,7 @@ jobs:
       TEST_BIN: ~/bin/fluvio-test
       UNINSTALL: noclean
       SERVER_LOG: fluvio=debug
+      FLV_CLIENT_DEFAULT_COMPRESSION_CODEC: ${{ matrix.compression }}
     strategy:
       fail-fast: false
       matrix:
@@ -613,6 +617,7 @@ jobs:
         test: [smoke-test-k8, smoke-test-k8-tls, smoke-test-k8-tls-root-unclean]
         k8: [k3d, minikube]
         spu: [2]
+        compression: [none, gzip, snappy, lz4]
     steps:
       - uses: actions/checkout@v2
       # Download artifacts
@@ -717,11 +722,14 @@ jobs:
     name: Upgrade cluster test on (${{ matrix.run }})
     needs: build_image
     runs-on: ${{ matrix.os }}
+    env:
+      FLV_CLIENT_DEFAULT_COMPRESSION_CODEC: ${{ matrix.compression }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         run: [r1]
+        compression: [none, gzip, snappy, lz4]
     steps:
       - uses: actions/checkout@v2
       - name: Setup K3d
@@ -822,10 +830,13 @@ jobs:
     needs: build_image
     #if: ${{ false }}
     runs-on: ${{ matrix.os }}
+    env:
+      FLV_CLIENT_DEFAULT_COMPRESSION_CODEC: ${{ matrix.compression }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
+        compression: [none, gzip, snappy, lz4]
     steps:
       - uses: actions/checkout@v2
       - name: Install Rust ${{ matrix.rust }}
@@ -1038,4 +1049,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Done
-        run: echo "Done!"
+        run: echo "Done!"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 * Report SPU error codes to FutureRecordMetadata ([#2228](https://github.com/infinyon/fluvio/issues/2228))
 * Optimize partition size computation ([#2230](https://github.com/infinyon/fluvio/issues/2230))
 * Fix fluvio-test configuration to support data generator ([#2237](https://github.com/infinyon/fluvio/pull/2237))
+* Add compression support. ([#2082](https://github.com/infinyon/fluvio/issues/2082))
 
 ## Platform Version 0.9.20 - 2022-02-17
 * Add `connector update -c config` to update the running configuration of a given existing managed connector ([#2188](https://github.com/infinyon/fluvio/pull/2188))

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,6 +13,7 @@ members = [
     "crates/fluvio-cli",
     "crates/fluvio-cli-common",
     "crates/fluvio-cluster",
+    "crates/fluvio-compression",
     "crates/fluvio-controlplane",
     "crates/fluvio-controlplane-metadata",
     "crates/fluvio-dataplane-protocol",

diff --git a/crates/fluvio-compression/Cargo.toml b/crates/fluvio-compression/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "fluvio-compression"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+authors = ["Fluvio Contributors <team@fluvio.io>"]
+categories = ["compression"]
+keywords = ["fluvio", "compression"]
+repository = "https://github.com/infinyon/fluvio"
+description = "Fluvio Compression library"
+
+[features]
+default = []
+
+[dependencies]
+serde = { version = "1.0.110", features = ['derive'] }
+flate2 = { version = "1.0.20"}
+snap = { version = "1" }
+lz4_flex = { version = "0.9", default-features = false, features = ["safe-decode", "safe-encode", "frame"] }
+thiserror = "1.0.30"