Use brotli for compressing message data

celenium-io · Jun 11, 2024 · bc3f2cc · bc3f2cc
1 parent e5ae7a8
commit bc3f2cc
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 5 deletions.
diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.22.3
 require (
 	cosmossdk.io/errors v1.0.0
 	cosmossdk.io/math v1.1.2
+	github.com/andybalholm/brotli v1.0.5
 	github.com/aws/aws-sdk-go-v2 v1.26.1
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
@@ -70,7 +71,6 @@ require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Microsoft/hcsshim v0.11.4 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/aws/aws-sdk-go v1.44.122 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect

diff --git a/internal/storage/types/packed_bytes.go b/internal/storage/types/packed_bytes.go
@@ -1,9 +1,11 @@
 package types
 
 import (
+	"bytes"
 	"database/sql"
 	"database/sql/driver"
 
+	"github.com/andybalholm/brotli"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 )
@@ -23,7 +25,8 @@ func (pb *PackedBytes) Scan(src interface{}) error {
 		return errors.Errorf("invalid packed bytes type: %T", src)
 	}
 
-	return json.Unmarshal(b, pb)
+	result := bytes.NewBuffer(b)
+	return json.NewDecoder(brotli.NewReader(result)).Decode(pb)
 }
 
 var _ driver.Valuer = (*PackedBytes)(nil)
@@ -33,5 +36,18 @@ func (pb PackedBytes) Value() (driver.Value, error) {
 }
 
 func (pb PackedBytes) ToBytes() ([]byte, error) {
-	return json.Marshal(pb)
+	b, err := json.Marshal(pb)
+	if err != nil {
+		return nil, err
+	}
+	result := bytes.NewBuffer(nil)
+	writer := brotli.NewWriterLevel(result, brotli.BestSpeed)
+
+	if _, err := writer.Write(b); err != nil {
+		return nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+	return result.Bytes(), nil
 }
diff --git a/test/data/message.yml b/test/data/message.yml
@@ -29,7 +29,7 @@
   type: MsgPayForBlobs
   tx_id: 2
   size: 200
-  data: 0x7B0A2020202022426C6F6253697A6573223A205B0A20202020202031320A202020205D2C0A20202020224E616D65737061636573223A205B0A20202020202022414141414141414141414141414141414141414141414141414141414141437436643674366434220A202020205D2C0A20202020225368617265436F6D6D69746D656E7473223A205B0A202020202020223043734C58363330636A696A394452366E716F57665163434832704351536F537571363364546B643442773D220A202020205D2C0A2020202022536861726556657273696F6E73223A205B0A202020202020300A202020205D2C0A20202020225369676E6572223A202263656C6573746961316875767865726B6D336167677779326164617A3678306D673833676661356E64656D7235346E220A20207D
+  data: 0x1b2e01c01c07762cb666885e04476efadbdc774ae2c8fb2af2c65c0324749e1e71405a6b6d71222da1346bcba2139fe47cbd6194d8f16bc2ef700b11c6987425f5fb6989b8a1804c16d98349c0851075b239e9a8f7ace2bf889e2744b474b98f7ecaa05874afd23d737e2ee6d23f81172afd6d51d1d513bbf9f5c659adb7d9a93b5b2fe6fe7bd1b7aedb004f364991aefc8da70a1884fc4002d3e23f19ce763c304439f7a1d4f5b39fff27287a6999967394fd14943fc84621af34a0311b2e01c01c07762cb666885e04476efadbdc774ae2c8fb2af2c65c0324749e1e71405a6b6d71222da1346bcba2139fe47cbd6194d8f16bc2ef700b11c6987425f5fb6989b8a1804c16d98349c0851075b239e9a8f7ace2bf889e2744b474b98f7ecaa05874afd23d737e2ee6d23f81172afd6d51d1d513bbf9f5c659adb7d9a93b5b2fe6fe7bd1b7aedb004f364991aefc8da70a1884fc4002d3e23f19ce763c304439f7a1d4f5b39fff27287a6999967394fd14943fc84621af34a031
 - id: 5
   height: 999
   position: 0

diff --git a/test/data/rollback/message.yml b/test/data/rollback/message.yml
@@ -29,7 +29,7 @@
   type: MsgPayForBlobs
   tx_id: 2
   size: 200
-  data: 0x7B0A2020202022426C6F6253697A6573223A205B0A20202020202031320A202020205D2C0A20202020224E616D65737061636573223A205B0A20202020202022414141414141414141414141414141414141414141414141414141414141437436643674366434220A202020205D2C0A20202020225368617265436F6D6D69746D656E7473223A205B0A202020202020223043734C58363330636A696A394452366E716F57665163434832704351536F537571363364546B643442773D220A202020205D2C0A2020202022536861726556657273696F6E73223A205B0A202020202020300A202020205D2C0A20202020225369676E6572223A202263656C6573746961316875767865726B6D336167677779326164617A3678306D673833676661356E64656D7235346E220A20207D
+  data: 0x1b2e01c01c07762cb666885e04476efadbdc774ae2c8fb2af2c65c0324749e1e71405a6b6d71222da1346bcba2139fe47cbd6194d8f16bc2ef700b11c6987425f5fb6989b8a1804c16d98349c0851075b239e9a8f7ace2bf889e2744b474b98f7ecaa05874afd23d737e2ee6d23f81172afd6d51d1d513bbf9f5c659adb7d9a93b5b2fe6fe7bd1b7aedb004f364991aefc8da70a1884fc4002d3e23f19ce763c304439f7a1d4f5b39fff27287a6999967394fd14943fc84621af34a0311b2e01c01c07762cb666885e04476efadbdc774ae2c8fb2af2c65c0324749e1e71405a6b6d71222da1346bcba2139fe47cbd6194d8f16bc2ef700b11c6987425f5fb6989b8a1804c16d98349c0851075b239e9a8f7ace2bf889e2744b474b98f7ecaa05874afd23d737e2ee6d23f81172afd6d51d1d513bbf9f5c659adb7d9a93b5b2fe6fe7bd1b7aedb004f364991aefc8da70a1884fc4002d3e23f19ce763c304439f7a1d4f5b39fff27287a6999967394fd14943fc84621af34a031
 - id: 5
   height: 999
   position: 0