dumpling: compress supports snappy/zstd

dumpling/export/config.go

@@ -288,7 +288,7 @@ func (*Config) DefineFlags(flags *pflag.FlagSet) {
 	_ = flags.MarkHidden(flagReadTimeout)
 	flags.Bool(flagTransactionalConsistency, true, "Only support transactional consistency")
 	_ = flags.MarkHidden(flagTransactionalConsistency)
-	flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'no-compression' now")
+	flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'snappy', 'zstd', 'no-compression' now")
 }
 
 // ParseFromFlags parses dumpling's export.Config from flags
@@ -593,6 +593,10 @@ func ParseCompressType(compressType string) (storage.CompressType, error) {
 		return storage.NoCompression, nil
 	case "gzip", "gz":
 		return storage.Gzip, nil
+	case "snappy":
+		return storage.Snappy, nil
+	case "zst":
+		return storage.Zstd, nil
 	default:
 		return storage.NoCompression, errors.Errorf("unknown compress type %s", compressType)
 	}

dumpling/export/writer_util.go

@@ -591,6 +591,10 @@ func compressFileSuffix(compressType storage.CompressType) string {
 		return ""
 	case storage.Gzip:
 		return ".gz"
+	case storage.Snappy:
+		return ".snappy"
+	case storage.Zstd:
+		return ".zst"
 	default:
 		return ""
 	}

dumpling/install.sh

@@ -15,3 +15,11 @@ chmod a+x bin/minio
 
 wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc
 chmod a+x bin/mc
+
+go get github.com/ma6174/snappy
+go install github.com/ma6174/snappy
+
+wget https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz
+tar xvfz zstd-1.5.2.tar.gz
+cd zstd-1.5.2
+make

dumpling/tests/s3/run.sh

@@ -80,6 +80,7 @@ echo "1st records count is ${cnt}"
 # run dumpling with compress option
 export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress
 
+# test gzip
 run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip"
 
 mkdir -p "${HOST_DIR}/compress"
@@ -97,4 +98,44 @@ diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
 gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d
 diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
 
+rm "${HOST_DIR}/compress/s3-schema-create.sql"
+rm "${HOST_DIR}/compress/s3.t-schema.sql"
+rm "${HOST_DIR}/compress/s3.t.000000000.sql"
+
+# test snappy
+run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "snappy"
+
+bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.snappy "${HOST_DIR}/compress/s3-schema-create.sql.snappy"
+bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.snappy "${HOST_DIR}/compress/s3.t-schema.sql.snappy"
+bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.snappy "${HOST_DIR}/compress/s3.t.000000000.sql.snappy"
+
+snappy -d "${HOST_DIR}/compress/s3-schema-create.sql.snappy"
+diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
+
+snappy -d "${HOST_DIR}/compress/s3.t-schema.sql.snappy"
+diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
+
+snappy -d "${HOST_DIR}/compress/s3.t.000000000.sql.snappy"
+diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+
+rm "${HOST_DIR}/compress/s3-schema-create.sql"
+rm "${HOST_DIR}/compress/s3.t-schema.sql"
+rm "${HOST_DIR}/compress/s3.t.000000000.sql"
+
+# test zstd
+run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "zst"
+
+bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.zst "${HOST_DIR}/compress/s3-schema-create.sql.zst"
+bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.zst "${HOST_DIR}/compress/s3.t-schema.sql.zst"
+bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.zst "${HOST_DIR}/compress/s3.t.000000000.sql.zst"
+
+zstd "${HOST_DIR}/compress/s3-schema-create.sql.zst" -d
+diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
+
+zstd "${HOST_DIR}/compress/s3.t-schema.sql.zst" -d
+diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
+
+zstd "${HOST_DIR}/compress/s3.t.000000000.sql.zst" -d
+diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+
 run_sql "drop database if exists \`$DB_NAME\`;"