-
Notifications
You must be signed in to change notification settings - Fork 343
Zlib shim with ISA_L
Pablo de Lara edited this page Mar 16, 2026
·
1 revision
Use LD_PRELOAD before the following commands to preload the shim:
export LD_PRELOAD=/path/to/isal-shim.so./db_bench --benchmarks=fillrandom --compression_type=zlib \
--db=/tmp/rocksdb_benchdb --num=100000 --value_size=4096 --key_size=16./db_bench --benchmarks=readseq --compression_type=zlib \
--use_existing_db=1 --db=/tmp/rocksdb_benchdb \
--num=100000 --value_size=4096#!/bin/bash
ROCKSDB_DIR="rocksdb"
NUM_KEYS=100000
VALUE_SIZE=4096
KEY_SIZE=16
SEED=12345
ZLIB_DB="/tmp/rocksdb_benchdb_zlib"
SHIM_DB="/tmp/rocksdb_benchdb_shim"
rm -rf "$ZLIB_DB" "$SHIM_DB"
$ROCKSDB_DIR/db_bench --benchmarks=fillrandom --compression_type=zlib \
--db="$ZLIB_DB" --num=$NUM_KEYS --value_size=$VALUE_SIZE \
--key_size=$KEY_SIZE --seed=$SEED > /dev/null
ORIG_SIZE=$(du -sb "$ZLIB_DB" | cut -f1)
cp -r "$ZLIB_DB" "$SHIM_DB"
# --- Baseline (standard zlib) ---
START_TIME=$(date +%s.%N)
$ROCKSDB_DIR/db_bench --benchmarks=readseq --compression_type=zlib \
--use_existing_db=1 --db="$ZLIB_DB" --num=$NUM_KEYS --value_size=$VALUE_SIZE
END_TIME=$(date +%s.%N)
DB_SIZE=$(du -sb "$ZLIB_DB" | cut -f1)
COMPRESSION_RATIO=$(echo "scale=2; ($DB_SIZE * 100) / $ORIG_SIZE" | bc)
COMPRESSION_TIME=$(echo "scale=3; $END_TIME - $START_TIME" | bc)
echo "Compression time: ${COMPRESSION_TIME}s"
echo "Compression ratio: ${COMPRESSION_RATIO}%"
echo "Original file size: ${ORIG_SIZE} bytes"
echo "Compressed file size: ${DB_SIZE} bytes"
echo "Throughput: $(echo "scale=2; $ORIG_SIZE / $COMPRESSION_TIME / 1048576" | bc) MB/s"
# --- With ISA-L shim ---
export LD_PRELOAD="/home/vkarpenk/isal/libraries.performance.storage.isa-l/igzip/shim/build/isal-shim.so"
START_TIME=$(date +%s.%N)
$ROCKSDB_DIR/db_bench --benchmarks=readseq --compression_type=zlib \
--use_existing_db=1 --db="$SHIM_DB" --num=$NUM_KEYS --value_size=$VALUE_SIZE
END_TIME=$(date +%s.%N)
DB_SIZE=$(du -sb "$SHIM_DB" | cut -f1)
COMPRESSION_RATIO=$(echo "scale=2; ($DB_SIZE * 100) / $ORIG_SIZE" | bc)
COMPRESSION_TIME=$(echo "scale=3; $END_TIME - $START_TIME" | bc)
echo "Compression time with shim: ${COMPRESSION_TIME}s"
echo "Compression ratio with shim: ${COMPRESSION_RATIO}%"
echo "Original file size: ${ORIG_SIZE} bytes"
echo "Compressed file size with shim: ${DB_SIZE} bytes"
echo "Throughput with shim: $(echo "scale=2; $ORIG_SIZE / $COMPRESSION_TIME / 1048576" | bc) MB/s"
unset LD_PRELOAD./QATzip/utils/qzip -k -o dickens.gz dickens -O deflate_4B -g none./QATzip/utils/qzip -d -k -o dickens_decompresseds dickens.gz -O deflate_4B -g noneBenchmarking data is printed by
qzip.
./qpdf --compress-streams=y --object-streams=generate reymont_compressed reymont./qpdf --stream-data=uncompress reymont_decompressed reymont_compressed#!/bin/bash
# Set library path to find libqpdf.so.30
export LD_LIBRARY_PATH="../libqpdf:$LD_LIBRARY_PATH"
INPUT_FILE="reymont"
COMPRESSED_FILE="reymont_compressed"
DECOMPRESSED_FILE="reymont_decompressed"
# Compress and measure time
START_COMPRESS=$(date +%s%N)
./qpdf --compress-streams=y --object-streams=generate "$INPUT_FILE" "$COMPRESSED_FILE"
END_COMPRESS=$(date +%s%N)
COMPRESS_TIME=$(echo "scale=3; ($END_COMPRESS - $START_COMPRESS) / 1000000000" | bc)
# Decompress and measure time
START_DECOMPRESS=$(date +%s%N)
./qpdf --stream-data=uncompress "$COMPRESSED_FILE" "$DECOMPRESSED_FILE"
END_DECOMPRESS=$(date +%s%N)
DECOMPRESS_TIME=$(echo "scale=3; ($END_DECOMPRESS - $START_DECOMPRESS) / 1000000000" | bc)
# Calculate file sizes and ratio
ORIGINAL_SIZE=$(stat -c%s "$INPUT_FILE")
COMPRESSED_SIZE=$(stat -c%s "$COMPRESSED_FILE")
COMPRESSION_RATIO=$(echo "scale=2; $COMPRESSED_SIZE / $ORIGINAL_SIZE * 100" | bc)
# Summary
echo "Summary:"
echo "Compression time: $COMPRESS_TIME seconds"
echo "Decompression time: $DECOMPRESS_TIME seconds"
echo "Original size: $ORIGINAL_SIZE bytes"
echo "Compressed size: $COMPRESSED_SIZE bytes"
echo "Compression ratio: $COMPRESSION_RATIO%"
echo "Throughput: $(echo "scale=2; $ORIGINAL_SIZE / $COMPRESS_TIME / 1024 / 1024" | bc) MB/s"java GzipFileCompressor dickensjava GzipFileDecompressor dickens.gzjavac GzipFileCompressor.java
javac GzipFileDecompressor.javaimport java.io.*;
import java.util.zip.GZIPOutputStream;
import java.util.Random;
public class GzipFileCompressor {
public static void main(String[] args) {
if (args.length == 0 || args[0].equals("--help")) {
System.out.println("Usage:");
System.out.println(" java GzipFileCompressor <filename> [outputfile] - Compress existing file");
System.out.println(" java GzipFileCompressor --help - Show this help");
return;
}
try {
String inputFile = args[0];
String outputFile = (args.length > 1) ? args[1] : inputFile + ".gz";
compressFile(inputFile, outputFile);
System.out.println("Compressed " + inputFile + " to " + outputFile);
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
private static void compressFile(String inputFile, String outputFile) throws IOException {
File inFile = new File(inputFile);
long startTime = System.currentTimeMillis();
long inputSize = inFile.length();
try (FileInputStream fis = new FileInputStream(inputFile);
FileOutputStream fos = new FileOutputStream(outputFile);
GZIPOutputStream gzos = new GZIPOutputStream(fos)) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = fis.read(buffer)) != -1) {
gzos.write(buffer, 0, bytesRead);
}
}
long endTime = System.currentTimeMillis();
long outputSize = new File(outputFile).length();
double compressionRatio = (double) inputSize / outputSize;
double compressionPercent = (1.0 - (double) outputSize / inputSize) * 100;
double elapsedTimeSeconds = (endTime - startTime) / 1000.0;
double throughputMBps = (inputSize / (1024.0 * 1024.0)) / elapsedTimeSeconds;
System.out.println("Compression statistics:");
System.out.println(" Original size: " + inputSize + " bytes");
System.out.println(" Compressed size: " + outputSize + " bytes");
System.out.println(" Compression ratio: " + String.format("%.2f", compressionRatio) + ":1");
System.out.println(" Space savings: " + String.format("%.2f", compressionPercent) + "%");
System.out.println(" Compression time: " + String.format("%.3f", elapsedTimeSeconds) + " seconds");
System.out.println(" Throughput: " + String.format("%.2f", throughputMBps) + " MB/s");
}
}import java.io.*;
import java.util.zip.GZIPInputStream;
public class GzipFileDecompressor {
public static void main(String[] args) {
if (args.length == 0 || args[0].equals("--help")) {
System.out.println("Usage:");
System.out.println(" java GzipFileDecompressor <filename.gz> - Decompress gzip file");
System.out.println(" java GzipFileDecompressor --help - Show this help");
return;
}
try {
String inputFile = args[0];
// Check if file has .gz extension
if (!inputFile.toLowerCase().endsWith(".gz")) {
System.err.println("Error: Input file must have .gz extension");
return;
}
// Remove .gz extension for output filename
String outputFile = inputFile.substring(0, inputFile.length() - 3);
decompressFile(inputFile, outputFile);
System.out.println("Decompressed " + inputFile + " to " + outputFile);
// Show file sizes for comparison
File compressed = new File(inputFile);
File decompressed = new File(outputFile);
if (compressed.exists() && decompressed.exists()) {
long compressedSize = compressed.length();
long decompressedSize = decompressed.length();
double ratio = (double) compressedSize / decompressedSize * 100;
System.out.println("Original compressed size: " + formatBytes(compressedSize));
System.out.println("Decompressed size: " + formatBytes(decompressedSize));
System.out.println("Compression ratio: " + String.format("%.1f%%", ratio));
}
} catch (FileNotFoundException e) {
System.err.println("Error: File not found - " + e.getMessage());
} catch (IOException e) {
System.err.println("Error: " + e.getMessage());
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
private static void decompressFile(String inputFile, String outputFile) throws IOException {
long startTime = System.currentTimeMillis();
long bytesProcessed = 0;
try (FileInputStream fis = new FileInputStream(inputFile);
GZIPInputStream gzis = new GZIPInputStream(fis);
FileOutputStream fos = new FileOutputStream(outputFile)) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = gzis.read(buffer)) != -1) {
fos.write(buffer, 0, bytesRead);
bytesProcessed += bytesRead;
}
}
long endTime = System.currentTimeMillis();
double seconds = (endTime - startTime) / 1000.0;
double mbPerSecond = bytesProcessed / (1024.0 * 1024.0) / seconds;
System.out.println("Decompression completed in " + String.format("%.2f", seconds) + " seconds");
System.out.println("Throughput: " + String.format("%.2f MB/s", mbPerSecond));
}
private static String formatBytes(long bytes) {
if (bytes < 1024) {
return bytes + " B";
} else if (bytes < 1024 * 1024) {
return String.format("%.1f KB", bytes / 1024.0);
} else if (bytes < 1024 * 1024 * 1024) {
return String.format("%.1f MB", bytes / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", bytes / (1024.0 * 1024.0 * 1024.0));
}
}
}