Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

extents: Add support for Snappy

Snappy is a fast compression/decompression library from Google.

The support for Snappy is optional: if the environment variable
$SNAPPY_HOME is defined, P-Store is compiled with Snappy support
and linked with 'libsnappy.a', otherwise P-Store is compiled
without Snappy support.

Here is a comparison of FastLZ and Snappy with a 1 GiB CSV file
generated by 'torture/gencsv':

    FastLZ                 Snappy

    import
      real  0m15.634s      0m16.675s
      user  0m9.868s       0m10.621s
      sys   0m2.537s       0m2.630s

    export
      real  0m17.847s      0m17.013s
      user  0m6.279s       0m5.963s
      sys   0m2.301s       0m2.442s

    compression ratio
            33.64%         40.35%

Snappy was built with default options.

Signed-off-by: Jussi Virtanen <jussi.virtanen@reaktor.fi>
Signed-off-by: Pekka Enberg <penberg@kernel.org>
  • Loading branch information...
commit b5b852069a95808cd36d18219274f7e59c2db5ee 1 parent 87b52fa
Jussi Virtanen jvirtanen authored committed
33 Makefile
View
@@ -3,6 +3,7 @@ uname_R := $(shell sh -c 'uname -r 2>/dev/null || echo not')
# External programs
CC := gcc
+CXX := g++
AR := ar
# Set up source directory for GNU Make
@@ -32,7 +33,8 @@ EXTRA_WARNINGS += -Wstrict-prototypes
EXTRA_WARNINGS += -Wdeclaration-after-statement
# Compile flags
-CFLAGS := -I$(srcdir)/include -Wall $(EXTRA_WARNINGS) -g -O3 -std=gnu99
+CFLAGS := -I$(srcdir)/include -Wall $(EXTRA_WARNINGS) -g -O3 -std=gnu99
+CXXFLAGS := -I$(srcdir)/snappy -Wall -g -O3 -std=c++98
# Output to current directory by default
O =
@@ -52,8 +54,10 @@ export E Q
PROGRAM := pstore
DEFINES =
+INCLUDES =
CONFIG_OPTS =
COMPAT_OBJS =
+EXTRA_LIBS =
ifeq ($(uname_S),Darwin)
CONFIG_OPTS += -DCONFIG_NEED_STRNDUP=1
@@ -72,6 +76,12 @@ ifeq ($(uname_S),SunOS)
CONFIG_OPTS += -DCONFIG_NEED_STRNDUP=1
COMPAT_OBJS += compat/strndup.o
endif
+ifneq ($(SNAPPY_HOME),)
+ INCLUDES += -I$(SNAPPY_HOME)/include
+
+ CONFIG_OPTS += -DCONFIG_HAVE_SNAPPY=1
+ EXTRA_LIBS += -lstdc++ $(SNAPPY_HOME)/lib/libsnappy.a
+endif
OBJS += builtin-cat.o
OBJS += builtin-export.o
@@ -85,10 +95,16 @@ OBJS += pstore.o
OBJS += $(COMPAT_OBJS)
LIBS := -L. -lpstore
+LIBS += $(EXTRA_LIBS)
CFLAGS += $(DEFINES)
+CFLAGS += $(INCLUDES)
CFLAGS += $(CONFIG_OPTS)
+CXXFLAGS += $(DEFINES)
+CXXFLAGS += $(INCLUDES)
+CXXFLAGS += $(CONFIG_OPTS)
+
DEPS := $(patsubst %.o,%.d,$(OBJS))
LIB_FILE := libpstore.a
@@ -104,6 +120,9 @@ LIB_OBJS += header.o
LIB_OBJS += mmap-window.o
LIB_OBJS += read-write.o
LIB_OBJS += segment.o
+ifneq ($(SNAPPY_HOME),)
+LIB_OBJS += snappy/snappy_compat.o
+endif
LIB_OBJS += table.o
LIB_DEPS := $(patsubst %.o,%.d,$(LIB_OBJS))
@@ -148,10 +167,17 @@ endif
%.d: %.c
$(Q) $(CC) -M -MT $(patsubst %.d,%.o,$@) $(CFLAGS) $< -o $@
+%.d: %.cc
+ $(Q) $(CXX) -M -MT $(patsubst %.d,%.o,$@) $(CXXFLAGS) $< -o $@
+
%.o: %.c
$(E) " CC " $@
$(Q) $(CC) -c $(CFLAGS) $< -o $@
+%.o: %.cc
+ $(E) " CXX " $@
+ $(Q) $(CXX) -c $(CXXFLAGS) $< -o $@
+
$(PROGRAM): $(DEPS) $(LIB_FILE) $(OBJS)
$(E) " LINK " $@
$(Q) $(CC) $(OBJS) $(LIBS) -o $(PROGRAM)
@@ -187,7 +213,10 @@ clean:
regress: $(PROGRAM)
$(E) " CUCUMBER"
- $(Q) cucumber --format progress
+ $(Q) cucumber --tags ~@snappy --format progress
+ifneq ($(SNAPPY_HOME),)
+ $(Q) cucumber --tags @snappy --format progress
+endif
ifneq ($(JAVA_HOME),)
$(Q) $(MAKE) -C java check
endif
13 builtins-common.c
View
@@ -23,10 +23,14 @@ unsigned long parse_storage_arg(char *arg)
uint8_t parse_comp_arg(char *arg)
{
- if (strcmp(arg, "fastlz") == 0)
- return PSTORE_COMP_FASTLZ;
- else if (strcmp(arg, "none") == 0)
+ if (strcmp(arg, "none") == 0)
return PSTORE_COMP_NONE;
+ else if (strcmp(arg, "fastlz") == 0)
+ return PSTORE_COMP_FASTLZ;
+#ifdef CONFIG_HAVE_SNAPPY
+ else if (strcmp(arg, "snappy") == 0)
+ return PSTORE_COMP_SNAPPY;
+#endif
return NR_PSTORE_COMP;
}
@@ -36,6 +40,9 @@ void comp_arg_usage(void)
printf("\n The supported compression schemes are:\n");
printf(" none\n");
printf(" fastlz\n");
+#ifdef CONFIG_HAVE_SNAPPY
+ printf(" snappy\n");
+#endif
}
unsigned long parse_int_arg(char *arg)
63 compress.c
View
@@ -9,6 +9,9 @@
#include "pstore/die.h"
#include "fastlz/fastlz.h"
+#ifdef CONFIG_HAVE_SNAPPY
+#include "snappy/snappy_compat.h"
+#endif
#include <stdlib.h>
#include <stdio.h>
@@ -89,3 +92,63 @@ struct pstore_extent_ops extent_fastlz_ops = {
.next_value = extent__next_value,
.flush = extent__fastlz_compress,
};
+
+#ifdef CONFIG_HAVE_SNAPPY
+
+/*
+ * Snappy
+ */
+
+static void extent__snappy_compress(struct pstore_extent *self, int fd)
+{
+ void *input;
+ size_t input_length;
+ void *compressed;
+ size_t compressed_length;
+
+ input_length = buffer__size(self->write_buffer);
+ input = buffer__start(self->write_buffer);
+
+ compressed = malloc(snappy_max_compressed_length(input_length));
+ if (!compressed)
+ die("out of memory");
+
+ snappy_raw_compress(input, input_length, compressed, &compressed_length);
+
+ self->lsize = input_length;
+
+ write_or_die(fd, compressed, compressed_length);
+
+ free(compressed);
+}
+
+static void *extent__snappy_decompress(struct pstore_extent *self, int fd, off_t offset)
+{
+ struct mmap_window *mmap;
+ void *compressed;
+ void *uncompressed;
+
+ buffer__resize(self->parent->buffer, self->lsize);
+
+ mmap = mmap_window__map(self->psize, fd, offset + sizeof(struct pstore_file_extent), self->psize);
+ compressed = mmap_window__start(mmap);
+
+ uncompressed = buffer__start(self->parent->buffer);
+
+ if (!snappy_raw_uncompress(compressed, self->psize, uncompressed))
+ die("decompression failed");
+
+ self->parent->buffer->offset = self->lsize;
+
+ mmap_window__unmap(mmap);
+
+ return uncompressed;
+}
+
+struct pstore_extent_ops extent_snappy_ops = {
+ .read = extent__snappy_decompress,
+ .next_value = extent__next_value,
+ .flush = extent__snappy_compress,
+};
+
+#endif
5 extent.c
View
@@ -73,7 +73,10 @@ static const struct pstore_extent_ops extent_uncomp_ops = {
static const struct pstore_extent_ops *extent_ops_table[NR_PSTORE_COMP] = {
[PSTORE_COMP_NONE] = &extent_uncomp_ops,
- [PSTORE_COMP_FASTLZ] = &extent_fastlz_ops
+ [PSTORE_COMP_FASTLZ] = &extent_fastlz_ops,
+#ifdef CONFIG_HAVE_SNAPPY
+ [PSTORE_COMP_SNAPPY] = &extent_snappy_ops,
+#endif
};
struct pstore_extent *pstore_extent__new(struct pstore_column *parent, uint8_t comp)
6 features/import.feature
View
@@ -13,6 +13,12 @@ Feature: CSV import
When I run "pstore import --compress=fastlz"
Then the database should contain the same data in column order
+ @snappy
+ Scenario: Snappy compressed import
+ Given a 32K long CSV file
+ When I run "pstore import --compress=snappy"
+ Then the database should contain the same data in column order
+
Scenario: Uncompressed append
Given a 2K long CSV file
And a 4K long database
4 include/pstore/compress.h
View
@@ -5,4 +5,8 @@
extern struct pstore_extent_ops extent_fastlz_ops;
+#ifdef CONFIG_HAVE_SNAPPY
+extern struct pstore_extent_ops extent_snappy_ops;
+#endif
+
#endif /* PSTORE_COMPRESS_H */
3  include/pstore/disk-format.h
View
@@ -46,6 +46,9 @@ struct pstore_file_column {
enum pstore_comp {
PSTORE_COMP_NONE = 0,
PSTORE_COMP_FASTLZ = 2,
+#ifdef CONFIG_HAVE_SNAPPY
+ PSTORE_COMP_SNAPPY = 3,
+#endif
NR_PSTORE_COMP /* keep this last */
};
12 java/Makefile
View
@@ -53,6 +53,14 @@ JAVA_HEADER_FILES := $(patsubst %.o,include/%.h,$(OBJS))
# pstore-jni.h is not part of $(JAVA_HEADER_FILES).
OBJS += pstore-jni.o
+CONFIG_OPTS =
+EXTRA_LIBS =
+
+ifneq ($(SNAPPY_HOME),)
+ CONFIG_OPTS += -DCONFIG_HAVE_SNAPPY=1
+ EXTRA_LIBS += -lstdc++ $(SNAPPY_HOME)/lib/libsnappy.a
+endif
+
CFLAGS := -Wall -g -O6 -std=gnu99
DEFINES := -D_FILE_OFFSET_BITS=64
INCLUDE := \
@@ -66,7 +74,11 @@ ifeq ($(uname_S),Linux)
-I$(JAVA_HOME)/include/linux \
-I$(JAVA_HOME)/include
endif
+
LIBS := -L../ -lpstore
+LIBS += $(EXTRA_LIBS)
+
+CFLAGS += $(CONFIG_OPTS)
all: test dist
.DEFAULT: all
18 snappy/snappy_compat.cc
View
@@ -0,0 +1,18 @@
+#include "snappy_compat.h"
+
+#include <snappy.h>
+
+size_t snappy_max_compressed_length(size_t source_bytes)
+{
+ return snappy::MaxCompressedLength(source_bytes);
+}
+
+void snappy_raw_compress(const char *input, size_t input_length, char *compressed, size_t *compressed_length)
+{
+ snappy::RawCompress(input, input_length, compressed, compressed_length);
+}
+
+int snappy_raw_uncompress(const char *compressed, size_t compressed_length, char *uncompressed)
+{
+ return snappy::RawUncompress(compressed, compressed_length, uncompressed);
+}
20 snappy/snappy_compat.h
View
@@ -0,0 +1,20 @@
+#ifndef SNAPPY_COMPAT_H
+#define SNAPPY_COMPAT_H
+
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+size_t snappy_max_compressed_length(size_t source_bytes);
+
+void snappy_raw_compress(const char *input, size_t input_length, char *compressed, size_t *compressed_length);
+
+int snappy_raw_uncompress(const char *compressed, size_t compressed_length, char *uncompressed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SNAPPY_COMPAT_H */
Please sign in to comment.
Something went wrong with that request. Please try again.