Update (#48)

Update * encoder: refactor bit-writing * common: turn dcheck to be conditional check * add experimaental code for "groups" encoding
google · Oct 18, 2019 · 071877e · 071877e
1 parent 239b10b
commit 071877e
Show file tree

Hide file tree

Showing 20 changed files with 953 additions and 347 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "third_party/googletest"]
 	path = third_party/googletest
 	url = https://github.com/google/googletest
+[submodule "third_party/highwayhash"]
+	path = third_party/highwayhash
+	url = https://github.com/google/highwayhash.git
diff --git a/BUILD b/BUILD
@@ -157,24 +157,65 @@ cc_library(
     ],
 )
 
+config_setting(
+    name = "experimental",
+    define_values = {
+        "brunsli_groups": "experimental",
+    },
+)
+
+EXPERIMENTAL_DEPS = select({
+    ":experimental": [
+        ":groups",
+        "@highwayhash//:highwayhash_inc",
+    ],
+    "//conditions:default": [],
+})
+
+EXPERIMENTAL_DEFINES = select({
+    ":experimental": ["BRUNSLI_EXPERIMENTAL_GROUPS"],
+    "//conditions:default": [],
+})
+
+EXPERIMENTAL_LINKOPTS = select({
+    ":experimental": ["-pthread"],
+    "//conditions:default": [],
+})
+
+cc_library(
+    name = "groups",
+    srcs = ["c/experimental/groups.cc"],
+    hdrs = ["c/experimental/groups.h"],
+    copts = STRICT_C_OPTIONS,
+    deps = [
+        ":brunslicommon",
+        ":brunslidec",
+        ":brunslienc",
+    ],
+)
+
 cc_binary(
     name = "cbrunsli",
     srcs = ["c/tools/cbrunsli.cc"],
     copts = STRICT_C_OPTIONS,
+    defines = EXPERIMENTAL_DEFINES,
+    linkopts = EXPERIMENTAL_LINKOPTS,
     deps = [
         ":brunslicommon",
         ":brunslienc",
-    ],
+    ] + EXPERIMENTAL_DEPS,
 )
 
 cc_binary(
     name = "dbrunsli",
     srcs = ["c/tools/dbrunsli.cc"],
     copts = STRICT_C_OPTIONS,
+    defines = EXPERIMENTAL_DEFINES,
+    linkopts = EXPERIMENTAL_LINKOPTS,
     deps = [
         ":brunslicommon",
         ":brunslidec",
-    ],
+    ] + EXPERIMENTAL_DEPS,
 )
 
 cc_test(

diff --git a/WORKSPACE b/WORKSPACE
@@ -15,3 +15,16 @@ new_local_repository(
     path = "third_party/googletest",
     build_file = "third_party/googletest/BUILD.bazel",
 )
+
+new_local_repository(
+    name = "highwayhash",
+    path = "third_party/highwayhash",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+cc_library(
+    name = "highwayhash_inc",
+    hdrs = glob(["highwayhash/*.h"]),
+    strip_include_prefix = "",
+)
+    """,
+)
diff --git a/brunsli.cmake b/brunsli.cmake
@@ -37,6 +37,7 @@ set(BRUNSLI_ENC_SOURCES
   c/enc/histogram_encode.cc
   c/enc/jpeg_data_reader.cc
   c/enc/jpeg_huffman_decode.cc
+  c/enc/write_bits.cc
 )
 
 # TODO(eustas): split public/private headers.

diff --git a/c/common/platform.h b/c/common/platform.h
@@ -46,10 +46,6 @@
 #define BRUNSLI_X_BIG_ENDIAN BIG_ENDIAN
 #endif
 
-#if defined(BRUNSLI_DEBUG)
-#include <cassert>
-#endif
-
 /* The following macros were borrowed from https://github.com/nemequ/hedley
  * with permission of original author - Evan Nemerson <evan@nemerson.com> */
 
@@ -492,12 +488,6 @@ static BRUNSLI_INLINE void BRUNSLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
 #define BRUNSLI_LOG_ERROR() BRUNSLI_LOG_(ERROR)
 #endif  // defined(BRUNSLI_DISABLE_LOG)
 
-#if defined(BRUNSLI_DEBUG)
-#define BRUNSLI_DCHECK(V) assert(V)
-#else
-#define BRUNSLI_DCHECK(V)
-#endif
-
 namespace brunsli {
 void BrunsliDumpAndAbort(const char* f, int l, const char* fn);
 }  // namespace brunsli
@@ -510,6 +500,12 @@ void BrunsliDumpAndAbort(const char* f, int l, const char* fn);
     while (true) ;                                                    \
   }
 
+#if defined(BRUNSLI_DEBUG)
+#define BRUNSLI_DCHECK(V) BRUNSLI_CHECK(V)
+#else
+#define BRUNSLI_DCHECK(V)
+#endif
+
 // TODO: Pick up upgrade after https://github.com/google/brotli/pull/636
 //               is landed and merged.
 inline int Log2FloorNonZero(uint32_t n) {

diff --git a/c/enc/ans_encode.cc b/c/enc/ans_encode.cc
@@ -10,6 +10,7 @@
 
 #include <brunsli/types.h>
 #include "./histogram_encode.h"
+#include "./write_bits.h"
 
 namespace brunsli {
 
@@ -38,17 +39,16 @@ void ANSBuildInfoTable(const int* counts, int alphabet_size,
 
 }  // namespace
 
-void BuildAndStoreANSEncodingData(const int* histogram,
-                                  ANSTable* table,
-                                  size_t* storage_ix, uint8_t* storage) {
+void BuildAndStoreANSEncodingData(const int* histogram, ANSTable* table,
+                                  Storage* storage) {
   int num_symbols;
-  int symbols[kMaxNumSymbolsForSmallCode] = { 0 };
+  int symbols[kMaxNumSymbolsForSmallCode] = {0};
   std::vector<int> counts(histogram, histogram + ANS_MAX_SYMBOLS);
   int omit_pos;
   NormalizeCounts(&counts[0], &omit_pos, ANS_MAX_SYMBOLS, ANS_LOG_TAB_SIZE,
                   &num_symbols, symbols);
   ANSBuildInfoTable(&counts[0], ANS_MAX_SYMBOLS, table->info_);
-  EncodeCounts(&counts[0], omit_pos, num_symbols, symbols, storage_ix, storage);
+  EncodeCounts(&counts[0], omit_pos, num_symbols, symbols, storage);
 }
 
 }  // namespace brunsli
diff --git a/c/enc/ans_encode.h b/c/enc/ans_encode.h
@@ -12,6 +12,7 @@
 
 #include "../common/ans_params.h"
 #include <brunsli/types.h>
+#include "./write_bits.h"
 
 namespace brunsli {
 
@@ -52,8 +53,8 @@ class ANSCoder {
     const uint32_t offset = state_ - v * t.freq_ + t.start_;
     state_ = (v << ANS_LOG_TAB_SIZE) + offset;
 #else
-    state_ = ((state_ / t.freq_) << ANS_LOG_TAB_SIZE)
-           + (state_ % t.freq_) + t.start_;
+    state_ = ((state_ / t.freq_) << ANS_LOG_TAB_SIZE) + (state_ % t.freq_) +
+             t.start_;
 #endif
     return bits;
   }
@@ -64,9 +65,8 @@ class ANSCoder {
   uint32_t state_;
 };
 
-void BuildAndStoreANSEncodingData(const int* histogram,
-                                  ANSTable* table,
-                                  size_t* storage_ix, uint8_t* storage);
+void BuildAndStoreANSEncodingData(const int* histogram, ANSTable* table,
+                                  Storage* storage);
 
 }  // namespace brunsli