Let us time minify and make sure AVX-512 is used by default. (#1830)

* Let us time minify
* Making AVX-512 available by default.
* Silencing some maybe-uninitialized warning under GCC (warning appears in the standard library).
* Making the Python amalgamation script a bit more Windows friendly.
* We do not try to silence -Wmaybe-uninitialized under clang.

benchmark/benchmarker.h

@@ -2,7 +2,7 @@
 #define __BENCHMARKER_H
 
 #include "event_counter.h"
-#include "simdjson.h" // For SIMDJSON_DISABLE_DEPRECATED_WARNINGS
+#include "simdjson.h"
 
 #include <cassert>
 #include <cctype>

include/simdjson/common_defs.h

@@ -122,6 +122,9 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
   // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
+  // We do it separately for clang since it has different warnings.
+  #ifdef __clang__
+  // clang is missing -Wmaybe-uninitialized.
   #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
     SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
@@ -134,6 +137,22 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
     SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable)
+  #else // __clang__
+  #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
+    SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
+  #endif // __clang__
+
   #define SIMDJSON_PRAGMA(P) _Pragma(#P)
   #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING)
   #if defined(SIMDJSON_CLANG_VISUAL_STUDIO)

include/simdjson/implementations.h

@@ -21,6 +21,11 @@
 #endif
 #endif
 
+// By default, we allow AVX512.
+#ifndef SIMDJSON_AVX512_ALLOWED
+#define SIMDJSON_AVX512_ALLOWED 1
+#endif
+
 // Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected
 // at runtime.
 #ifndef SIMDJSON_IMPLEMENTATION_ICELAKE

singleheader/amalgamate.py

@@ -81,8 +81,10 @@ def dofile(fid, prepath, filename):
     # print(f"// dofile: invoked with prepath={prepath}, filename={filename}",file=fid)
     file = os.path.join(prepath, filename)
     RELFILE = os.path.relpath(file, PROJECTPATH)
+    # Windows use \ as a directory separator, but we do not want that:
+    OSRELFILE = RELFILE.replace('\\','/')
     # Last lines are always ignored. Files should end by an empty lines.
-    print(f"/* begin file {RELFILE} */", file=fid)
+    print(f"/* begin file {OSRELFILE} */", file=fid)
     includepattern = re.compile('^#include "(.*)"')
     redefines_simdjson_implementation = re.compile('^#define\s+SIMDJSON_IMPLEMENTATION\s+(.*)')
     undefines_simdjson_implementation = re.compile('^#undef\s+SIMDJSON_IMPLEMENTATION\s*$')
@@ -114,7 +116,7 @@ def dofile(fid, prepath, filename):
                 else:
                     # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
                     print(uses_simdjson_implementation.sub(current_implementation+"\\1",line), file=fid)
-    print(f"/* end file {RELFILE} */", file=fid)
+    print(f"/* end file {OSRELFILE} */", file=fid)
 
 
 # Get the generation date from git, so the output is reproducible.

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2022-05-25 11:24:40 -0400. Do not edit! */
+/* auto-generated on 2022-05-26 14:07:15 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 
@@ -7589,6 +7589,9 @@ simdjson_really_inline error_code json_structural_indexer::finish(dom_parser_imp
  * naked intrinsics.
  * TODO: make this code more elegant.
  */
+// Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'.
+// as a workaround, we disable warnings within the following function.
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 namespace simdjson { namespace icelake { namespace { namespace stage1 {
 simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     // In some instances, the next branch is expensive because it is mispredicted.
@@ -7623,6 +7626,7 @@ simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     this->tail += count;
 }
 }}}}
+SIMDJSON_POP_DISABLE_WARNINGS
 
 /* begin file src/generic/stage1/utf8_validator.h */
 namespace simdjson {

singleheader/simdjson.h

@@ -1,4 +1,4 @@
-/* auto-generated on 2022-05-25 11:24:40 -0400. Do not edit! */
+/* auto-generated on 2022-05-26 14:07:15 -0400. Do not edit! */
 /* begin file include/simdjson.h */
 #ifndef SIMDJSON_H
 #define SIMDJSON_H
@@ -414,6 +414,9 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
   // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
+  // We do it separately for clang since it has different warnings.
+  #ifdef __clang__
+  // clang is missing -Wmaybe-uninitialized.
   #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
     SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
@@ -426,6 +429,22 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
     SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable)
+  #else // __clang__
+  #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
+    SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
+  #endif // __clang__
+
   #define SIMDJSON_PRAGMA(P) _Pragma(#P)
   #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING)
   #if defined(SIMDJSON_CLANG_VISUAL_STUDIO)
@@ -9482,6 +9501,11 @@ extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256];
 #endif
 #endif
 
+// By default, we allow AVX512.
+#ifndef SIMDJSON_AVX512_ALLOWED
+#define SIMDJSON_AVX512_ALLOWED 1
+#endif
+
 // Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected
 // at runtime.
 #ifndef SIMDJSON_IMPLEMENTATION_ICELAKE

src/icelake/dom_parser_implementation.cpp

@@ -114,6 +114,9 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
  * naked intrinsics.
  * TODO: make this code more elegant.
  */
+// Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'.
+// as a workaround, we disable warnings within the following function.
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 {
 simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     // In some instances, the next branch is expensive because it is mispredicted.
@@ -148,6 +151,7 @@ simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     this->tail += count;
 }
 }}}}
+SIMDJSON_POP_DISABLE_WARNINGS
 
 #include "generic/stage1/utf8_validator.h"
 

tools/minify.cpp

@@ -1,3 +1,4 @@
+#include <chrono>
 #include <iostream>
 #if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__))
 #include <dirent.h>
@@ -38,6 +39,7 @@ int main(int argc, const char *argv[]) {
   }
   options.add_options()
     ("a,arch", ss.str(), cxxopts::value<std::string>())
+    ("t,timing", "Report only timing.")
     ("f,file", "File name.", cxxopts::value<std::string>())
     ("h,help", "Print usage.")
   ;
@@ -79,7 +81,34 @@ int main(int argc, const char *argv[]) {
   size_t copy_len;
   error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
   if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
-  printf("%s", copy.data());
+  /**
+   * If a user only wants to time the required time, we do not output
+   * the result and we simply do the processing in a tight loop.
+   * At this point in time, we can assume that the processing will
+   * succeed.
+   */
+  if(result.count("timing")) {
+      uint64_t beforens = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+      error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
+      uint64_t afterns = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+      size_t times = 1;
+      while(afterns - beforens < 1000000000) {
+          error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
+          afterns = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+          times += 1;
+      }
+      if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+      printf("%.3f GB/s\n", double(p.length() * times) / double(afterns - beforens));
+  } else {
+      // This is the expected path:
+      printf("%s", copy.data());
+  }
   return EXIT_SUCCESS;
 #ifdef __cpp_exceptions
   } catch (const cxxopts::OptionException& e) {