Add openalex and format

README.md

@@ -23,19 +23,24 @@ single dependency checkout across build trees.
 
 ```
 jsonbench --dataset <file.ndjson> [options]
-  --query <name>       twitter|bestbuy|google_map|nspl|walmart|wiki
+  --query <name>       twitter|bestbuy|google_map|nspl|walmart|wiki|openalex
                        (default: inferred from the filename)
   --threads a,b,c      thread counts to sweep (default: 1..hw, doubling)
   --reps <n>           repetitions per configuration, best wins (default 3)
   --slice-kb <n>       parse_many_parallel slice size (default 1024)
-  --sections <list>    load,verify,single,scaling,e2e (default: all)
+  --sections <list>    load,verify,single,scaling,e2e,format (default: all
+                       but format)
   --single-record      treat the input as one bulky JSON document
   --verify             check that the engines agree, then exit
   --dump <n>           print the first n extracted values from each engine
 ```
 
+The `format` section compares comma-delimited against newline-delimited
+encoding of the same records, serially and on simdjson's two-thread pipeline.
+
 Output is one `RESULT key=value ...` line per measured configuration.
 
+
 ## How the comparison is kept fair
 
 Both engines answer the **same** query and must produce the same answer. The
@@ -90,9 +95,15 @@ per gigabyte from `getrusage`, which does aggregate all threads.
 It downloads the six bulky records from the public collection the Pison and
 cuJSON papers use, then derives the JSON-lines form of each with `make_ndjson`,
 which minifies every element of the dataset's dominating array onto its own line
-(`tweets`, `data`, `products`, `items`, `items`, `items`). Roughly 12 GB of disk
+(`tweets`, `data`, `products`, `items`, `items`, `items`). Roughly 18 GB of disk
 and a `pip install gdown`. The result is `~/jsonbench/ndjson/<dataset>.ndjson`.
 
+OpenAlex authors joins the corpus the same way: the 232,330 author records
+updated on 2026-03-30 in the OpenAlex snapshot (CC0, about 6.1 GB, largest
+record about 1.37 MB), hosted on Zenodo and pinned by record and size, with
+query `$.display_name, $.works_count`. With `--corpus-from` the peer's copy
+is reused when present, exactly as for the six cuJSON datasets.
+
 The collection publishes all six datasets as bulky records but only two of the
 six JSON-lines files, which is why the rest are derived. On the two published in
 both forms, the derived file has exactly the same record count as the published

datasets.sh

@@ -22,8 +22,11 @@
 # header, which does not exist in the data rows that form the JSON-lines
 # version, so we define our own two-column extraction for it (see src/common.h).
 #
-# The corpus is roughly 6 GB of JSON-lines plus 6 GB of bulky records. Nothing
-# is committed to this repository.
+# OpenAlex authors (a paper dataset, CC0, hosted on Zenodo) is part of the
+# corpus by default; see the README for its description.
+#
+# The corpus is roughly 6 GB of JSON-lines plus 6 GB of bulky records, plus
+# about 6.1 GB for OpenAlex authors. Nothing is committed to this repository.
 set -euo pipefail
 
 DIR=""
@@ -34,12 +37,32 @@ DRIVE_FOLDER="1PkDEy0zWOkVREfL7VuINI-m9wJe45P2Q"
 log() { printf '\033[1;34m[datasets]\033[0m %s\n' "$*" >&2; }
 die() { printf '\033[1;31m[datasets] ERR:\033[0m %s\n' "$*" >&2; exit 1; }
 
+fetch_openalex() {
+  local out="$DIR/ndjson/openalex_authors.ndjson"
+  if [ -s "$out" ]; then
+    log "openalex authors already present: $out"
+    return 0
+  fi
+  local url="https://zenodo.org/records/21813521/files/openalex-authors.ndjson?download=1"
+  local expected=6419215096
+  command -v curl >/dev/null 2>&1 || die "curl required for OpenAlex"
+  log "downloading OpenAlex authors from Zenodo (6.1 GB) ..."
+  curl -fL --retry 3 -o "$out" "$url" >&2
+  local actual
+  actual="$(stat -c%s "$out")"
+  if [ "$actual" != "$expected" ]; then
+    rm -f "$out"
+    die "OpenAlex download size mismatch (expected $expected, got $actual); the pinned object changed"
+  fi
+  log "openalex authors ready: $(ls -la "$out")"
+}
+
 while [ $# -gt 0 ]; do
   case "$1" in
     --dir)   DIR="${2:?}"; shift 2 ;;
     --build) BUILD="${2:?}"; shift 2 ;;
     --corpus-from) CORPUS_FROM="${2:?}"; shift 2 ;;
-    -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
     *) die "unknown option: $1" ;;
   esac
 done
@@ -69,6 +92,8 @@ if [ -n "$CORPUS_FROM" ]; then
   exit 0
 fi
 
+fetch_openalex
+
 # Fallback: fetch the bulky records from the published Drive folder and derive
 # the JSON-lines form locally.
 #

src/common.h

@@ -29,13 +29,18 @@ namespace jsonbench {
 // harnesses shipped in cuJSON's `related_works/pison/JSON_lines`, which contain
 // copy-paste errors (google.cpp runs the bestbuy query; nspl.cpp runs the
 // google query and misspells "text" as "tex", matching nothing).
+//
+// OpenAlex authors is our own addition (see datasets.sh): the corpus is one
+// daily change partition of the public snapshot, and the query selects the two
+// root-level fields that identify an author record.
 enum class query_id {
   twitter,    // $.user.lang, $.lang
   bestbuy,    // $.categoryPath[1:3].id
   google_map, // $.routes[*].legs[*].steps[*].distance.text
   nspl,       // $[8], $[9]   (see note above)
   walmart,    // $.bestMarketplacePrice.price, $.name
   wiki,       // $.claims.P150[*].mainsnak.property
+  openalex,   // $.display_name, $.works_count  (see note below)
 };
 
 const char *query_name(query_id q);

src/dom_queries.h

@@ -129,6 +129,13 @@ template <class T> void q_wiki(typename T::node doc, extraction &out) {
   });
 }
 
+// $.display_name, $.works_count
+template <class T> void q_openalex(typename T::node doc, extraction &out) {
+  typename T::node v;
+  if (T::obj_get(doc, "display_name", v)) { T::feed(v, out); }
+  if (T::obj_get(doc, "works_count", v)) { T::feed(v, out); }
+}
+
 template <class T>
 void run_query(query_id q, typename T::node doc, extraction &out) {
   switch (q) {
@@ -138,6 +145,7 @@ void run_query(query_id q, typename T::node doc, extraction &out) {
   case query_id::nspl:       q_nspl<T>(doc, out);    return;
   case query_id::walmart:    q_walmart<T>(doc, out); return;
   case query_id::wiki:       q_wiki<T>(doc, out);    return;
+  case query_id::openalex:   q_openalex<T>(doc, out); return;
   }
 }
 

src/main.cpp

@@ -65,7 +65,7 @@ void usage() {
       "usage: jsonbench --dataset <file> [options]\n"
       "  --dataset <path>     JSON-lines file to benchmark (required)\n"
       "  --label <name>       short name for the output (default: filename stem)\n"
-      "  --query <name>       twitter|bestbuy|google_map|nspl|walmart|wiki\n"
+      "  --query <name>       twitter|bestbuy|google_map|nspl|walmart|wiki|openalex\n"
       "                       (default: inferred from the filename)\n"
       "  --threads a,b,c      thread counts to sweep (default: 1..hw, doubling)\n"
       "  --reps <n>           repetitions per configuration, best wins (default 3)\n"
@@ -75,8 +75,8 @@ void usage() {
       "  --verify             check that the engines agree, then exit\n"
       "  --dump <n>           print the first n extracted values from each\n"
       "                       engine side by side, then exit\n"
-      "  --sections <list>    comma list of load,verify,single,scaling,e2e\n"
-      "                       (default: all)\n");
+      "  --sections <list>    comma list of load,verify,single,scaling,e2e,format\n"
+      "                       (default: all but format)\n");
 }
 
 bool parse_args(int argc, char **argv, options &o) {
@@ -309,12 +309,18 @@ int main(int argc, char **argv) {
   // Sizes each DOM worker's arena. One pass over the input, taken once here so
   // that no timed region pays for it.
   const size_t dom_longest = dom::longest_document(data, bytes);
+  // iterate_many's batch must exceed the longest document, or the serial
+  // baseline fails with CAPACITY. Keep the 1 MiB batch the established corpora
+  // were measured with; grow it only when a corpus demands it (the OpenAlex
+  // corpus has records of up to 1.37 MB).
+  const size_t serial_batch =
+      dom_longest > (1u << 20) ? dom_longest + (1u << 20) : (1u << 20);
   bool agree = true;
   extraction pison_ref;
   if (o.wants("verify") || o.dump > 0) {
     pison_ref = pison::run_stream(ptext, tbl, q, pison::workload::query, levels, 1);
     extraction sj_ref =
-        sj::run_serial(data, bytes, q, sj::workload::query, false, 1u << 20);
+        sj::run_serial(data, bytes, q, sj::workload::query, false, serial_batch);
     agree = (pison_ref.matches == sj_ref.matches) &&
             (pison_ref.sum == sj_ref.sum);
     std::printf("# agreement query: pison matches=%llu hash=%llu | "
@@ -328,7 +334,7 @@ int main(int argc, char **argv) {
     extraction pison_dec =
         pison::run_stream(ptext, tbl, q, pison::workload::decode, levels, 1);
     extraction sj_dec =
-        sj::run_serial(data, bytes, q, sj::workload::decode, false, 1u << 20);
+        sj::run_serial(data, bytes, q, sj::workload::decode, false, serial_batch);
     std::printf("# agreement decode: pison matches=%llu hash=%llu | "
                 "simdjson matches=%llu hash=%llu | %s\n",
                 (unsigned long long)pison_dec.matches,
@@ -368,7 +374,7 @@ int main(int argc, char **argv) {
     std::vector<std::string> pt, st;
     pison::run_stream(ptext, tbl, q, pison::workload::query, levels, 1, &pt,
                       o.dump);
-    sj::run_serial(data, bytes, q, sj::workload::query, false, 1u << 20, &st,
+    sj::run_serial(data, bytes, q, sj::workload::query, false, serial_batch, &st,
                    o.dump);
     std::printf("# %-4s %-38s %-38s %s\n", "i", "pison", "simdjson", "same");
     for (size_t i = 0; i < o.dump; i++) {
@@ -409,9 +415,9 @@ int main(int argc, char **argv) {
     }
     for (const auto &ph : sj_phases) {
       auto s = measure_single(o.reps, [&] {
-        sj::run_serial(data, bytes, q, ph.w, false, 1u << 20);
+        sj::run_serial(data, bytes, q, ph.w, false, serial_batch);
       });
-      extraction e = sj::run_serial(data, bytes, q, ph.w, false, 1u << 20);
+      extraction e = sj::run_serial(data, bytes, q, ph.w, false, serial_batch);
       emit("simdjson", ph.phase, sj::workload_name(ph.w), 1, o, label, bytes,
            docs, s, e);
     }
@@ -438,6 +444,66 @@ int main(int argc, char **argv) {
     }
   }
 
+  // -----------------------------------------------------------------------
+  // Stream-format overhead (opt-in: sections list contains "format"): the
+  // same documents as comma-delimited input, against newline-delimited, on
+  // the serial path and on simdjson's built-in two-thread pipeline. The comma
+  // form is rebuilt from the record table, so both runs see identical bytes
+  // modulo the separators.
+  // -----------------------------------------------------------------------
+  if (o.wants("format")) {
+    std::string comma;
+    comma.reserve(bytes);
+    for (size_t i = 0; i < tbl.count(); i++) {
+      size_t len = tbl.length[i];
+      while (len > 0 && (ptext[tbl.offset[i] + len - 1] == '\n' ||
+                         ptext[tbl.offset[i] + len - 1] == '\r')) {
+        len--;
+      }
+      comma.append(ptext + tbl.offset[i], len);
+      comma.push_back(',');
+    }
+    comma.append(64, '\0');
+    struct encoding {
+      const char *name;
+      const char *data;
+      size_t size;
+      simdjson::stream_format format;
+    };
+    const encoding encodings[] = {
+        {"whitespace", data, bytes,
+         simdjson::stream_format::whitespace_delimited},
+        {"comma", comma.data(), comma.size(),
+         simdjson::stream_format::comma_delimited},
+    };
+    // Measure one encoding at one thread count. The threaded run skips the
+    // per-thread performance counters (see the methodology section of the
+    // paper), hence the two timing helpers.
+    auto measure_encoding = [&](const encoding &enc,
+                                bool threaded) -> measurement {
+      if (threaded) {
+        return measure_parallel(o.reps, [&] {
+          sj::run_serial_format(enc.data, enc.size, q, sj::workload::query,
+                                threaded, serial_batch, enc.format);
+        });
+      }
+      return measure_single(o.reps, [&] {
+        sj::run_serial_format(enc.data, enc.size, q, sj::workload::query,
+                              threaded, serial_batch, enc.format);
+      });
+    };
+    for (bool threaded : {false, true}) {
+      for (const encoding &enc : encodings) {
+        measurement s = measure_encoding(enc, threaded);
+        extraction e = sj::run_serial_format(
+            enc.data, enc.size, q, sj::workload::query, threaded, serial_batch,
+            enc.format);
+        emit("simdjson-format", "format", enc.name, threaded ? 2 : 1, o, label,
+             bytes, docs, s, e);
+      }
+    }
+  }
+
   // -----------------------------------------------------------------------
   // Thread scaling.
   // -----------------------------------------------------------------------

src/pison_engine.cpp

@@ -262,6 +262,18 @@ void q_wiki(BitmapIterator *iter, extraction &out, workload w) {
   iter->up();
 }
 
+// $.display_name, $.works_count
+void q_openalex(BitmapIterator *iter, extraction &out, workload w) {
+  if (!iter->isObject()) { return; }
+  std::unordered_set<char *> keys;
+  keys.insert(const_cast<char *>("display_name"));
+  keys.insert(const_cast<char *>("works_count"));
+  char *key = nullptr;
+  while ((key = iter->moveToKey(keys)) != nullptr) {
+    take(iter, out, w);
+  }
+}
+
 void run_query(BitmapIterator *iter, extraction &out, query_id q, workload w) {
   switch (q) {
   case query_id::twitter: q_twitter(iter, out, w); break;
@@ -270,6 +282,7 @@ void run_query(BitmapIterator *iter, extraction &out, query_id q, workload w) {
   case query_id::nspl: q_nspl(iter, out, w); break;
   case query_id::walmart: q_walmart(iter, out, w); break;
   case query_id::wiki: q_wiki(iter, out, w); break;
+  case query_id::openalex: q_openalex(iter, out, w); break;
   }
 }
 

src/queries.cpp

@@ -12,6 +12,7 @@ const char *query_name(query_id q) {
   case query_id::nspl: return "nspl";
   case query_id::walmart: return "walmart";
   case query_id::wiki: return "wiki";
+  case query_id::openalex: return "openalex";
   }
   return "?";
 }
@@ -23,6 +24,7 @@ bool query_from_name(const std::string &s, query_id &out) {
   if (s == "nspl") { out = query_id::nspl; return true; }
   if (s == "walmart") { out = query_id::walmart; return true; }
   if (s == "wiki") { out = query_id::wiki; return true; }
+  if (s == "openalex") { out = query_id::openalex; return true; }
   return false;
 }
 
@@ -46,6 +48,7 @@ bool query_from_name(const std::string &s, query_id &out) {
 //   nspl        $[8], $[9]                                    1        1
 //   walmart     $.bestMarketplacePrice.price, $.name          2        2
 //   wiki        $.claims.P150[*].mainsnak.property            4        5
+//   openalex    $.display_name, $.works_count                 1        1
 int query_levels(query_id q) {
   switch (q) {
   case query_id::twitter: return 2;
@@ -54,14 +57,16 @@ int query_levels(query_id q) {
   case query_id::nspl: return 1;
   case query_id::walmart: return 2;
   case query_id::wiki: return 5;
+  case query_id::openalex: return 1;
   }
   return 22; // MAX_LEVEL: always safe, never fast
 }
 
 bool query_from_path(const std::string &path, query_id &out) {
   // Longest names first so "google_map" wins over a bare "google".
   static const char *names[] = {"google_map", "bestbuy", "twitter",
-                                "walmart",    "wiki",    "nspl"};
+                                "walmart",    "wiki",    "nspl",
+                                "openalex"};
   for (const char *n : names) {
     if (path.find(n) != std::string::npos) { return query_from_name(n, out); }
   }