diff --git a/README.md b/README.md index 4c8bc5d..5d05df9 100644 --- a/README.md +++ b/README.md @@ -23,19 +23,24 @@ single dependency checkout across build trees. ``` jsonbench --dataset [options] - --query twitter|bestbuy|google_map|nspl|walmart|wiki + --query twitter|bestbuy|google_map|nspl|walmart|wiki|openalex (default: inferred from the filename) --threads a,b,c thread counts to sweep (default: 1..hw, doubling) --reps repetitions per configuration, best wins (default 3) --slice-kb parse_many_parallel slice size (default 1024) - --sections load,verify,single,scaling,e2e (default: all) + --sections load,verify,single,scaling,e2e,format (default: all + but format) --single-record treat the input as one bulky JSON document --verify check that the engines agree, then exit --dump print the first n extracted values from each engine ``` +The `format` section compares comma-delimited against newline-delimited +encoding of the same records, serially and on simdjson's two-thread pipeline. + Output is one `RESULT key=value ...` line per measured configuration. + ## How the comparison is kept fair Both engines answer the **same** query and must produce the same answer. The @@ -90,9 +95,15 @@ per gigabyte from `getrusage`, which does aggregate all threads. It downloads the six bulky records from the public collection the Pison and cuJSON papers use, then derives the JSON-lines form of each with `make_ndjson`, which minifies every element of the dataset's dominating array onto its own line -(`tweets`, `data`, `products`, `items`, `items`, `items`). Roughly 12 GB of disk +(`tweets`, `data`, `products`, `items`, `items`, `items`). Roughly 18 GB of disk and a `pip install gdown`. The result is `~/jsonbench/ndjson/.ndjson`. +OpenAlex authors joins the corpus the same way: the 232,330 author records +updated on 2026-03-30 in the OpenAlex snapshot (CC0, about 6.1 GB, largest +record about 1.37 MB), hosted on Zenodo and pinned by record and size, with +query `$.display_name, $.works_count`. With `--corpus-from` the peer's copy +is reused when present, exactly as for the six cuJSON datasets. + The collection publishes all six datasets as bulky records but only two of the six JSON-lines files, which is why the rest are derived. On the two published in both forms, the derived file has exactly the same record count as the published diff --git a/datasets.sh b/datasets.sh index c0749dd..b8c408e 100755 --- a/datasets.sh +++ b/datasets.sh @@ -22,8 +22,11 @@ # header, which does not exist in the data rows that form the JSON-lines # version, so we define our own two-column extraction for it (see src/common.h). # -# The corpus is roughly 6 GB of JSON-lines plus 6 GB of bulky records. Nothing -# is committed to this repository. +# OpenAlex authors (a paper dataset, CC0, hosted on Zenodo) is part of the +# corpus by default; see the README for its description. +# +# The corpus is roughly 6 GB of JSON-lines plus 6 GB of bulky records, plus +# about 6.1 GB for OpenAlex authors. Nothing is committed to this repository. set -euo pipefail DIR="" @@ -34,12 +37,32 @@ DRIVE_FOLDER="1PkDEy0zWOkVREfL7VuINI-m9wJe45P2Q" log() { printf '\033[1;34m[datasets]\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31m[datasets] ERR:\033[0m %s\n' "$*" >&2; exit 1; } +fetch_openalex() { + local out="$DIR/ndjson/openalex_authors.ndjson" + if [ -s "$out" ]; then + log "openalex authors already present: $out" + return 0 + fi + local url="https://zenodo.org/records/21813521/files/openalex-authors.ndjson?download=1" + local expected=6419215096 + command -v curl >/dev/null 2>&1 || die "curl required for OpenAlex" + log "downloading OpenAlex authors from Zenodo (6.1 GB) ..." + curl -fL --retry 3 -o "$out" "$url" >&2 + local actual + actual="$(stat -c%s "$out")" + if [ "$actual" != "$expected" ]; then + rm -f "$out" + die "OpenAlex download size mismatch (expected $expected, got $actual); the pinned object changed" + fi + log "openalex authors ready: $(ls -la "$out")" +} + while [ $# -gt 0 ]; do case "$1" in --dir) DIR="${2:?}"; shift 2 ;; --build) BUILD="${2:?}"; shift 2 ;; --corpus-from) CORPUS_FROM="${2:?}"; shift 2 ;; - -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) die "unknown option: $1" ;; esac done @@ -69,6 +92,8 @@ if [ -n "$CORPUS_FROM" ]; then exit 0 fi +fetch_openalex + # Fallback: fetch the bulky records from the published Drive folder and derive # the JSON-lines form locally. # diff --git a/src/common.h b/src/common.h index 02f18c5..3855ed6 100644 --- a/src/common.h +++ b/src/common.h @@ -29,6 +29,10 @@ namespace jsonbench { // harnesses shipped in cuJSON's `related_works/pison/JSON_lines`, which contain // copy-paste errors (google.cpp runs the bestbuy query; nspl.cpp runs the // google query and misspells "text" as "tex", matching nothing). +// +// OpenAlex authors is our own addition (see datasets.sh): the corpus is one +// daily change partition of the public snapshot, and the query selects the two +// root-level fields that identify an author record. enum class query_id { twitter, // $.user.lang, $.lang bestbuy, // $.categoryPath[1:3].id @@ -36,6 +40,7 @@ enum class query_id { nspl, // $[8], $[9] (see note above) walmart, // $.bestMarketplacePrice.price, $.name wiki, // $.claims.P150[*].mainsnak.property + openalex, // $.display_name, $.works_count (see note below) }; const char *query_name(query_id q); diff --git a/src/dom_queries.h b/src/dom_queries.h index cd1072d..95fe9a7 100644 --- a/src/dom_queries.h +++ b/src/dom_queries.h @@ -129,6 +129,13 @@ template void q_wiki(typename T::node doc, extraction &out) { }); } +// $.display_name, $.works_count +template void q_openalex(typename T::node doc, extraction &out) { + typename T::node v; + if (T::obj_get(doc, "display_name", v)) { T::feed(v, out); } + if (T::obj_get(doc, "works_count", v)) { T::feed(v, out); } +} + template void run_query(query_id q, typename T::node doc, extraction &out) { switch (q) { @@ -138,6 +145,7 @@ void run_query(query_id q, typename T::node doc, extraction &out) { case query_id::nspl: q_nspl(doc, out); return; case query_id::walmart: q_walmart(doc, out); return; case query_id::wiki: q_wiki(doc, out); return; + case query_id::openalex: q_openalex(doc, out); return; } } diff --git a/src/main.cpp b/src/main.cpp index ea9b4a7..d79c9ae 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,7 +65,7 @@ void usage() { "usage: jsonbench --dataset [options]\n" " --dataset JSON-lines file to benchmark (required)\n" " --label short name for the output (default: filename stem)\n" - " --query twitter|bestbuy|google_map|nspl|walmart|wiki\n" + " --query twitter|bestbuy|google_map|nspl|walmart|wiki|openalex\n" " (default: inferred from the filename)\n" " --threads a,b,c thread counts to sweep (default: 1..hw, doubling)\n" " --reps repetitions per configuration, best wins (default 3)\n" @@ -75,8 +75,8 @@ void usage() { " --verify check that the engines agree, then exit\n" " --dump print the first n extracted values from each\n" " engine side by side, then exit\n" - " --sections comma list of load,verify,single,scaling,e2e\n" - " (default: all)\n"); + " --sections comma list of load,verify,single,scaling,e2e,format\n" + " (default: all but format)\n"); } bool parse_args(int argc, char **argv, options &o) { @@ -309,12 +309,18 @@ int main(int argc, char **argv) { // Sizes each DOM worker's arena. One pass over the input, taken once here so // that no timed region pays for it. const size_t dom_longest = dom::longest_document(data, bytes); + // iterate_many's batch must exceed the longest document, or the serial + // baseline fails with CAPACITY. Keep the 1 MiB batch the established corpora + // were measured with; grow it only when a corpus demands it (the OpenAlex + // corpus has records of up to 1.37 MB). + const size_t serial_batch = + dom_longest > (1u << 20) ? dom_longest + (1u << 20) : (1u << 20); bool agree = true; extraction pison_ref; if (o.wants("verify") || o.dump > 0) { pison_ref = pison::run_stream(ptext, tbl, q, pison::workload::query, levels, 1); extraction sj_ref = - sj::run_serial(data, bytes, q, sj::workload::query, false, 1u << 20); + sj::run_serial(data, bytes, q, sj::workload::query, false, serial_batch); agree = (pison_ref.matches == sj_ref.matches) && (pison_ref.sum == sj_ref.sum); std::printf("# agreement query: pison matches=%llu hash=%llu | " @@ -328,7 +334,7 @@ int main(int argc, char **argv) { extraction pison_dec = pison::run_stream(ptext, tbl, q, pison::workload::decode, levels, 1); extraction sj_dec = - sj::run_serial(data, bytes, q, sj::workload::decode, false, 1u << 20); + sj::run_serial(data, bytes, q, sj::workload::decode, false, serial_batch); std::printf("# agreement decode: pison matches=%llu hash=%llu | " "simdjson matches=%llu hash=%llu | %s\n", (unsigned long long)pison_dec.matches, @@ -368,7 +374,7 @@ int main(int argc, char **argv) { std::vector pt, st; pison::run_stream(ptext, tbl, q, pison::workload::query, levels, 1, &pt, o.dump); - sj::run_serial(data, bytes, q, sj::workload::query, false, 1u << 20, &st, + sj::run_serial(data, bytes, q, sj::workload::query, false, serial_batch, &st, o.dump); std::printf("# %-4s %-38s %-38s %s\n", "i", "pison", "simdjson", "same"); for (size_t i = 0; i < o.dump; i++) { @@ -409,9 +415,9 @@ int main(int argc, char **argv) { } for (const auto &ph : sj_phases) { auto s = measure_single(o.reps, [&] { - sj::run_serial(data, bytes, q, ph.w, false, 1u << 20); + sj::run_serial(data, bytes, q, ph.w, false, serial_batch); }); - extraction e = sj::run_serial(data, bytes, q, ph.w, false, 1u << 20); + extraction e = sj::run_serial(data, bytes, q, ph.w, false, serial_batch); emit("simdjson", ph.phase, sj::workload_name(ph.w), 1, o, label, bytes, docs, s, e); } @@ -438,6 +444,66 @@ int main(int argc, char **argv) { } } + // ----------------------------------------------------------------------- + // Stream-format overhead (opt-in: sections list contains "format"): the + // same documents as comma-delimited input, against newline-delimited, on + // the serial path and on simdjson's built-in two-thread pipeline. The comma + // form is rebuilt from the record table, so both runs see identical bytes + // modulo the separators. + // ----------------------------------------------------------------------- + if (o.wants("format")) { + std::string comma; + comma.reserve(bytes); + for (size_t i = 0; i < tbl.count(); i++) { + size_t len = tbl.length[i]; + while (len > 0 && (ptext[tbl.offset[i] + len - 1] == '\n' || + ptext[tbl.offset[i] + len - 1] == '\r')) { + len--; + } + comma.append(ptext + tbl.offset[i], len); + comma.push_back(','); + } + comma.append(64, '\0'); + struct encoding { + const char *name; + const char *data; + size_t size; + simdjson::stream_format format; + }; + const encoding encodings[] = { + {"whitespace", data, bytes, + simdjson::stream_format::whitespace_delimited}, + {"comma", comma.data(), comma.size(), + simdjson::stream_format::comma_delimited}, + }; + // Measure one encoding at one thread count. The threaded run skips the + // per-thread performance counters (see the methodology section of the + // paper), hence the two timing helpers. + auto measure_encoding = [&](const encoding &enc, + bool threaded) -> measurement { + if (threaded) { + return measure_parallel(o.reps, [&] { + sj::run_serial_format(enc.data, enc.size, q, sj::workload::query, + threaded, serial_batch, enc.format); + }); + } + return measure_single(o.reps, [&] { + sj::run_serial_format(enc.data, enc.size, q, sj::workload::query, + threaded, serial_batch, enc.format); + }); + }; + for (bool threaded : {false, true}) { + for (const encoding &enc : encodings) { + measurement s = measure_encoding(enc, threaded); + extraction e = sj::run_serial_format( + enc.data, enc.size, q, sj::workload::query, threaded, serial_batch, + enc.format); + emit("simdjson-format", "format", enc.name, threaded ? 2 : 1, o, label, + bytes, docs, s, e); + } + } + } + // ----------------------------------------------------------------------- // Thread scaling. // ----------------------------------------------------------------------- diff --git a/src/pison_engine.cpp b/src/pison_engine.cpp index b450cbf..83733ba 100644 --- a/src/pison_engine.cpp +++ b/src/pison_engine.cpp @@ -262,6 +262,18 @@ void q_wiki(BitmapIterator *iter, extraction &out, workload w) { iter->up(); } +// $.display_name, $.works_count +void q_openalex(BitmapIterator *iter, extraction &out, workload w) { + if (!iter->isObject()) { return; } + std::unordered_set keys; + keys.insert(const_cast("display_name")); + keys.insert(const_cast("works_count")); + char *key = nullptr; + while ((key = iter->moveToKey(keys)) != nullptr) { + take(iter, out, w); + } +} + void run_query(BitmapIterator *iter, extraction &out, query_id q, workload w) { switch (q) { case query_id::twitter: q_twitter(iter, out, w); break; @@ -270,6 +282,7 @@ void run_query(BitmapIterator *iter, extraction &out, query_id q, workload w) { case query_id::nspl: q_nspl(iter, out, w); break; case query_id::walmart: q_walmart(iter, out, w); break; case query_id::wiki: q_wiki(iter, out, w); break; + case query_id::openalex: q_openalex(iter, out, w); break; } } diff --git a/src/queries.cpp b/src/queries.cpp index 8bf5e06..bab477d 100644 --- a/src/queries.cpp +++ b/src/queries.cpp @@ -12,6 +12,7 @@ const char *query_name(query_id q) { case query_id::nspl: return "nspl"; case query_id::walmart: return "walmart"; case query_id::wiki: return "wiki"; + case query_id::openalex: return "openalex"; } return "?"; } @@ -23,6 +24,7 @@ bool query_from_name(const std::string &s, query_id &out) { if (s == "nspl") { out = query_id::nspl; return true; } if (s == "walmart") { out = query_id::walmart; return true; } if (s == "wiki") { out = query_id::wiki; return true; } + if (s == "openalex") { out = query_id::openalex; return true; } return false; } @@ -46,6 +48,7 @@ bool query_from_name(const std::string &s, query_id &out) { // nspl $[8], $[9] 1 1 // walmart $.bestMarketplacePrice.price, $.name 2 2 // wiki $.claims.P150[*].mainsnak.property 4 5 +// openalex $.display_name, $.works_count 1 1 int query_levels(query_id q) { switch (q) { case query_id::twitter: return 2; @@ -54,6 +57,7 @@ int query_levels(query_id q) { case query_id::nspl: return 1; case query_id::walmart: return 2; case query_id::wiki: return 5; + case query_id::openalex: return 1; } return 22; // MAX_LEVEL: always safe, never fast } @@ -61,7 +65,8 @@ int query_levels(query_id q) { bool query_from_path(const std::string &path, query_id &out) { // Longest names first so "google_map" wins over a bare "google". static const char *names[] = {"google_map", "bestbuy", "twitter", - "walmart", "wiki", "nspl"}; + "walmart", "wiki", "nspl", + "openalex"}; for (const char *n : names) { if (path.find(n) != std::string::npos) { return query_from_name(n, out); } } diff --git a/src/simdjson_engine.cpp b/src/simdjson_engine.cpp index 48e0ca7..b08571e 100644 --- a/src/simdjson_engine.cpp +++ b/src/simdjson_engine.cpp @@ -177,6 +177,20 @@ template error_code q_wiki(Doc doc, extraction &out, workload w) return SUCCESS; } +// $.display_name, $.works_count +template error_code q_openalex(Doc doc, extraction &out, workload w) { + ondemand::object root; + if (auto e = doc.get_object().get(root)) { return e; } + for (auto field : root) { + std::string_view key; + if (auto e = field.unescaped_key().get(key)) { return e; } + if (key == "display_name" || key == "works_count") { + take(field.value(), out, w); + } + } + return SUCCESS; +} + template error_code dispatch(Doc doc, extraction &out, query_id q, workload w) { if (w == workload::structure) { @@ -193,6 +207,7 @@ error_code dispatch(Doc doc, extraction &out, query_id q, workload w) { case query_id::nspl: return q_nspl(doc, out, w); case query_id::walmart: return q_walmart(doc, out, w); case query_id::wiki: return q_wiki(doc, out, w); + case query_id::openalex: return q_openalex(doc, out, w); } return SUCCESS; } @@ -202,14 +217,21 @@ error_code dispatch(Doc doc, extraction &out, query_id q, workload w) { extraction run_serial(const char *data, size_t size, query_id q, workload w, bool threaded, size_t batch_bytes, std::vector *trace, size_t trace_limit) { + return run_serial_format(data, size, q, w, threaded, batch_bytes, + stream_format::whitespace_delimited, trace, + trace_limit); +} + +extraction run_serial_format(const char *data, size_t size, query_id q, + workload w, bool threaded, size_t batch_bytes, + stream_format format, + std::vector *trace, + size_t trace_limit) { ondemand::parser parser; parser.threaded = threaded; extraction total; ondemand::document_stream stream; - if (!parser - .iterate_many(data, size, batch_bytes, - stream_format::whitespace_delimited) - .get(stream)) { + if (!parser.iterate_many(data, size, batch_bytes, format).get(stream)) { for (auto it = stream.begin(); it != stream.end(); ++it) { auto doc = *it; extraction one; diff --git a/src/simdjson_engine.h b/src/simdjson_engine.h index ed0dc73..3f74605 100644 --- a/src/simdjson_engine.h +++ b/src/simdjson_engine.h @@ -2,6 +2,7 @@ #define JSONBENCH_SIMDJSON_ENGINE_H #include "common.h" +#include "simdjson.h" #include @@ -31,6 +32,14 @@ extraction run_serial(const char *data, size_t size, query_id q, workload w, std::vector *trace = nullptr, size_t trace_limit = 0); +// As run_serial, but with an explicit stream format (used by the stream-format +// overhead study: whitespace_delimited vs comma_delimited on the same records). +extraction run_serial_format(const char *data, size_t size, query_id q, + workload w, bool threaded, size_t batch_bytes, + simdjson::stream_format format, + std::vector *trace = nullptr, + size_t trace_limit = 0); + // experimental::parse_many_parallel from simdjson PR #2788. extraction run_parallel(const char *data, size_t size, query_id q, workload w, size_t threads, size_t slice_bytes);