From c571ee9b45ab268ac83371bea82b0c77fd7baf02 Mon Sep 17 00:00:00 2001 From: Jordan Rome Date: Fri, 15 Mar 2024 11:47:23 -0700 Subject: [PATCH] Runtime test support for multi-line json output bpftrace can output multiple lines of valid json instead of one large valid json blob. This adds test runner support to check multiple json lines. - Added a test for multiple map json output as an example. - Fixed a currently flakey test: histogram-finegrain Related issues: https://github.com/bpftrace/bpftrace/issues/2902 https://github.com/bpftrace/bpftrace/issues/3035 --- man/adoc/bpftrace.adoc | 2 + tests/runtime/engine/runner.py | 71 ++++++++++++++++++---- tests/runtime/json-output | 8 ++- tests/runtime/outputs/multiple_maps.ndjson | 2 + 4 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 tests/runtime/outputs/multiple_maps.ndjson diff --git a/man/adoc/bpftrace.adoc b/man/adoc/bpftrace.adoc index 1dcfdd76714..030a1f4ae41 100644 --- a/man/adoc/bpftrace.adoc +++ b/man/adoc/bpftrace.adoc @@ -92,6 +92,8 @@ Valid values are:: *json* + *text* +Note: the json output is ndjson, meaning each line of the streamed output is a single blob of valid json. + === *-h, --help* Print the help summary. diff --git a/tests/runtime/engine/runner.py b/tests/runtime/engine/runner.py index 80be6a537f1..d1fa9f61439 100644 --- a/tests/runtime/engine/runner.py +++ b/tests/runtime/engine/runner.py @@ -219,7 +219,31 @@ def check_expect(expect, output): return output.strip() == expect_file.read().strip() else: with open(expect.expect) as expect_file: - return json.loads(output) == json.load(expect_file) + _, file_extension = os.path.splitext(expect.expect) + stripped_output = output.strip() + output_lines = stripped_output.splitlines() + + # ndjson files are new line delimited blocks of json + # https://github.com/ndjson/ndjson-spec + if file_extension == ".ndjson": + stripped_file = expect_file.read().strip() + file_lines = stripped_file.splitlines() + + if len(file_lines) != len(output_lines): + return False + + for x in range(len(file_lines)): + if json.loads(output_lines[x]) != json.loads(file_lines[x]): + return False + + return True + + if len(output_lines) != 1: + print(f"Expected a single line of json ouput. Got {len(output_lines)} lines") + return False + return json.loads(stripped_output) == json.load(expect_file) + + except Exception as err: print("ERROR in check_result: ", err) return False @@ -470,16 +494,41 @@ def print_befores_and_after_output(): print('\tExpected no REGEX: ' + failed_expect.expect) print('\tFound:\n' + to_utf8(output)) elif failed_expect.mode == "json": - try: - expected = json.dumps(json.loads(open(failed_expect.expect).read()), indent=2) - except json.decoder.JSONDecodeError as err: - expected = "Could not parse JSON: " + str(err) - try: - found = json.dumps(json.loads(output), indent=2) - except json.decoder.JSONDecodeError as err: - found = "Could not parse JSON: " + str(err) - print('\tExpected JSON:\n' + expected) - print('\tFound:\n' + found) + _, file_extension = os.path.splitext(failed_expect.expect) + # ndjson files are new line delimited blocks of json + # https://github.com/ndjson/ndjson-spec + if file_extension == ".ndjson": + with open(failed_expect.expect) as expect_file: + stripped_file = expect_file.read().strip() + file_lines = stripped_file.splitlines() + + print('\tExpected JSON:\n') + for x in file_lines: + try: + print(json.dumps(json.loads(x), indent=2)) + except json.decoder.JSONDecodeError as err: + print("Could not parse JSON: " + str(err) + '\n' + "Raw Line: " + x) + + stripped_output = output.strip() + output_lines = stripped_output.splitlines() + + print('\tFound:\n') + for x in output_lines: + try: + print(json.dumps(json.loads(x), indent=2)) + except json.decoder.JSONDecodeError as err: + print("Could not parse JSON: " + str(err) + '\n' + "Raw Output: " + x) + else: + try: + expected = json.dumps(json.loads(open(failed_expect.expect).read()), indent=2) + except json.decoder.JSONDecodeError as err: + expected = "Could not parse JSON: " + str(err) + '\n' + "Raw File: " + expected + try: + found = json.dumps(json.loads(output), indent=2) + except json.decoder.JSONDecodeError as err: + found = "Could not parse JSON: " + str(err) + '\n' + "Raw Output: " + output + print('\tExpected JSON:\n' + expected) + print('\tFound:\n' + found) else: print('\tExpected FILE:\n\t\t' + to_utf8(open(failed_expect.expect).read())) print('\tFound:\n\t\t' + to_utf8(output)) diff --git a/tests/runtime/json-output b/tests/runtime/json-output index c269555af1b..393662b9fd2 100644 --- a/tests/runtime/json-output +++ b/tests/runtime/json-output @@ -24,6 +24,11 @@ PROG BEGIN { @map["key1"] = 2; @map["key2"] = 3; exit(); } EXPECT_JSON runtime/outputs/map.json TIMEOUT 5 +NAME multiple maps +PROG BEGIN { @map1["key1"] = 2; @map2["key2"] = 3; exit(); } +EXPECT_JSON runtime/outputs/multiple_maps.ndjson +TIMEOUT 5 + NAME histogram PROG BEGIN { @hist = hist(2); @hist = hist(1025); exit(); } EXPECT_JSON runtime/outputs/hist.json @@ -40,11 +45,10 @@ EXPECT_JSON runtime/outputs/hist_multiple.json TIMEOUT 5 NAME histogram-finegrain -PROG i:us:100 { @ = hist(@n++,3); if (@n > 1023) { delete(@n); exit(); }} +PROG BEGIN { $i = 0; while ($i < 1024) { @ = hist($i, 3); $i++; } exit(); } EXPECT_JSON runtime/outputs/hist_2args.json TIMEOUT 5 - NAME linear histogram PROG BEGIN { @h = lhist(2, 0, 100, 10); @h = lhist(50, 0, 100, 10); @h = lhist(1000, 0, 100, 10); exit(); } EXPECT_JSON runtime/outputs/lhist.json diff --git a/tests/runtime/outputs/multiple_maps.ndjson b/tests/runtime/outputs/multiple_maps.ndjson new file mode 100644 index 00000000000..93157d71cf0 --- /dev/null +++ b/tests/runtime/outputs/multiple_maps.ndjson @@ -0,0 +1,2 @@ +{"type": "map", "data": { "@map1": { "key1": 2} }} +{"type": "map", "data": { "@map2": { "key2": 3} }}