Runtime test support for multi-line json output

bpftrace can output multiple lines of valid json
instead of one large valid json blob. This adds
test runner support to check multiple json lines.

- Added a test for multiple map json output as an
example.
- Fixed a currently flakey test: histogram-finegrain

Related issues:
bpftrace#2902
bpftrace#3035

man/adoc/bpftrace.adoc

@@ -92,6 +92,8 @@ Valid values are::
 *json* +
 *text*
 
+Note: the json output is ndjson, meaning each line of the streamed output is a single blob of valid json.
+
 === *-h, --help*
 
 Print the help summary.

tests/runtime/engine/runner.py

@@ -219,7 +219,31 @@ def check_expect(expect, output):
                         return output.strip() == expect_file.read().strip()
                 else:
                     with open(expect.expect) as expect_file:
-                        return json.loads(output) == json.load(expect_file)
+                        _, file_extension = os.path.splitext(expect.expect)
+                        stripped_output = output.strip()
+                        output_lines = stripped_output.splitlines()
+
+                        # ndjson files are new line delimited blocks of json
+                        # https://github.com/ndjson/ndjson-spec
+                        if file_extension == ".ndjson":
+                            stripped_file = expect_file.read().strip()
+                            file_lines = stripped_file.splitlines()
+
+                            if len(file_lines) != len(output_lines):
+                                return False
+
+                            for x in range(len(file_lines)):
+                                if json.loads(output_lines[x]) != json.loads(file_lines[x]):
+                                    return False
+
+                            return True
+
+                        if len(output_lines) != 1:
+                            print(f"Expected a single line of json ouput. Got {len(output_lines)} lines")
+                            return False
+                        return json.loads(stripped_output) == json.load(expect_file)
+
+
             except Exception as err:
                 print("ERROR in check_result: ", err)
                 return False
@@ -470,16 +494,41 @@ def print_befores_and_after_output():
                     print('\tExpected no REGEX: ' + failed_expect.expect)
                     print('\tFound:\n' + to_utf8(output))
                 elif failed_expect.mode == "json":
-                    try:
-                        expected = json.dumps(json.loads(open(failed_expect.expect).read()), indent=2)
-                    except json.decoder.JSONDecodeError as err:
-                        expected = "Could not parse JSON: " + str(err)
-                    try:
-                        found = json.dumps(json.loads(output), indent=2)
-                    except json.decoder.JSONDecodeError as err:
-                        found = "Could not parse JSON: " + str(err)
-                    print('\tExpected JSON:\n' + expected)
-                    print('\tFound:\n' + found)
+                    _, file_extension = os.path.splitext(failed_expect.expect)
+                    # ndjson files are new line delimited blocks of json
+                    # https://github.com/ndjson/ndjson-spec
+                    if file_extension == ".ndjson":
+                        with open(failed_expect.expect) as expect_file:
+                            stripped_file = expect_file.read().strip()
+                            file_lines = stripped_file.splitlines()
+
+                            print('\tExpected JSON:\n')
+                            for x in file_lines:
+                                try:
+                                    print(json.dumps(json.loads(x), indent=2))
+                                except json.decoder.JSONDecodeError as err:
+                                    print("Could not parse JSON: " + str(err)  + '\n' + "Raw Line: " + x)
+
+                        stripped_output = output.strip()
+                        output_lines = stripped_output.splitlines()
+
+                        print('\tFound:\n')
+                        for x in output_lines:
+                            try:
+                                print(json.dumps(json.loads(x), indent=2))
+                            except json.decoder.JSONDecodeError as err:
+                                print("Could not parse JSON: " + str(err) + '\n' + "Raw Output: " + x)
+                    else:
+                        try:
+                            expected = json.dumps(json.loads(open(failed_expect.expect).read()), indent=2)
+                        except json.decoder.JSONDecodeError as err:
+                            expected = "Could not parse JSON: " + str(err) + '\n' + "Raw File: " + expected
+                        try:
+                            found = json.dumps(json.loads(output), indent=2)
+                        except json.decoder.JSONDecodeError as err:
+                            found = "Could not parse JSON: " + str(err) + '\n' + "Raw Output: " + output
+                        print('\tExpected JSON:\n' + expected)
+                        print('\tFound:\n' + found)
                 else:
                     print('\tExpected FILE:\n\t\t' + to_utf8(open(failed_expect.expect).read()))
                     print('\tFound:\n\t\t' + to_utf8(output))

tests/runtime/json-output

@@ -24,6 +24,11 @@ PROG BEGIN { @map["key1"] = 2; @map["key2"] = 3; exit(); }
 EXPECT_JSON runtime/outputs/map.json
 TIMEOUT 5
 
+NAME multiple maps
+PROG BEGIN { @map1["key1"] = 2; @map2["key2"] = 3; exit(); }
+EXPECT_JSON runtime/outputs/multiple_maps.ndjson
+TIMEOUT 5
+
 NAME histogram
 PROG BEGIN { @hist = hist(2); @hist = hist(1025); exit(); }
 EXPECT_JSON runtime/outputs/hist.json
@@ -40,11 +45,10 @@ EXPECT_JSON runtime/outputs/hist_multiple.json
 TIMEOUT 5
 
 NAME histogram-finegrain
-PROG i:us:100 { @ = hist(@n++,3); if (@n > 1023) { delete(@n); exit(); }}
+PROG BEGIN { $i = 0; while ($i < 1024) { @ = hist($i, 3); $i++; } exit(); }
 EXPECT_JSON runtime/outputs/hist_2args.json
 TIMEOUT 5
 
-
 NAME linear histogram
 PROG BEGIN { @h = lhist(2, 0, 100, 10); @h = lhist(50, 0, 100, 10); @h = lhist(1000, 0, 100, 10); exit(); }
 EXPECT_JSON runtime/outputs/lhist.json

tests/runtime/outputs/multiple_maps.ndjson

@@ -0,0 +1,2 @@
+{"type": "map", "data": { "@map1": { "key1": 2} }}
+{"type": "map", "data": { "@map2": { "key2": 3} }}