From 8a8a3ad6df53dbd2d60b3e04edba521eed75fd0b Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Tue, 8 Jan 2019 19:15:29 +0100 Subject: [PATCH 1/7] [benchmark] Limit setup overhead detection (>20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For really small runtimes < 20 μs this method of setup overhead detection doesn’t work. Even 1μs change in 20μs runtime is 5%. Just return no overhead. --- benchmark/scripts/Benchmark_Driver | 2 +- benchmark/scripts/test_Benchmark_Driver.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmark/scripts/Benchmark_Driver b/benchmark/scripts/Benchmark_Driver index 4faa9a1465e4c..549bd18e370b3 100755 --- a/benchmark/scripts/Benchmark_Driver +++ b/benchmark/scripts/Benchmark_Driver @@ -442,7 +442,7 @@ class BenchmarkDoctor(object): [[result.samples.min for result in i_series] for i_series in [select(measurements, num_iters=i) for i in [1, 2]]]] - setup = int(round(2.0 * (ti1 - ti2))) + setup = int(round(2.0 * (ti1 - ti2))) if ti2 > 20 else 0 ratio = (setup / ti1) if ti1 > 0 else 0 return (setup, ratio) diff --git a/benchmark/scripts/test_Benchmark_Driver.py b/benchmark/scripts/test_Benchmark_Driver.py index 6906074d75d1b..56eeb630b4094 100644 --- a/benchmark/scripts/test_Benchmark_Driver.py +++ b/benchmark/scripts/test_Benchmark_Driver.py @@ -728,11 +728,18 @@ def test_benchmark_has_no_significant_setup_overhead(self): 'SO O i2a': _PTR(min=67), 'SO O i2b': _PTR(min=68)}) doctor.analyze({'name': 'Zero', 'Zero O i1a': _PTR(min=0), 'Zero O i2a': _PTR(min=0)}) + doctor.analyze({ + 'name': 'OOO', # Out Of Order + # Impossible to detect overhead -- limits of precision: + # Even 1μs change in 20μs runtime is 5%. + 'OOO O i1a': _PTR(min=21), + 'OOO O i2a': _PTR(min=20)}) output = out.getvalue() self.assertIn('runtime: ', output) self.assertNotIn('NoOverhead', output) self.assertNotIn('ZeroRuntime', output) + self.assertNotIn('OOO', output) self.assert_contains( ["'SO' has setup overhead of 4 μs (5.8%)."], self.logs['error']) From 2096151ee9f0e2925026a8d2fa2442b0294b4a18 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Tue, 8 Jan 2019 19:16:40 +0100 Subject: [PATCH 2/7] [benchmark] BenchmarkDoctor: Lower runtime limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warn about runtimes under 20 μs and flag 0 μs runtimes as errors. --- benchmark/scripts/Benchmark_Driver | 13 +++++++++++++ benchmark/scripts/test_Benchmark_Driver.py | 16 +++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/benchmark/scripts/Benchmark_Driver b/benchmark/scripts/Benchmark_Driver index 549bd18e370b3..746402807724d 100755 --- a/benchmark/scripts/Benchmark_Driver +++ b/benchmark/scripts/Benchmark_Driver @@ -435,6 +435,19 @@ class BenchmarkDoctor(object): "Decrease the workload of '%s' by a factor of %d (%d), to be " "less than %d μs.", name, factor(2), factor(10), threshold) + threshold = 20 + if runtime < threshold: + log = (BenchmarkDoctor.log_runtime.error if runtime == 0 else + BenchmarkDoctor.log_runtime.warning) + log("'%s' execution took %d μs.", name, runtime) + + BenchmarkDoctor.log_runtime.info( + "Ensure the workload of '%s' has a properly measurable size" + " (runtime > %d μs) and is not eliminated by the compiler (use" + " `blackHole` function if necessary)." if runtime == 0 else + "Increase the workload of '%s' to be more than %d μs.", + name, threshold) + @staticmethod def _setup_overhead(measurements): select = BenchmarkDoctor._select diff --git a/benchmark/scripts/test_Benchmark_Driver.py b/benchmark/scripts/test_Benchmark_Driver.py index 56eeb630b4094..e1ed6d332154b 100644 --- a/benchmark/scripts/test_Benchmark_Driver.py +++ b/benchmark/scripts/test_Benchmark_Driver.py @@ -667,7 +667,7 @@ def test_benchmark_name_is_at_most_40_chars_long(self): self.logs['info']) def test_benchmark_runtime_range(self): - """Optimized benchmark should run in less then 1000 μs. + """Optimized benchmark should have runtime between 20 μs and 1000 μs. Even on calm machine, benchmark with runtime of 2500 μs has 1:4 chance of being interrupted in the middle of measurement due to elapsed 10 ms @@ -687,6 +687,8 @@ def measurements(name, runtime): with captured_output() as (out, _): doctor = BenchmarkDoctor(self.args, BenchmarkDriverMock([])) + doctor.analyze(measurements('Sylph', 0)) + doctor.analyze(measurements('Unicorn', 3)) doctor.analyze(measurements('Cheetah', 200)) doctor.analyze(measurements('Hare', 1001)) doctor.analyze(measurements('Tortoise', 500000)) @@ -697,6 +699,18 @@ def measurements(name, runtime): self.assertIn('runtime: ', output) self.assertNotIn('Cheetah', output) + self.assert_contains(["'Sylph' execution took 0 μs."], + self.logs['error']) + self.assert_contains( + ["Ensure the workload of 'Sylph' has a properly measurable size" + " (runtime > 20 μs) and is not eliminated by the compiler (use " + "`blackHole` function if necessary)."], + self.logs['info']) + self.assert_contains(["'Unicorn' execution took 3 μs."], + self.logs['warning']) + self.assert_contains( + ["Increase the workload of 'Unicorn' to be more than 20 μs."], + self.logs['info']) self.assert_contains(["'Hare' execution took at least 1001 μs."], self.logs['warning']) self.assert_contains( From 84580f88fec6a8dfa36a0d2f6ee4d2746cf690bd Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Tue, 8 Jan 2019 21:53:18 +0100 Subject: [PATCH 3/7] [benchmark] Adjust new tests to Naming Convention Most of these recently added benchmarks have too low loop multiplier that results in near zero or zero measured runtime. Since fixing this will change the runtimes, it is also an opportunity to properly apply the new naming convention. --- benchmark/single-source/SetTests.swift | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmark/single-source/SetTests.swift b/benchmark/single-source/SetTests.swift index b317ff815ff2d..20776c8d4517c 100644 --- a/benchmark/single-source/SetTests.swift +++ b/benchmark/single-source/SetTests.swift @@ -58,7 +58,7 @@ let setQ: Set = { public let SetTests = [ // Mnemonic: number after name is percentage of common elements in input sets. BenchmarkInfo( - name: "Set.Empty.IsSubsetInt0", + name: "Set.isSubset.Empty.Int0", runFunction: { n in run_SetIsSubsetInt(setE, setAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), @@ -94,46 +94,46 @@ public let SetTests = [ setUpFunction: { blackHole([setP, setQ]) }), BenchmarkInfo( - name: "Set.Empty.IsDisjointInt0", + name: "Set.isDisjoint.Empty.Int0", runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( - name: "Set.Empty.IsDisjointBox0", + name: "Set.isDisjoint.Empty.Box0", runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), BenchmarkInfo( - name: "SetIsDisjointInt0", + name: "Set.isDisjoint.Int0", runFunction: { n in run_SetIsDisjointInt(setAB, setCD, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setCD]) }), BenchmarkInfo( - name: "SetIsDisjointBox0", + name: "Set.isDisjoint.Box0", runFunction: { n in run_SetIsDisjointBox(setOAB, setOCD, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOAB, setOCD]) }), BenchmarkInfo( - name: "SetIsDisjointInt25", + name: "Set.isDisjoint.Int25", runFunction: { n in run_SetIsDisjointInt(setB, setAB, false, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setB, setAB]) }), BenchmarkInfo( - name: "SetIsDisjointBox25", + name: "Set.isDisjoint.Box25", runFunction: { n in run_SetIsDisjointBox(setOB, setOAB, false, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOB, setOAB]) }), BenchmarkInfo( - name: "SetIsDisjointInt50", + name: "Set.isDisjoint.Int50", runFunction: { n in run_SetIsDisjointInt(setY, setXY, false, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setY, setXY]) }), BenchmarkInfo( - name: "SetIsDisjointInt100", + name: "Set.isDisjoint.Int100", runFunction: { n in run_SetIsDisjointInt(setP, setQ, false, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setP, setQ]) }), - + BenchmarkInfo( name: "SetSymmetricDifferenceInt0", runFunction: { n in run_SetSymmetricDifferenceInt(setAB, setCD, countABCD, 10 * n) }, @@ -228,12 +228,12 @@ public let SetTests = [ setUpFunction: { blackHole([setP, setQ]) }), BenchmarkInfo( - name: "Set.Empty.SubtractingInt0", + name: "Set.subtracting.Empty.Int0", runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 10 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( - name: "Set.Empty.SubtractingBox0", + name: "Set.subtracting.Empty.Box0", runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 10 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), From c011bd70c3780b4540c26616fe54e6ba41b76674 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Tue, 8 Jan 2019 21:54:59 +0100 Subject: [PATCH 4/7] [benchmark] Add swapped Set.*.Empty variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let’s also test Empty sets as the right-hand side parameter. --- benchmark/single-source/SetTests.swift | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/benchmark/single-source/SetTests.swift b/benchmark/single-source/SetTests.swift index 20776c8d4517c..629fc77df38af 100644 --- a/benchmark/single-source/SetTests.swift +++ b/benchmark/single-source/SetTests.swift @@ -62,6 +62,11 @@ public let SetTests = [ runFunction: { n in run_SetIsSubsetInt(setE, setAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), + BenchmarkInfo( + name: "Set.isSubset.Int0.Empty", + runFunction: { n in run_SetIsSubsetInt(setAB, setE, false, 5000 * n) }, + tags: [.validation, .api, .Set], + setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( name: "SetIsSubsetInt0", runFunction: { n in run_SetIsSubsetInt(setAB, setCD, false, 5000 * n) }, @@ -98,11 +103,21 @@ public let SetTests = [ runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), + BenchmarkInfo( + name: "Set.isDisjoint.Int0.Empty", + runFunction: { n in run_SetIsDisjointInt(setAB, setE, true, 50 * n) }, + tags: [.validation, .api, .Set], + setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( name: "Set.isDisjoint.Empty.Box0", runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 50 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), + BenchmarkInfo( + name: "Set.isDisjoint.Box0.Empty", + runFunction: { n in run_SetIsDisjointBox(setOAB, setOE, true, 50 * n) }, + tags: [.validation, .api, .Set], + setUpFunction: { blackHole([setOAB, setOE]) }), BenchmarkInfo( name: "Set.isDisjoint.Int0", runFunction: { n in run_SetIsDisjointInt(setAB, setCD, true, 50 * n) }, @@ -232,11 +247,21 @@ public let SetTests = [ runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 10 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), + BenchmarkInfo( + name: "Set.subtracting.Int0.Empty", + runFunction: { n in run_SetSubtractingInt(setAB, setE, countAB, 10 * n) }, + tags: [.validation, .api, .Set], + setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( name: "Set.subtracting.Empty.Box0", runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 10 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), + BenchmarkInfo( + name: "Set.subtracting.Box0.Empty", + runFunction: { n in run_SetSubtractingBox(setOAB, setOE, countAB, 10 * n) }, + tags: [.validation, .api, .Set], + setUpFunction: { blackHole([setOAB, setOE]) }), BenchmarkInfo( name: "SetSubtractingInt0", runFunction: { n in run_SetSubtractingInt(setAB, setCD, countAB, 10 * n) }, From b4d8172228867526f8dad759e1fda7cf3429e306 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Wed, 9 Jan 2019 12:22:48 +0100 Subject: [PATCH 5/7] [benchmark] Adjust loop multipliers Set.* Increase the multipliers to get reliably measurable runtimes. --- benchmark/single-source/SetTests.swift | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmark/single-source/SetTests.swift b/benchmark/single-source/SetTests.swift index 629fc77df38af..2a3d5dd058953 100644 --- a/benchmark/single-source/SetTests.swift +++ b/benchmark/single-source/SetTests.swift @@ -100,22 +100,22 @@ public let SetTests = [ BenchmarkInfo( name: "Set.isDisjoint.Empty.Int0", - runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 50 * n) }, + runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( name: "Set.isDisjoint.Int0.Empty", - runFunction: { n in run_SetIsDisjointInt(setAB, setE, true, 50 * n) }, + runFunction: { n in run_SetIsDisjointInt(setAB, setE, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( name: "Set.isDisjoint.Empty.Box0", - runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 50 * n) }, + runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), BenchmarkInfo( name: "Set.isDisjoint.Box0.Empty", - runFunction: { n in run_SetIsDisjointBox(setOAB, setOE, true, 50 * n) }, + runFunction: { n in run_SetIsDisjointBox(setOAB, setOE, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOAB, setOE]) }), BenchmarkInfo( @@ -130,22 +130,22 @@ public let SetTests = [ setUpFunction: { blackHole([setOAB, setOCD]) }), BenchmarkInfo( name: "Set.isDisjoint.Int25", - runFunction: { n in run_SetIsDisjointInt(setB, setAB, false, 50 * n) }, + runFunction: { n in run_SetIsDisjointInt(setB, setAB, false, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setB, setAB]) }), BenchmarkInfo( name: "Set.isDisjoint.Box25", - runFunction: { n in run_SetIsDisjointBox(setOB, setOAB, false, 50 * n) }, + runFunction: { n in run_SetIsDisjointBox(setOB, setOAB, false, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOB, setOAB]) }), BenchmarkInfo( name: "Set.isDisjoint.Int50", - runFunction: { n in run_SetIsDisjointInt(setY, setXY, false, 50 * n) }, + runFunction: { n in run_SetIsDisjointInt(setY, setXY, false, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setY, setXY]) }), BenchmarkInfo( name: "Set.isDisjoint.Int100", - runFunction: { n in run_SetIsDisjointInt(setP, setQ, false, 50 * n) }, + runFunction: { n in run_SetIsDisjointInt(setP, setQ, false, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setP, setQ]) }), @@ -244,22 +244,22 @@ public let SetTests = [ BenchmarkInfo( name: "Set.subtracting.Empty.Int0", - runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 10 * n) }, + runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( name: "Set.subtracting.Int0.Empty", - runFunction: { n in run_SetSubtractingInt(setAB, setE, countAB, 10 * n) }, + runFunction: { n in run_SetSubtractingInt(setAB, setE, countAB, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( name: "Set.subtracting.Empty.Box0", - runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 10 * n) }, + runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), BenchmarkInfo( name: "Set.subtracting.Box0.Empty", - runFunction: { n in run_SetSubtractingBox(setOAB, setOE, countAB, 10 * n) }, + runFunction: { n in run_SetSubtractingBox(setOAB, setOE, countAB, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOAB, setOE]) }), BenchmarkInfo( From 2c271493d5ae1fa27b0b21e6485714c59806081b Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Wed, 9 Jan 2019 18:01:06 +0100 Subject: [PATCH 6/7] [benchmark] Limit of Accuracy in Setup Overhead Clarified limit of accuracy in setup overhead detection. --- benchmark/scripts/Benchmark_Driver | 3 ++- benchmark/scripts/test_Benchmark_Driver.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmark/scripts/Benchmark_Driver b/benchmark/scripts/Benchmark_Driver index 746402807724d..fe70d7d4108ab 100755 --- a/benchmark/scripts/Benchmark_Driver +++ b/benchmark/scripts/Benchmark_Driver @@ -455,7 +455,8 @@ class BenchmarkDoctor(object): [[result.samples.min for result in i_series] for i_series in [select(measurements, num_iters=i) for i in [1, 2]]]] - setup = int(round(2.0 * (ti1 - ti2))) if ti2 > 20 else 0 + setup = (int(round(2.0 * (ti1 - ti2))) if ti2 > 20 # limit of accuracy + else 0) ratio = (setup / ti1) if ti1 > 0 else 0 return (setup, ratio) diff --git a/benchmark/scripts/test_Benchmark_Driver.py b/benchmark/scripts/test_Benchmark_Driver.py index e1ed6d332154b..51cc95b29afbf 100644 --- a/benchmark/scripts/test_Benchmark_Driver.py +++ b/benchmark/scripts/test_Benchmark_Driver.py @@ -743,17 +743,17 @@ def test_benchmark_has_no_significant_setup_overhead(self): doctor.analyze({'name': 'Zero', 'Zero O i1a': _PTR(min=0), 'Zero O i2a': _PTR(min=0)}) doctor.analyze({ - 'name': 'OOO', # Out Of Order - # Impossible to detect overhead -- limits of precision: + 'name': 'LOA', # Limit of Accuracy + # Impossible to detect overhead: # Even 1μs change in 20μs runtime is 5%. - 'OOO O i1a': _PTR(min=21), - 'OOO O i2a': _PTR(min=20)}) + 'LOA O i1a': _PTR(min=21), + 'LOA O i2a': _PTR(min=20)}) output = out.getvalue() self.assertIn('runtime: ', output) self.assertNotIn('NoOverhead', output) self.assertNotIn('ZeroRuntime', output) - self.assertNotIn('OOO', output) + self.assertNotIn('LOA', output) self.assert_contains( ["'SO' has setup overhead of 4 μs (5.8%)."], self.logs['error']) From b4f1f4747db9eacd6030fa0edf97bd60819d9c38 Mon Sep 17 00:00:00 2001 From: Pavol Vaskovic Date: Thu, 10 Jan 2019 18:04:10 +0100 Subject: [PATCH 7/7] [benchmark] Remove 0s from Empty names. --- benchmark/single-source/SetTests.swift | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmark/single-source/SetTests.swift b/benchmark/single-source/SetTests.swift index 2a3d5dd058953..caa64a6f23118 100644 --- a/benchmark/single-source/SetTests.swift +++ b/benchmark/single-source/SetTests.swift @@ -58,12 +58,12 @@ let setQ: Set = { public let SetTests = [ // Mnemonic: number after name is percentage of common elements in input sets. BenchmarkInfo( - name: "Set.isSubset.Empty.Int0", + name: "Set.isSubset.Empty.Int", runFunction: { n in run_SetIsSubsetInt(setE, setAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( - name: "Set.isSubset.Int0.Empty", + name: "Set.isSubset.Int.Empty", runFunction: { n in run_SetIsSubsetInt(setAB, setE, false, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setE]) }), @@ -99,22 +99,22 @@ public let SetTests = [ setUpFunction: { blackHole([setP, setQ]) }), BenchmarkInfo( - name: "Set.isDisjoint.Empty.Int0", + name: "Set.isDisjoint.Empty.Int", runFunction: { n in run_SetIsDisjointInt(setE, setAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( - name: "Set.isDisjoint.Int0.Empty", + name: "Set.isDisjoint.Int.Empty", runFunction: { n in run_SetIsDisjointInt(setAB, setE, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( - name: "Set.isDisjoint.Empty.Box0", + name: "Set.isDisjoint.Empty.Box", runFunction: { n in run_SetIsDisjointBox(setOE, setOAB, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), BenchmarkInfo( - name: "Set.isDisjoint.Box0.Empty", + name: "Set.isDisjoint.Box.Empty", runFunction: { n in run_SetIsDisjointBox(setOAB, setOE, true, 5000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOAB, setOE]) }), @@ -243,22 +243,22 @@ public let SetTests = [ setUpFunction: { blackHole([setP, setQ]) }), BenchmarkInfo( - name: "Set.subtracting.Empty.Int0", + name: "Set.subtracting.Empty.Int", runFunction: { n in run_SetSubtractingInt(setE, setAB, 0, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setE, setAB]) }), BenchmarkInfo( - name: "Set.subtracting.Int0.Empty", + name: "Set.subtracting.Int.Empty", runFunction: { n in run_SetSubtractingInt(setAB, setE, countAB, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setAB, setE]) }), BenchmarkInfo( - name: "Set.subtracting.Empty.Box0", + name: "Set.subtracting.Empty.Box", runFunction: { n in run_SetSubtractingBox(setOE, setOAB, 0, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOE, setOAB]) }), BenchmarkInfo( - name: "Set.subtracting.Box0.Empty", + name: "Set.subtracting.Box.Empty", runFunction: { n in run_SetSubtractingBox(setOAB, setOE, countAB, 1000 * n) }, tags: [.validation, .api, .Set], setUpFunction: { blackHole([setOAB, setOE]) }),