Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Created benchmark suite for event delivery and consumption #4

Merged
merged 1 commit into from

1 participant

@ghost

The suite has 7 benchmarks that measures the performance of not only event creation but also event consumption. One of the benchmarks is a simple event dispatcher that manages callbacks with weak references - this is how I envision working with non-OO aware pure callback style APIs such as AnyEvent.

Here's some sample output from my machine:

$ ./bench.pl
Configuration: 500000 events and 60 seconds minimum test time
Loading benchmarks from bench/... found 7 files
loading bench/method-array.pl... done
loading bench/method-hash.pl... done
loading bench/objectmethod-array.pm... done
loading bench/objectmethod-cbmanager.pm... done
loading bench/objectmethod-hash.pm... done
loading bench/reflex-watches+no_event_methods.pm... done
loading bench/reflex-watches.pm... done
Generating test data... done
Measuring bench/method-array.pl... 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 done
Measuring bench/method-hash.pl... 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 done
Measuring bench/objectmethod-array.pm... 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 done
Measuring bench/objectmethod-cbmanager.pm... 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 done
Measuring bench/objectmethod-hash.pm... 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 done
Measuring bench/reflex-watches+no_event_methods.pm... 1 done
Measuring bench/reflex-watches.pm... 1 done
Report format: file events per second cost of solution
bench/method-array.pl 801392 1
bench/objectmethod-array.pm 717160 1.11745217245803
bench/method-hash.pl 481298 1.66506405594871
bench/objectmethod-hash.pm 433866 1.8470956470431
bench/objectmethod-cbmanager.pm 135025 5.935137937419
bench/reflex-watches.pm 12757 62.8197852159599
bench/reflex-watches+no_event_methods.pm 12648 63.3611638203669

@rcaputo rcaputo merged commit 8011d01 into rcaputo:master
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Aug 18, 2012
This page is out of date. Refresh to see the latest.
View
169 bench.pl
@@ -0,0 +1,169 @@
+#!/usr/bin/env perl
+
+use lib qw(bench/lib);
+
+use strict;
+use warnings;
+use Data::Dumper;
+
+$ENV{EVENT_SEND_COUNT} = 500000 unless exists $ENV{EVENT_SEND_COUNT};
+$ENV{MIN_TEST_TIME} = 60 unless exists $ENV{MIN_TEST_TIME};
+$ENV{BENCH_DIR} = 'bench' unless exists $ENV{BENCH_DIR};
+
+logger("Configuration: $ENV{EVENT_SEND_COUNT} events and $ENV{MIN_TEST_TIME} seconds minimum test time\n");
+
+my @results = doTests();
+my $error = shift(@results);
+doLog(@results);
+exit($error);
+
+exit(0);
+
+sub logger {
+ print STDERR @_;
+}
+
+sub genFiles {
+ my ($dir) = @_;
+ my $dh;
+ my @paths;
+
+ die "$dir is not a directory or is not accessable" unless -d $dir;
+
+ die "Could not opendir($dir): $!" unless opendir($dh, $dir);
+
+ foreach(readdir($dh)) {
+ next if m/^\./;
+ next if -d $_;
+
+ push(@paths, "$dir/$_");
+ }
+
+ die "Could not closedir($dir): $!" unless closedir($dh);
+
+ return @paths;
+}
+
+sub genTests {
+ my @files;
+ my @buf;
+
+ if (scalar(@ARGV) > 0) {
+ @files = @ARGV;
+ } else {
+ logger("Loading benchmarks from $ENV{BENCH_DIR}/... ");
+ @files = genFiles($ENV{BENCH_DIR});
+ logger("found ", scalar(@files), " files\n");
+ }
+
+ foreach(@files) {
+ my %test = ( file => $_ );
+
+ logger("loading $_... ");
+
+ $test{cb} = require $_;
+
+ logger("done\n");
+
+ push(@buf, \%test);
+ }
+
+ return @buf;
+}
+
+sub randomInt {
+ return int(rand(10));
+}
+
+sub genData {
+ my @buf = @_;
+
+ logger("Generating test data... ");
+
+ foreach(1 .. $ENV{EVENT_SEND_COUNT}) {
+ push(@buf, [ randomInt(), randomInt() ]);
+ }
+
+ logger("done\n");
+ return @buf;
+}
+
+sub sumData {
+ my $sum = 0;
+
+ foreach(@_) {
+ $sum += $_->[0] + $_->[1];
+ }
+
+ return $sum;
+}
+
+sub doTests {
+ my @tests = genTests();
+ my @testData = genData();
+ my $sum = sumData(@testData);
+ my $error = 0;
+
+ foreach(@tests) {
+ my $seconds = 0;
+ my $eventCount = 0;
+ my $iterations = 0;
+
+ logger("Measuring $_->{file}... ");
+
+ while(1) {
+ my ($results, $bench);
+
+ $iterations++;
+
+ $results = $_->{cb}(@testData);
+ $bench = $results->{bench};
+
+ if ($results->{sum} != $sum) {
+ logger("INVALID SUM $results->{sum} != $sum ");
+ $error = 1;
+ }
+
+ $seconds += $bench->[0];
+ $eventCount += $ENV{EVENT_SEND_COUNT};
+
+ if ($seconds >= $ENV{MIN_TEST_TIME}) {
+ last;
+ }
+
+ logger("$iterations ");
+ }
+
+ $_->{analysis} = {
+ cpuTime => $seconds,
+ eventCount => $eventCount,
+ eventsPerSecond => $eventCount / $seconds,
+ };
+
+ logger("done\n");
+ }
+
+ return ($error, @tests);
+}
+
+sub doLog {
+ my (@tests) = @_;
+
+ logger("Report format: file\tevents per second\tcost of solution\n");
+
+ foreach(sort({ $b->{analysis}->{eventsPerSecond} <=> $a->{analysis}->{eventsPerSecond} } @tests)) {
+ my $analysis = $_->{analysis};
+ my $eventsPerSecond = int($analysis->{eventsPerSecond});
+ our $fastest;
+ my $cost;
+
+ if (! defined($fastest)) {
+ $fastest = $eventsPerSecond;
+ }
+
+ $cost = $fastest / $eventsPerSecond;
+
+ print "$_->{file}\t$eventsPerSecond\t$cost\n";
+ }
+}
+
View
13 bench/lib/EventBench/Reflex/Event.pm
@@ -0,0 +1,13 @@
+package EventBench::Reflex::Event;
+
+use Moose;
+use Reflex;
+
+extends 'Reflex::Event';
+
+has arg1 => ( is => 'ro', isa => 'Num', required => 1 );
+has arg2 => ( is => 'ro', isa => 'Num', required => 1 );
+
+__PACKAGE__->meta->make_immutable;
+
+1;
View
27 bench/method-array.pl
@@ -0,0 +1,27 @@
+package EventBench::Method::Array;
+
+use strict;
+use warnings;
+use Benchmark ':hireswallclock';
+
+sub receive_event {
+ my ($arg1, $arg2) = @_;
+ our($sum);
+
+ $sum += $arg1 + $arg2;
+}
+
+return sub {
+ my (@testData) = @_;
+ our $sum = 0;
+ my $bench;
+
+ $bench = timeit(1, sub {
+
+ foreach(@testData) {
+ receive_event($_->[0], $_->[1]);
+ }
+ });
+
+ return { bench => $bench, sum => $sum };
+};
View
26 bench/method-hash.pl
@@ -0,0 +1,26 @@
+package EventBench::Method::Hash;
+
+use strict;
+use warnings;
+use Benchmark ':hireswallclock';
+
+sub receive_event {
+ my (%event) = @_;
+ our $sum;
+
+ $sum += $event{arg1} + $event{arg2};
+}
+
+return sub {
+ my (@testData) = @_;
+ our $sum = 0;
+ my $bench;
+
+ $bench = timeit(1, sub {
+ foreach(@testData) {
+ receive_event(arg1 => $_->[0], arg2 => $_->[1]);
+ }
+ });
+
+ return { bench => $bench, sum => $sum };
+};
View
32 bench/objectmethod-array.pm
@@ -0,0 +1,32 @@
+package EventBench::ObjectMethod::Array;
+
+use strict;
+use warnings;
+use Benchmark ':hireswallclock';
+
+sub new {
+ return bless({}, $_[0]);
+}
+
+sub receive_event {
+ my ($self, $arg1, $arg2) = @_;
+ our $sum;
+
+ $sum += $arg1 + $arg2;
+}
+
+return sub {
+ my (@testData) = @_;
+ my $test = EventBench::ObjectMethod::Array->new;
+ our $sum = 0;
+ my $bench;
+
+ $bench = timeit(1, sub {
+
+ foreach(@testData) {
+ $test->receive_event($_->[0], $_->[1]);
+ }
+ });
+
+ return { bench => $bench, sum => $sum };
+};
View
57 bench/objectmethod-cbmanager.pm
@@ -0,0 +1,57 @@
+package EventBench::ObjectMethod::CBManager;
+
+use strict;
+use warnings;
+use Benchmark ':hireswallclock';
+use Scalar::Util qw(weaken);
+
+sub new {
+ my $self = bless({ }, $_[0]);
+ our $sum;
+
+ $self->{handlers}->{sum} = $self->weakcb(sub {
+ my ($self, %event) = @_;
+
+ $sum += $event{arg1} + $event{arg2};
+ });
+
+ return $self;
+}
+
+sub receive_event {
+ my ($self, %event) = @_;
+
+ $self->{handlers}->{$event{name}}->($self, %event);
+}
+
+#pass in code ref or method name as string
+sub weakcb {
+ my ($self, $cb) = @_;
+
+ weaken($self);
+
+ return sub {
+ my ($self) = shift(@_);
+
+ die "expected weak reference to self to be valid but it was undefined" unless defined $self;
+
+ return $self->$cb(@_);
+ };
+}
+
+return sub {
+ my (@testData) = @_;
+ my $test = EventBench::ObjectMethod::CBManager->new;
+ my $bench;
+ our $sum;
+
+ $bench = timeit(1, sub {
+ our $sum = 0;
+
+ foreach(@testData) {
+ $test->receive_event(name => 'sum', arg1 => $_->[0], arg2 => $_->[1]);
+ }
+ });
+
+ return { bench => $bench, sum => $sum };
+};
View
31 bench/objectmethod-hash.pm
@@ -0,0 +1,31 @@
+package EventBench::ObjectMethod::Hash;
+
+use strict;
+use warnings;
+use Benchmark ':hireswallclock';
+
+sub new {
+ return bless({}, $_[0]);
+}
+
+sub receive_event {
+ my ($self, %event) = @_;
+ our $sum;
+
+ $sum += $event{arg1} + $event{arg2};
+}
+
+return sub {
+ my (@testData) = @_;
+ my $test = EventBench::ObjectMethod::Hash->new;
+ our $sum = 0;
+ my $bench;
+
+ $bench = timeit(1, sub {
+ foreach(@testData) {
+ $test->receive_event(arg1 => $_->[0], arg2 => $_->[1]);
+ }
+ });
+
+ return { bench => $bench, sum => $sum };
+};
Something went wrong with that request. Please try again.