Skip to content

Commit

Permalink
Create benchmark harness for file-based rules/pipelines (#107)
Browse files Browse the repository at this point in the history 
* initial wip version of a parameterized jmh runner for rules and pipelines

* implement in-memory rule service

to be able to use guice in the benchmarks, so we can reuse the production bindings, we need in-memory variants of the services we have in order to avoid benchmarking database access
the mongodb backed services have been renamed and are bound by default

* [wip] adding in-memory rule service

use in-memory grok service from #2914
add some license headers
some in-memory services are still missing and the pom packaging needs work

* add in-memory pipeline and stream connections service

dummy stream service implementation because upstream will change this anyway in the near future
use in-memory module instead of anonymous module

* make in-memory services singletons so we can load data into them

actually read benchmarks from disk and populate the stores etc
see `match_all_rule` `benchmark.toml` for an example of how to set up a benchmark

* split pipeline plugin into parent, benchmarks and actual plugin

lots of refactoring needed because the benchmarks need to be build as a separate module due to jmh requirements
explicit mention of jmh annotation processing necessary because something overrides the setting, making classpath autodetection barf
this requires an updated manifest to work with graylog-project, will push separately

slight tweaks to the benchmark class

* support no-fork mode (-Dprofile) and running a single benchark (-Dbenchmark.name) for debugging/profiling

add yourkit profiling commands as comments (useful to enable when digging into why a benchmark is slow) (manually include the yourkit -redist jar to classpath to use)
add another multi-stage benchmark as baseline for stage performance

* make build work again

another web-parent artifact: we need to disable the globally enabled frontend plugin

* add benchmark

* move misplaced files due to rebasing madness

* move eslint file

* remove old files left by improper rebase
  • Loading branch information
@kroepke
kroepke committed Oct 7, 2016
1 parent 95562d5 commit 191e3b9
Show file tree
Hide file tree
Showing 271 changed files with 1,709 additions and 523 deletions.
56 changes: 38 additions & 18 deletions benchmarks/pom.xml
View file
Expand Up @@ -33,9 +33,14 @@ THE POSSIBILITY OF SUCH DAMAGE.
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.graylog.benchmarks</groupId>
<parent>
<groupId>org.graylog.plugins</groupId>
<artifactId>graylog-plugin-pipeline-processor-parent</artifactId>
<version>1.2.0-SNAPSHOT</version>
</parent>

<artifactId>pipeline-benchmarks</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
<packaging>jar</packaging>

<name>Message processing benchmarks</name>
Expand All @@ -53,8 +58,14 @@ THE POSSIBILITY OF SUCH DAMAGE.
</dependency>
<dependency>
<groupId>org.graylog.plugins</groupId>
<artifactId>pipeline-processor</artifactId>
<version>1.0.0-SNAPSHOT</version>
<artifactId>graylog-plugin-pipeline-processor</artifactId>
<version>1.2.0-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>com.moandjiezana.toml</groupId>
<artifactId>toml4j</artifactId>
<version>0.7.1</version>
</dependency>

<dependency>
Expand All @@ -65,7 +76,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.0.40-beta</version>
<version>2.0.86-beta</version>
</dependency>

<!-- jmh -->
Expand All @@ -84,22 +95,31 @@ THE POSSIBILITY OF SUCH DAMAGE.

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jmh.version>1.11.3</jmh.version>
<jmh.version>1.14.1</jmh.version>
<javac.target>1.8</javac.target>
<uberjar.name>pipeline-benchmarks</uberjar.name>
<graylog.version>2.0.0-alpha.5-SNAPSHOT</graylog.version>
<graylog.version>2.2.0-SNAPSHOT</graylog.version>
</properties>

<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<compilerVersion>${javac.target}</compilerVersion>
<source>${javac.target}</source>
<target>${javac.target}</target>
<!-- this seems to be necessary because one of the graylog parent modules explicitly sets the processors -->
<annotationProcessors>
<annotationProcessor>org.openjdk.jmh.generators.BenchmarkProcessor</annotationProcessor>
</annotationProcessors>
</configuration>
</plugin>
<plugin>
Expand All @@ -109,7 +129,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.2</version>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -147,39 +167,39 @@ THE POSSIBILITY OF SUCH DAMAGE.
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>2.5</version>
<version>2.6.1</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.1</version>
<version>2.8.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.1</version>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>2.10.4</version>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<version>3.0.1</version>
</plugin>
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.3</version>
<version>3.5.1</version>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<version>3.0.1</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<version>2.19.1</version>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
Expand Down
24 changes: 15 additions & 9 deletions ...enchmarks/pipeline/PipelineBenchmark.java → ...marks/pipeline/FilterchainVsPipeline.java
View file
Expand Up @@ -24,6 +24,9 @@
import com.google.common.eventbus.EventBus;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineStreamConnectionsService;
import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService;
import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService;
import org.graylog.plugins.pipelineprocessor.db.PipelineDao;
import org.graylog.plugins.pipelineprocessor.db.PipelineService;
import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService;
Expand Down Expand Up @@ -84,15 +87,15 @@
import static org.mockito.Mockito.withSettings;

@SuppressWarnings("Duplicates")
public class PipelineBenchmark {
public class FilterchainVsPipeline {

@State(Scope.Benchmark)
public static class InterpreterState {

private final PipelineInterpreter interpreter;

public InterpreterState() {
final RuleService ruleService = mock(RuleService.class);
final RuleService ruleService = mock(MongoDbRuleService.class);
when(ruleService.loadAll()).thenReturn(Collections.singleton(
RuleDao.create("abc",
"title",
Expand All @@ -106,7 +109,7 @@ public InterpreterState() {
null)
));

final PipelineService pipelineService = mock(PipelineService.class);
final PipelineService pipelineService = mock(MongoDbPipelineService.class);
when(pipelineService.loadAll()).thenReturn(Collections.singleton(
PipelineDao.create("cde", "title", "description",
"pipeline \"pipeline\"\n" +
Expand All @@ -117,7 +120,7 @@ public InterpreterState() {
null)
));

final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(PipelineStreamConnectionsService.class);
final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(MongoDbPipelineStreamConnectionsService.class);
final PipelineConnections pipelineStreamConnection = PipelineConnections.create(null,
"default",
newHashSet("cde"));
Expand All @@ -131,13 +134,14 @@ public InterpreterState() {

final PipelineRuleParser parser = setupParser(functions);

final MetricRegistry metricRegistry = new MetricRegistry();
interpreter = new PipelineInterpreter(
ruleService,
pipelineService,
pipelineStreamConnectionsService,
parser,
mock(Journal.class),
mock(MetricRegistry.class),
metricRegistry,
Executors.newScheduledThreadPool(1),
mock(EventBus.class)
);
Expand Down Expand Up @@ -196,6 +200,7 @@ public void setup() {
mock(NotificationService.class),
streamService
);
final EventBus eventBus = new EventBus();
ExecutorService daemonExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(
true).build());
when(engineFactory.create(any(), any())).thenReturn(
Expand All @@ -207,18 +212,19 @@ public void setup() {
final StreamRouter streamRouter = new StreamRouter(streamService,
serverStatus,
engineFactory,
eventBus,
scheduler);

// drools
final DroolsEngine droolsEngine = new DroolsEngine(Collections.emptySet());
final FilterService filterService = mock(FilterService.class);
when(filterService.loadAll()).thenReturn(Collections.emptySet());

filters.add(new ExtractorFilter(inputService));
filters.add(new StaticFieldFilter(inputService));
filters.add(new ExtractorFilter(inputService, eventBus, scheduler));
filters.add(new StaticFieldFilter(inputService, eventBus, scheduler));
filters.add(new StreamMatcherFilter(streamRouter));
if (!noDrools) {
filters.add(new RulesFilter(droolsEngine, filterService));
filters.add(new RulesFilter(droolsEngine, filterService, eventBus, scheduler));
}

filterChain = new MessageFilterChainProcessor(metricRegistry,
Expand All @@ -243,7 +249,7 @@ public static <T> T mock(Class<T> classToMock) {

public static void main(String[] args) throws RunnerException {
Options opt = new OptionsBuilder()
.include(PipelineBenchmark.class.getSimpleName())
.include(FilterchainVsPipeline.class.getSimpleName())
.warmupIterations(5)
.measurementIterations(5)
.threads(1)
Expand Down

0 comments on commit 191e3b9

Please sign in to comment.