diff --git a/community/common/src/main/java/org/neo4j/scheduler/JobScheduler.java b/community/common/src/main/java/org/neo4j/scheduler/JobScheduler.java index 13379ea7d0e90..8427ce104d1e2 100644 --- a/community/common/src/main/java/org/neo4j/scheduler/JobScheduler.java +++ b/community/common/src/main/java/org/neo4j/scheduler/JobScheduler.java @@ -124,6 +124,11 @@ class Groups */ public static final Group boltLogRotation = new Group( "BoltLogRotation" ); + /** + * Rotates metrics csv files + */ + public static final Group metricsLogRotations = new Group( "MetricsLogRotations" ); + /** * Checkpoint and store flush */ diff --git a/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java b/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java index f9f007206a2f0..46d068909c799 100644 --- a/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java +++ b/community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java @@ -308,7 +308,6 @@ void rotate() finally { rotating.set( false ); - logFileLock.writeLock().unlock(); try { bufferingOutputStream.writeTo( streamWrapper ); @@ -317,6 +316,7 @@ void rotate() { rotationListener.rotationError( e, streamWrapper ); } + logFileLock.writeLock().unlock(); } }; diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsExtension.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsExtension.java index f77886a6569eb..562d81bb5b272 100644 --- a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsExtension.java +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsExtension.java @@ -21,6 +21,7 @@ import com.codahale.metrics.MetricRegistry; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.spi.KernelContext; @@ -30,6 +31,7 @@ import org.neo4j.metrics.output.CompositeEventReporter; import org.neo4j.metrics.output.EventReporterBuilder; import org.neo4j.metrics.source.Neo4jMetricsBuilder; +import org.neo4j.scheduler.JobScheduler; public class MetricsExtension implements Lifecycle { @@ -42,10 +44,13 @@ public class MetricsExtension implements Lifecycle { LogService logService = dependencies.logService(); Config configuration = dependencies.configuration(); + FileSystemAbstraction fileSystem = dependencies.fileSystemAbstraction(); + JobScheduler scheduler = dependencies.scheduler(); logger = logService.getUserLog( getClass() ); MetricRegistry registry = new MetricRegistry(); - reporter = new EventReporterBuilder( configuration, registry, logger, kernelContext, life ).build(); + reporter = new EventReporterBuilder( configuration, registry, logger, kernelContext, life, fileSystem, + scheduler ).build(); metricsBuilt = new Neo4jMetricsBuilder( registry, reporter, configuration, logService, kernelContext, dependencies, life ).build(); } diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsKernelExtensionFactory.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsKernelExtensionFactory.java index f182ec3d7e66c..20f0847b5d6a7 100644 --- a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsKernelExtensionFactory.java +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsKernelExtensionFactory.java @@ -19,12 +19,14 @@ */ package org.neo4j.metrics; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.lifecycle.Lifecycle; import org.neo4j.metrics.source.Neo4jMetricsBuilder; +import org.neo4j.scheduler.JobScheduler; public class MetricsKernelExtensionFactory extends KernelExtensionFactory { @@ -33,6 +35,10 @@ public interface Dependencies extends Neo4jMetricsBuilder.Dependencies Config configuration(); LogService logService(); + + FileSystemAbstraction fileSystemAbstraction(); + + JobScheduler scheduler(); } public MetricsKernelExtensionFactory() diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsSettings.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsSettings.java index 715176af71226..3190a5f79d285 100644 --- a/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsSettings.java +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsSettings.java @@ -28,12 +28,17 @@ import org.neo4j.helpers.HostnamePort; import static org.neo4j.kernel.configuration.Settings.BOOLEAN; +import static org.neo4j.kernel.configuration.Settings.BYTES; import static org.neo4j.kernel.configuration.Settings.DURATION; import static org.neo4j.kernel.configuration.Settings.FALSE; import static org.neo4j.kernel.configuration.Settings.HOSTNAME_PORT; +import static org.neo4j.kernel.configuration.Settings.INTEGER; import static org.neo4j.kernel.configuration.Settings.STRING; +import static org.neo4j.kernel.configuration.Settings.TRUE; import static org.neo4j.kernel.configuration.Settings.buildSetting; +import static org.neo4j.kernel.configuration.Settings.min; import static org.neo4j.kernel.configuration.Settings.pathSetting; +import static org.neo4j.kernel.configuration.Settings.range; import static org.neo4j.kernel.configuration.Settings.setting; /** @@ -52,7 +57,7 @@ public class MetricsSettings implements LoadableConfig @Description( "The default enablement value for all the supported metrics. Set this to `false` to turn off all " + "metrics by default. The individual settings can then be used to selectively re-enable specific " + "metrics." ) - public static Setting metricsEnabled = setting( "metrics.enabled", BOOLEAN, FALSE ); + public static Setting metricsEnabled = setting( "metrics.enabled", BOOLEAN, TRUE ); @Description( "The default enablement value for all Neo4j specific support metrics. Set this to `false` to turn " + "off all Neo4j specific metrics by default. The individual `metrics.neo4j.*` metrics can then be " + @@ -120,7 +125,7 @@ public class MetricsSettings implements LoadableConfig // CSV settings @Description( "Set to `true` to enable exporting metrics to CSV files" ) - public static Setting csvEnabled = setting( "metrics.csv.enabled", BOOLEAN, FALSE ); + public static Setting csvEnabled = setting( "metrics.csv.enabled", BOOLEAN, TRUE ); @Description( "The target location of the CSV files: a path to a directory wherein a CSV file per reported " + "field will be written." ) @@ -130,6 +135,15 @@ public class MetricsSettings implements LoadableConfig "the CSV files." ) public static Setting csvInterval = setting( "metrics.csv.interval", DURATION, "3s" ); + @Description( "The file size in bytes at which the csv files will auto-rotate. If set to zero then no " + + "rotation will occur. Accepts a binary suffix `k`, `m` or `g`." ) + public static final Setting csvRotationThreshold = buildSetting( "metrics.csv.rotation.size", + BYTES, "20m" ).constraint( range( 0L, Long.MAX_VALUE ) ).build(); + + @Description( "Maximum number of history files for the csv files." ) + public static final Setting csvMaxArchives = buildSetting( "metrics.csv.rotation.keep_number", + INTEGER, "7" ).constraint( min( 1 ) ).build(); + // Graphite settings @Description( "Set to `true` to enable exporting metrics to Graphite." ) public static Setting graphiteEnabled = setting( "metrics.graphite.enabled", BOOLEAN, FALSE ); diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/output/CsvOutput.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/CsvOutput.java index 8d53cc88a68b4..d5d2e33c144d7 100644 --- a/enterprise/metrics/src/main/java/org/neo4j/metrics/output/CsvOutput.java +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/CsvOutput.java @@ -20,13 +20,10 @@ package org.neo4j.metrics.output; import com.codahale.metrics.Counter; -import com.codahale.metrics.CsvReporter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; -import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; import com.codahale.metrics.Timer; import java.io.File; @@ -34,11 +31,16 @@ import java.util.Locale; import java.util.SortedMap; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.lifecycle.Lifecycle; import org.neo4j.logging.Log; +import org.neo4j.logging.RotatingFileOutputStreamSupplier; +import org.neo4j.metrics.MetricsSettings; +import org.neo4j.scheduler.JobScheduler; import static org.neo4j.metrics.MetricsSettings.csvEnabled; import static org.neo4j.metrics.MetricsSettings.csvInterval; @@ -50,19 +52,24 @@ public class CsvOutput implements Lifecycle, EventReporter private final MetricRegistry registry; private final Log logger; private final KernelContext kernelContext; - private ScheduledReporter csvReporter; + private final FileSystemAbstraction fileSystem; + private final JobScheduler scheduler; + private RotatableCsvReporter csvReporter; private File outputPath; - public CsvOutput( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext ) + CsvOutput( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext, + FileSystemAbstraction fileSystem, JobScheduler scheduler ) { this.config = config; this.registry = registry; this.logger = logger; this.kernelContext = kernelContext; + this.fileSystem = fileSystem; + this.scheduler = scheduler; } @Override - public void init() + public void init() throws IOException { // Setup CSV reporting File configuredPath = config.get( csvPath ); @@ -71,12 +78,15 @@ public void init() throw new IllegalArgumentException( csvPath.name() + " configuration is required since " + csvEnabled.name() + " is enabled" ); } + Long rotationThreshold = config.get( MetricsSettings.csvRotationThreshold ); + Integer maxArchives = config.get( MetricsSettings.csvMaxArchives ); outputPath = absoluteFileOrRelativeTo( kernelContext.storeDir(), configuredPath ); - csvReporter = CsvReporter.forRegistry( registry ) + csvReporter = RotatableCsvReporter.forRegistry( registry ) .convertRatesTo( TimeUnit.SECONDS ) .convertDurationsTo( TimeUnit.MILLISECONDS ) - .filter( MetricFilter.ALL ) .formatFor( Locale.US ) + .outputStreamSupplierFactory( + getFileRotatingFileOutputStreamSupplier( rotationThreshold, maxArchives ) ) .build( ensureDirectoryExists( outputPath ) ); } @@ -103,32 +113,37 @@ public void shutdown() public void report( SortedMap gauges, SortedMap counters, SortedMap histograms, SortedMap meters, SortedMap timers ) { - /* - * The synchronized is needed here since the `report` method called below is also called by the recurring - * scheduled thread. In order to avoid races with that thread we synchronize on the same monitor - * before reporting. - */ - synchronized ( csvReporter ) - { - csvReporter.report( gauges, counters, histograms, meters, timers ); - } + csvReporter.report( gauges, counters, histograms, meters, timers ); } - private File ensureDirectoryExists( File dir ) + private BiFunction getFileRotatingFileOutputStreamSupplier( + Long rotationThreshold, Integer maxArchives ) { - if ( !dir.exists() ) + return ( File file, RotatingFileOutputStreamSupplier.RotationListener listener ) -> { - if ( !dir.mkdirs() ) + try + { + return new RotatingFileOutputStreamSupplier( fileSystem, file, rotationThreshold, 0, maxArchives, + scheduler.executor( JobScheduler.Groups.metricsLogRotations ), listener ); + } + catch ( IOException e ) { - throw new IllegalStateException( - "Could not create path for CSV files: " + dir.getAbsolutePath() ); + throw new RuntimeException( e ); } + }; + } + + private File ensureDirectoryExists( File dir ) throws IOException + { + if ( !fileSystem.fileExists( dir ) ) + { + fileSystem.mkdirs( dir ); } - if ( dir.isFile() ) + if ( !fileSystem.isDirectory( dir ) ) { throw new IllegalStateException( "The given path for CSV files points to a file, but a directory is required: " + - dir.getAbsolutePath() ); + dir.getAbsolutePath() ); } return dir; } diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/output/EventReporterBuilder.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/EventReporterBuilder.java index 5b33b0138bea2..8aa3d8d7a4c24 100644 --- a/enterprise/metrics/src/main/java/org/neo4j/metrics/output/EventReporterBuilder.java +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/EventReporterBuilder.java @@ -23,11 +23,13 @@ import org.neo4j.cluster.ClusterSettings; import org.neo4j.helpers.HostnamePort; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.logging.Log; import org.neo4j.metrics.MetricsSettings; +import org.neo4j.scheduler.JobScheduler; import static org.neo4j.metrics.MetricsSettings.csvEnabled; import static org.neo4j.metrics.MetricsSettings.graphiteEnabled; @@ -41,15 +43,19 @@ public class EventReporterBuilder private final Log logger; private final KernelContext kernelContext; private final LifeSupport life; + private FileSystemAbstraction fileSystem; + private JobScheduler scheduler; public EventReporterBuilder( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext, - LifeSupport life ) + LifeSupport life, FileSystemAbstraction fileSystem, JobScheduler scheduler ) { this.config = config; this.registry = registry; this.logger = logger; this.kernelContext = kernelContext; this.life = life; + this.fileSystem = fileSystem; + this.scheduler = scheduler; } public CompositeEventReporter build() @@ -58,7 +64,7 @@ public CompositeEventReporter build() final String prefix = createMetricsPrefix( config ); if ( config.get( csvEnabled ) ) { - CsvOutput csvOutput = new CsvOutput( config, registry, logger, kernelContext ); + CsvOutput csvOutput = new CsvOutput( config, registry, logger, kernelContext, fileSystem, scheduler ); reporter.add( csvOutput ); life.add( csvOutput ); } diff --git a/enterprise/metrics/src/main/java/org/neo4j/metrics/output/RotatableCsvReporter.java b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/RotatableCsvReporter.java new file mode 100644 index 0000000000000..63e9662081aaa --- /dev/null +++ b/enterprise/metrics/src/main/java/org/neo4j/metrics/output/RotatableCsvReporter.java @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.metrics.output; + +import com.codahale.metrics.Clock; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.ScheduledReporter; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; + +import java.io.File; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.Map; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.neo4j.io.IOUtils; +import org.neo4j.logging.RotatingFileOutputStreamSupplier; + +public class RotatableCsvReporter extends ScheduledReporter +{ + private final Locale locale; + private final Clock clock; + private final File directory; + private final Map writers; + private final BiFunction fileSupplierStreamCreator; + + RotatableCsvReporter( MetricRegistry registry, Locale locale, TimeUnit rateUnit, TimeUnit durationUnit, Clock clock, + File directory, + BiFunction fileSupplierStreamCreator ) + { + super( registry, "csv-reporter", MetricFilter.ALL, rateUnit, durationUnit ); + this.locale = locale; + this.clock = clock; + this.directory = directory; + this.fileSupplierStreamCreator = fileSupplierStreamCreator; + this.writers = new ConcurrentHashMap<>(); + } + + public static Builder forRegistry( MetricRegistry registry ) + { + return new Builder( registry ); + } + + @Override + public void stop() + { + super.stop(); + writers.values().forEach( CsvRotatableWriter::close ); + } + + @Override + public void report( SortedMap gauges, SortedMap counters, + SortedMap histograms, SortedMap meters, SortedMap timers ) + { + final long timestamp = TimeUnit.MILLISECONDS.toSeconds( clock.getTime() ); + + for ( Map.Entry entry : gauges.entrySet() ) + { + reportGauge( timestamp, entry.getKey(), entry.getValue() ); + } + + for ( Map.Entry entry : counters.entrySet() ) + { + reportCounter( timestamp, entry.getKey(), entry.getValue() ); + } + + for ( Map.Entry entry : histograms.entrySet() ) + { + reportHistogram( timestamp, entry.getKey(), entry.getValue() ); + } + + for ( Map.Entry entry : meters.entrySet() ) + { + reportMeter( timestamp, entry.getKey(), entry.getValue() ); + } + + for ( Map.Entry entry : timers.entrySet() ) + { + reportTimer( timestamp, entry.getKey(), entry.getValue() ); + } + } + + private void reportTimer( long timestamp, String name, Timer timer ) + { + final Snapshot snapshot = timer.getSnapshot(); + + report( timestamp, name, + "count,max,mean,min,stddev,p50,p75,p95,p98,p99,p999,mean_rate,m1_rate,m5_rate,m15_rate,rate_unit,duration_unit", + "%d,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,calls/%s,%s", timer.getCount(), + convertDuration( snapshot.getMax() ), convertDuration( snapshot.getMean() ), + convertDuration( snapshot.getMin() ), convertDuration( snapshot.getStdDev() ), + convertDuration( snapshot.getMedian() ), convertDuration( snapshot.get75thPercentile() ), + convertDuration( snapshot.get95thPercentile() ), convertDuration( snapshot.get98thPercentile() ), + convertDuration( snapshot.get99thPercentile() ), convertDuration( snapshot.get999thPercentile() ), + convertRate( timer.getMeanRate() ), convertRate( timer.getOneMinuteRate() ), + convertRate( timer.getFiveMinuteRate() ), convertRate( timer.getFifteenMinuteRate() ), getRateUnit(), + getDurationUnit() ); + } + + private void reportMeter( long timestamp, String name, Meter meter ) + { + report( timestamp, name, "count,mean_rate,m1_rate,m5_rate,m15_rate,rate_unit", "%d,%f,%f,%f,%f,events/%s", + meter.getCount(), convertRate( meter.getMeanRate() ), convertRate( meter.getOneMinuteRate() ), + convertRate( meter.getFiveMinuteRate() ), convertRate( meter.getFifteenMinuteRate() ), getRateUnit() ); + } + + private void reportHistogram( long timestamp, String name, Histogram histogram ) + { + final Snapshot snapshot = histogram.getSnapshot(); + + report( timestamp, name, "count,max,mean,min,stddev,p50,p75,p95,p98,p99,p999", + "%d,%d,%f,%d,%f,%f,%f,%f,%f,%f,%f", histogram.getCount(), snapshot.getMax(), snapshot.getMean(), + snapshot.getMin(), snapshot.getStdDev(), snapshot.getMedian(), snapshot.get75thPercentile(), + snapshot.get95thPercentile(), snapshot.get98thPercentile(), snapshot.get99thPercentile(), + snapshot.get999thPercentile() ); + } + + private void reportCounter( long timestamp, String name, Counter counter ) + { + report( timestamp, name, "count", "%d", counter.getCount() ); + } + + private void reportGauge( long timestamp, String name, Gauge gauge ) + { + report( timestamp, name, "value", "%s", gauge.getValue() ); + } + + private void report( long timestamp, String name, String header, String line, Object... values ) + { + File file = new File( directory, name + ".csv" ); + CsvRotatableWriter csvRotatableWriter = writers.computeIfAbsent( file, + new RotatingCsvWriterSupplier( header, fileSupplierStreamCreator, writers ) ); + //noinspection SynchronizationOnLocalVariableOrMethodParameter + csvRotatableWriter.writeValues( locale, timestamp, line, values ); + } + + public static class Builder + { + private final MetricRegistry registry; + private Locale locale; + private TimeUnit rateUnit; + private TimeUnit durationUnit; + private Clock clock; + private BiFunction + outputStreamSupplierFactory; + + private Builder( MetricRegistry registry ) + { + this.registry = registry; + this.locale = Locale.getDefault(); + this.rateUnit = TimeUnit.SECONDS; + this.durationUnit = TimeUnit.MILLISECONDS; + this.clock = Clock.defaultClock(); + } + + public Builder formatFor( Locale locale ) + { + this.locale = locale; + return this; + } + + public Builder convertRatesTo( TimeUnit rateUnit ) + { + this.rateUnit = rateUnit; + return this; + } + + public Builder convertDurationsTo( TimeUnit durationUnit ) + { + this.durationUnit = durationUnit; + return this; + } + + public Builder outputStreamSupplierFactory( + BiFunction outputStreamSupplierFactory ) + { + this.outputStreamSupplierFactory = outputStreamSupplierFactory; + return this; + } + + /** + * Builds a {@link RotatableCsvReporter} with the given properties, writing {@code .csv} files to the + * given directory. + * + * @param directory the directory in which the {@code .csv} files will be created + * @return a {@link RotatableCsvReporter} + */ + public RotatableCsvReporter build( File directory ) + { + return new RotatableCsvReporter( registry, locale, rateUnit, durationUnit, clock, directory, + outputStreamSupplierFactory ); + } + } + + private static class RotatingCsvWriterSupplier implements Function + { + private final String header; + private final BiFunction fileSupplierStreamCreator; + private final Map writers; + + RotatingCsvWriterSupplier( String header, + BiFunction fileSupplierStreamCreator, + Map writers ) + { + this.header = header; + this.fileSupplierStreamCreator = fileSupplierStreamCreator; + this.writers = writers; + } + + @Override + public CsvRotatableWriter apply( File file ) + { + RotatingFileOutputStreamSupplier outputStreamSupplier = + fileSupplierStreamCreator.apply( file, new HeaderWriterRotationListener() ); + PrintWriter printWriter = createWriter( outputStreamSupplier.get() ); + CsvRotatableWriter writer = new CsvRotatableWriter( printWriter, outputStreamSupplier ); + writeHeader( printWriter, header ); + return writer; + } + + private class HeaderWriterRotationListener extends RotatingFileOutputStreamSupplier.RotationListener + { + + @Override + public void rotationCompleted( OutputStream out ) + { + super.rotationCompleted( out ); + try ( PrintWriter writer = createWriter( out ) ) + { + writeHeader( writer, header ); + } + } + } + private static PrintWriter createWriter( OutputStream outputStream ) + { + return new PrintWriter( new OutputStreamWriter( outputStream, StandardCharsets.UTF_8 ) ); + } + + private static void writeHeader( PrintWriter printWriter, String header ) + { + printWriter.println( "t," + header ); + printWriter.flush(); + } + } + + private static class CsvRotatableWriter + { + private final PrintWriter printWriter; + private final RotatingFileOutputStreamSupplier streamSupplier; + + CsvRotatableWriter( PrintWriter printWriter, RotatingFileOutputStreamSupplier streamSupplier ) + { + this.printWriter = printWriter; + this.streamSupplier = streamSupplier; + } + + void close() + { + IOUtils.closeAllSilently( printWriter, streamSupplier ); + } + + synchronized void writeValues( Locale locale, long timestamp, String line, Object[] values ) + { + streamSupplier.get(); + printWriter.printf( locale, String.format( locale, "%d,%s%n", timestamp, line ), values ); + printWriter.flush(); + } + } +} diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/BoltMetricsIT.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/BoltMetricsIT.java index d072b2a784ee2..ecdebe5d8a9ae 100644 --- a/enterprise/metrics/src/test/java/org/neo4j/metrics/BoltMetricsIT.java +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/BoltMetricsIT.java @@ -19,12 +19,11 @@ */ package org.neo4j.metrics; -import java.io.File; - import org.junit.After; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; + +import java.io.File; import org.neo4j.bolt.v1.messaging.message.InitMessage; import org.neo4j.bolt.v1.transport.socket.client.SocketConnection; @@ -37,6 +36,7 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.ports.allocation.PortAuthority; import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.Matchers.equalTo; @@ -57,7 +57,7 @@ public class BoltMetricsIT { @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); + public TestDirectory testDirectory = TestDirectory.testDirectory(); private GraphDatabaseAPI db; private TransportConnection conn; @@ -68,13 +68,14 @@ public void shouldMonitorBolt() throws Throwable int port = PortAuthority.allocatePort(); // Given - File metricsFolder = tmpDir.newFolder( "metrics" ); + File metricsFolder = testDirectory.directory( "metrics" ); db = (GraphDatabaseAPI) new TestGraphDatabaseFactory() - .newImpermanentDatabaseBuilder() + .newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ) .setConfig( new BoltConnector( "bolt" ).type, "BOLT" ) .setConfig( new BoltConnector( "bolt" ).enabled, "true" ) .setConfig( new BoltConnector( "bolt" ).listen_address, "localhost:" + port ) .setConfig( GraphDatabaseSettings.auth_enabled, "false" ) + .setConfig( MetricsSettings.metricsEnabled, "false" ) .setConfig( MetricsSettings.boltMessagesEnabled, "true" ) .setConfig( MetricsSettings.csvEnabled, "true" ) .setConfig( MetricsSettings.csvInterval, "100ms" ) diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/MetricsTestHelper.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/MetricsTestHelper.java index c788bf215fc2d..f09c2daefd88e 100644 --- a/enterprise/metrics/src/test/java/org/neo4j/metrics/MetricsTestHelper.java +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/MetricsTestHelper.java @@ -46,7 +46,7 @@ public static long readLongValue( File metricFile ) throws IOException, Interrup return readLongValueAndAssert( metricFile, ( one, two ) -> true ); } - static long readLongValueAndAssert( File metricFile, BiPredicate assumption ) + public static long readLongValueAndAssert( File metricFile, BiPredicate assumption ) throws IOException, InterruptedException { return readValueAndAssert( metricFile, 0L, Long::parseLong, assumption ); diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/PageCacheMetricsIT.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/PageCacheMetricsIT.java index 349e022b80db0..2b99bb2fc0aa0 100644 --- a/enterprise/metrics/src/test/java/org/neo4j/metrics/PageCacheMetricsIT.java +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/PageCacheMetricsIT.java @@ -69,6 +69,7 @@ public void setUp() { metricsDirectory = testDirectory.directory( "metrics" ); database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( MetricsSettings.metricsEnabled, Settings.FALSE ) .setConfig( MetricsSettings.neoPageCacheEnabled, Settings.TRUE ) .setConfig( MetricsSettings.csvEnabled, Settings.TRUE ) .setConfig( MetricsSettings.csvInterval, "100ms" ) diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/output/CsvOutputTest.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/CsvOutputTest.java index 284b75c9a528e..a1f00db1825d6 100644 --- a/enterprise/metrics/src/test/java/org/neo4j/metrics/output/CsvOutputTest.java +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/CsvOutputTest.java @@ -37,6 +37,8 @@ import org.neo4j.kernel.lifecycle.LifeRule; import org.neo4j.logging.NullLog; import org.neo4j.metrics.MetricsSettings; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.test.OnDemandJobScheduler; import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; @@ -50,6 +52,7 @@ public class CsvOutputTest private final LifeRule life = new LifeRule(); private final TestDirectory directory = TestDirectory.testDirectory(); private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule(); + private final JobScheduler jobScheduler = new OnDemandJobScheduler(); @Rule public RuleChain ruleChain = RuleChain.outerRule( directory ).around( fileSystemRule ).around( life ); @@ -73,7 +76,7 @@ public void shouldHaveRelativeMetricsCsvPathBeRelativeToNeo4jHome() throws Excep MetricsSettings.csvInterval.name(), "10ms", MetricsSettings.csvPath.name(), "the-metrics-dir", GraphDatabaseSettings.neo4j_home.name(), home.getAbsolutePath() ); - life.add( new CsvOutput( config, new MetricRegistry(), NullLog.getInstance(), kernelContext ) ); + life.add( createCsvOutput( config ) ); // WHEN life.start(); @@ -91,7 +94,7 @@ public void shouldHaveAbsoluteMetricsCsvPathBeAbsolute() throws Exception MetricsSettings.csvEnabled.name(), "true", MetricsSettings.csvInterval.name(), "10ms", MetricsSettings.csvPath.name(), outputFPath.getAbsolutePath() ); - life.add( new CsvOutput( config, new MetricRegistry(), NullLog.getInstance(), kernelContext ) ); + life.add( createCsvOutput( config ) ); // WHEN life.start(); @@ -100,6 +103,11 @@ public void shouldHaveAbsoluteMetricsCsvPathBeAbsolute() throws Exception waitForFileToAppear( outputFPath ); } + private CsvOutput createCsvOutput( Config config ) + { + return new CsvOutput( config, new MetricRegistry(), NullLog.getInstance(), kernelContext, fileSystemRule, jobScheduler ); + } + private Config config( String... keysValues ) { return Config.defaults( stringMap( keysValues ) ); diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvOutputIT.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvOutputIT.java new file mode 100644 index 0000000000000..f7dc8fee54a8f --- /dev/null +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvOutputIT.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.metrics.output; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.EnterpriseGraphDatabaseFactory; +import org.neo4j.kernel.configuration.Settings; +import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; +import org.neo4j.metrics.source.db.TransactionMetrics; +import org.neo4j.test.rule.TestDirectory; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.neo4j.metrics.MetricsSettings.csvPath; +import static org.neo4j.metrics.MetricsSettings.csvRotationThreshold; +import static org.neo4j.metrics.MetricsTestHelper.readLongValueAndAssert; +import static org.neo4j.test.assertion.Assert.assertEventually; + +public class RotatableCsvOutputIT +{ + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + + private File outputPath; + private GraphDatabaseService database; + + @Before + public void setup() throws Throwable + { + outputPath = testDirectory.directory( "metrics" ); + database = new EnterpriseGraphDatabaseFactory().newEmbeddedDatabaseBuilder( testDirectory.graphDbDir() ) + .setConfig( csvPath, outputPath.getAbsolutePath() ) + .setConfig( csvRotationThreshold, "21" ) + .setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) + .newGraphDatabase(); + } + + @After + public void tearDown() throws Exception + { + database.shutdown(); + } + + @Test + public void rotateMetricsFile() throws InterruptedException, IOException + { + try ( Transaction transaction = database.beginTx() ) + { + database.createNode(); + transaction.success(); + } + File metricsFile = metricsCsv( outputPath, TransactionMetrics.TX_COMMITTED ); + long committedTransactions = readLongValueAndAssert( metricsFile, + ( newValue, currentValue ) -> newValue >= currentValue ); + assertEquals( 1, committedTransactions ); + + try ( Transaction transaction = database.beginTx() ) + { + database.createNode(); + transaction.success(); + } + File metricsFile2 = metricsCsv( outputPath, TransactionMetrics.TX_COMMITTED, 1 ); + long oldCommittedTransactions = readLongValueAndAssert( metricsFile2, + ( newValue, currentValue ) -> newValue >= currentValue ); + assertEquals( 1, oldCommittedTransactions ); + } + + private static File metricsCsv( File dbDir, String metric ) throws InterruptedException + { + return metricsCsv( dbDir, metric, 0 ); + } + + private static File metricsCsv( File dbDir, String metric, long index ) throws InterruptedException + { + File csvFile = new File( dbDir, index > 0 ? metric + ".csv." + index : metric + ".csv" ); + assertEventually( "Metrics file should exist", csvFile::exists, is( true ), 40, SECONDS ); + return csvFile; + } +} diff --git a/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvReporterTest.java b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvReporterTest.java new file mode 100644 index 0000000000000..8d9b9d3c945f2 --- /dev/null +++ b/enterprise/metrics/src/test/java/org/neo4j/metrics/output/RotatableCsvReporterTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.metrics.output; + +import com.codahale.metrics.Clock; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Locale; +import java.util.TreeMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +import org.neo4j.logging.RotatingFileOutputStreamSupplier; +import org.neo4j.test.rule.TestDirectory; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RotatableCsvReporterTest +{ + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + private RotatingFileOutputStreamSupplier fileOutputStreamSupplier = mock( RotatingFileOutputStreamSupplier.class ); + + @Test + public void stopAllWritersOnStop() throws IOException + { + when( fileOutputStreamSupplier.get() ).thenReturn( mock( OutputStream.class ) ); + RotatableCsvReporter reporter = + new RotatableCsvReporter( mock( MetricRegistry.class ), Locale.US, TimeUnit.SECONDS, TimeUnit.SECONDS, + Clock.defaultClock(), testDirectory.directory(), + ( file, rotationListener ) -> fileOutputStreamSupplier ); + TreeMap gauges = new TreeMap<>(); + gauges.put( "a", () -> ThreadLocalRandom.current().nextLong() ); + gauges.put( "b", () -> ThreadLocalRandom.current().nextLong() ); + gauges.put( "c", () -> ThreadLocalRandom.current().nextLong() ); + reporter.report( gauges, new TreeMap<>(), new TreeMap<>(), new TreeMap<>(), new TreeMap<>() ); + + gauges.put( "b", () -> ThreadLocalRandom.current().nextLong() ); + gauges.put( "c", () -> ThreadLocalRandom.current().nextLong() ); + gauges.put( "d", () -> ThreadLocalRandom.current().nextLong() ); + reporter.report( gauges, new TreeMap<>(), new TreeMap<>(), new TreeMap<>(), new TreeMap<>() ); + + reporter.stop(); + verify( fileOutputStreamSupplier, times( 4 ) ).close(); + } +}