Enable metrics by default. Make csv files rotatable.

Previously metrics were disabled by default, this PR changes that
and enable them by default.
Also previously report csv files were not limited in size in any way
and that cause problems since over time they become quite big.
This PR will allow configure csv file rotation in a way how rotation
is configurable already for any kind of log file.

community/common/src/main/java/org/neo4j/scheduler/JobScheduler.java

@@ -124,6 +124,11 @@ class Groups
          */
         public static final Group boltLogRotation = new Group( "BoltLogRotation" );
 
+        /**
+         * Rotates metrics csv files
+         */
+        public static final Group metricsLogRotations = new Group( "MetricsLogRotations" );
+
         /**
          * Checkpoint and store flush
          */

community/logging/src/main/java/org/neo4j/logging/RotatingFileOutputStreamSupplier.java

@@ -308,7 +308,6 @@ void rotate()
             finally
             {
                 rotating.set( false );
-                logFileLock.writeLock().unlock();
                 try
                 {
                     bufferingOutputStream.writeTo( streamWrapper );
@@ -317,6 +316,7 @@ void rotate()
                 {
                     rotationListener.rotationError( e, streamWrapper );
                 }
+                logFileLock.writeLock().unlock();
             }
         };
 

enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsExtension.java

@@ -21,6 +21,7 @@
 
 import com.codahale.metrics.MetricRegistry;
 
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.spi.KernelContext;
@@ -30,6 +31,7 @@
 import org.neo4j.metrics.output.CompositeEventReporter;
 import org.neo4j.metrics.output.EventReporterBuilder;
 import org.neo4j.metrics.source.Neo4jMetricsBuilder;
+import org.neo4j.scheduler.JobScheduler;
 
 public class MetricsExtension implements Lifecycle
 {
@@ -42,10 +44,13 @@ public class MetricsExtension implements Lifecycle
     {
         LogService logService = dependencies.logService();
         Config configuration = dependencies.configuration();
+        FileSystemAbstraction fileSystem = dependencies.fileSystemAbstraction();
+        JobScheduler scheduler = dependencies.scheduler();
         logger = logService.getUserLog( getClass() );
 
         MetricRegistry registry = new MetricRegistry();
-        reporter = new EventReporterBuilder( configuration, registry, logger, kernelContext, life ).build();
+        reporter = new EventReporterBuilder( configuration, registry, logger, kernelContext, life, fileSystem,
+                scheduler ).build();
         metricsBuilt = new Neo4jMetricsBuilder( registry, reporter, configuration, logService, kernelContext,
                                                 dependencies, life ).build();
     }

enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsKernelExtensionFactory.java

@@ -19,12 +19,14 @@
  */
 package org.neo4j.metrics;
 
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.extension.KernelExtensionFactory;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.spi.KernelContext;
 import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.metrics.source.Neo4jMetricsBuilder;
+import org.neo4j.scheduler.JobScheduler;
 
 public class MetricsKernelExtensionFactory extends KernelExtensionFactory<MetricsKernelExtensionFactory.Dependencies>
 {
@@ -33,6 +35,10 @@ public interface Dependencies extends Neo4jMetricsBuilder.Dependencies
         Config configuration();
 
         LogService logService();
+
+        FileSystemAbstraction fileSystemAbstraction();
+
+        JobScheduler scheduler();
     }
 
     public MetricsKernelExtensionFactory()

enterprise/metrics/src/main/java/org/neo4j/metrics/MetricsSettings.java

@@ -28,12 +28,17 @@
 import org.neo4j.helpers.HostnamePort;
 
 import static org.neo4j.kernel.configuration.Settings.BOOLEAN;
+import static org.neo4j.kernel.configuration.Settings.BYTES;
 import static org.neo4j.kernel.configuration.Settings.DURATION;
 import static org.neo4j.kernel.configuration.Settings.FALSE;
 import static org.neo4j.kernel.configuration.Settings.HOSTNAME_PORT;
+import static org.neo4j.kernel.configuration.Settings.INTEGER;
 import static org.neo4j.kernel.configuration.Settings.STRING;
+import static org.neo4j.kernel.configuration.Settings.TRUE;
 import static org.neo4j.kernel.configuration.Settings.buildSetting;
+import static org.neo4j.kernel.configuration.Settings.min;
 import static org.neo4j.kernel.configuration.Settings.pathSetting;
+import static org.neo4j.kernel.configuration.Settings.range;
 import static org.neo4j.kernel.configuration.Settings.setting;
 
 /**
@@ -52,7 +57,7 @@ public class MetricsSettings implements LoadableConfig
     @Description( "The default enablement value for all the supported metrics. Set this to `false` to turn off all " +
                   "metrics by default. The individual settings can then be used to selectively re-enable specific " +
                   "metrics." )
-    public static Setting<Boolean> metricsEnabled = setting( "metrics.enabled", BOOLEAN, FALSE );
+    public static Setting<Boolean> metricsEnabled = setting( "metrics.enabled", BOOLEAN, TRUE );
 
     @Description( "The default enablement value for all Neo4j specific support metrics. Set this to `false` to turn " +
                   "off all Neo4j specific metrics by default. The individual `metrics.neo4j.*` metrics can then be " +
@@ -120,7 +125,7 @@ public class MetricsSettings implements LoadableConfig
 
     // CSV settings
     @Description( "Set to `true` to enable exporting metrics to CSV files" )
-    public static Setting<Boolean> csvEnabled = setting( "metrics.csv.enabled", BOOLEAN, FALSE );
+    public static Setting<Boolean> csvEnabled = setting( "metrics.csv.enabled", BOOLEAN, TRUE );
 
     @Description( "The target location of the CSV files: a path to a directory wherein a CSV file per reported " +
                   "field  will be written." )
@@ -130,6 +135,15 @@ public class MetricsSettings implements LoadableConfig
                   "the CSV files." )
     public static Setting<Duration> csvInterval = setting( "metrics.csv.interval", DURATION, "3s" );
 
+    @Description( "The file size in bytes at which the csv files will auto-rotate. If set to zero then no " +
+            "rotation will occur. Accepts a binary suffix `k`, `m` or `g`." )
+    public static final Setting<Long> csvRotationThreshold = buildSetting( "metrics.csv.rotation.size",
+            BYTES, "20m" ).constraint( range( 0L, Long.MAX_VALUE ) ).build();
+
+    @Description( "Maximum number of history files for the csv files." )
+    public static final Setting<Integer> csvMaxArchives = buildSetting( "metrics.csv.rotation.keep_number",
+            INTEGER, "7" ).constraint( min( 1 ) ).build();
+
     // Graphite settings
     @Description( "Set to `true` to enable exporting metrics to Graphite." )
     public static Setting<Boolean> graphiteEnabled = setting( "metrics.graphite.enabled", BOOLEAN, FALSE );

enterprise/metrics/src/main/java/org/neo4j/metrics/output/CsvOutput.java

@@ -20,25 +20,27 @@
 package org.neo4j.metrics.output;
 
 import com.codahale.metrics.Counter;
-import com.codahale.metrics.CsvReporter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricFilter;
 import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
 import com.codahale.metrics.Timer;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Locale;
 import java.util.SortedMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
 
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.spi.KernelContext;
 import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.logging.Log;
+import org.neo4j.logging.RotatingFileOutputStreamSupplier;
+import org.neo4j.metrics.MetricsSettings;
+import org.neo4j.scheduler.JobScheduler;
 
 import static org.neo4j.metrics.MetricsSettings.csvEnabled;
 import static org.neo4j.metrics.MetricsSettings.csvInterval;
@@ -50,19 +52,24 @@ public class CsvOutput implements Lifecycle, EventReporter
     private final MetricRegistry registry;
     private final Log logger;
     private final KernelContext kernelContext;
-    private ScheduledReporter csvReporter;
+    private final FileSystemAbstraction fileSystem;
+    private final JobScheduler scheduler;
+    private RotatableCsvReporter csvReporter;
     private File outputPath;
 
-    public CsvOutput( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext )
+    CsvOutput( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext,
+            FileSystemAbstraction fileSystem, JobScheduler scheduler )
     {
         this.config = config;
         this.registry = registry;
         this.logger = logger;
         this.kernelContext = kernelContext;
+        this.fileSystem = fileSystem;
+        this.scheduler = scheduler;
     }
 
     @Override
-    public void init()
+    public void init() throws IOException
     {
         // Setup CSV reporting
         File configuredPath = config.get( csvPath );
@@ -71,12 +78,15 @@ public void init()
             throw new IllegalArgumentException( csvPath.name() + " configuration is required since " +
                                                 csvEnabled.name() + " is enabled" );
         }
+        Long rotationThreshold = config.get( MetricsSettings.csvRotationThreshold );
+        Integer maxArchives = config.get( MetricsSettings.csvMaxArchives );
         outputPath = absoluteFileOrRelativeTo( kernelContext.storeDir(), configuredPath );
-        csvReporter = CsvReporter.forRegistry( registry )
+        csvReporter = RotatableCsvReporter.forRegistry( registry )
                 .convertRatesTo( TimeUnit.SECONDS )
                 .convertDurationsTo( TimeUnit.MILLISECONDS )
-                .filter( MetricFilter.ALL )
                 .formatFor( Locale.US )
+                .outputStreamSupplierFactory(
+                        getFileRotatingFileOutputStreamSupplier( rotationThreshold, maxArchives ) )
                 .build( ensureDirectoryExists( outputPath ) );
     }
 
@@ -103,32 +113,37 @@ public void shutdown()
     public void report( SortedMap<String,Gauge> gauges, SortedMap<String,Counter> counters,
             SortedMap<String,Histogram> histograms, SortedMap<String,Meter> meters, SortedMap<String,Timer> timers )
     {
-        /*
-         * The synchronized is needed here since the `report` method called below is also called by the recurring
-         * scheduled thread.  In order to avoid races with that thread we synchronize on the same monitor
-         * before reporting.
-         */
-        synchronized ( csvReporter )
-        {
-            csvReporter.report( gauges, counters, histograms, meters, timers );
-        }
+        csvReporter.report( gauges, counters, histograms, meters, timers );
     }
 
-    private File ensureDirectoryExists( File dir )
+    private BiFunction<File,RotatingFileOutputStreamSupplier.RotationListener,RotatingFileOutputStreamSupplier> getFileRotatingFileOutputStreamSupplier(
+            Long rotationThreshold, Integer maxArchives )
     {
-        if ( !dir.exists() )
+        return ( File file, RotatingFileOutputStreamSupplier.RotationListener listener ) ->
         {
-            if ( !dir.mkdirs() )
+            try
+            {
+                return new RotatingFileOutputStreamSupplier( fileSystem, file, rotationThreshold, 0, maxArchives,
+                        scheduler.executor( JobScheduler.Groups.metricsLogRotations ), listener );
+            }
+            catch ( IOException e )
             {
-                throw new IllegalStateException(
-                        "Could not create path for CSV files: " + dir.getAbsolutePath() );
+                throw new RuntimeException( e );
             }
+        };
+    }
+
+    private File ensureDirectoryExists( File dir ) throws IOException
+    {
+        if ( !fileSystem.fileExists( dir ) )
+        {
+            fileSystem.mkdirs( dir );
         }
-        if ( dir.isFile() )
+        if ( !fileSystem.isDirectory( dir ) )
         {
             throw new IllegalStateException(
                     "The given path for CSV files points to a file, but a directory is required: " +
-                    dir.getAbsolutePath() );
+                            dir.getAbsolutePath() );
         }
         return dir;
     }

enterprise/metrics/src/main/java/org/neo4j/metrics/output/EventReporterBuilder.java

@@ -23,11 +23,13 @@
 
 import org.neo4j.cluster.ClusterSettings;
 import org.neo4j.helpers.HostnamePort;
+import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.spi.KernelContext;
 import org.neo4j.kernel.lifecycle.LifeSupport;
 import org.neo4j.logging.Log;
 import org.neo4j.metrics.MetricsSettings;
+import org.neo4j.scheduler.JobScheduler;
 
 import static org.neo4j.metrics.MetricsSettings.csvEnabled;
 import static org.neo4j.metrics.MetricsSettings.graphiteEnabled;
@@ -41,15 +43,19 @@ public class EventReporterBuilder
     private final Log logger;
     private final KernelContext kernelContext;
     private final LifeSupport life;
+    private FileSystemAbstraction fileSystem;
+    private JobScheduler scheduler;
 
     public EventReporterBuilder( Config config, MetricRegistry registry, Log logger, KernelContext kernelContext,
-            LifeSupport life )
+            LifeSupport life, FileSystemAbstraction fileSystem, JobScheduler scheduler )
     {
         this.config = config;
         this.registry = registry;
         this.logger = logger;
         this.kernelContext = kernelContext;
         this.life = life;
+        this.fileSystem = fileSystem;
+        this.scheduler = scheduler;
     }
 
     public CompositeEventReporter build()
@@ -58,7 +64,7 @@ public CompositeEventReporter build()
         final String prefix = createMetricsPrefix( config );
         if ( config.get( csvEnabled ) )
         {
-            CsvOutput csvOutput = new CsvOutput( config, registry, logger, kernelContext );
+            CsvOutput csvOutput = new CsvOutput( config, registry, logger, kernelContext, fileSystem, scheduler );
             reporter.add( csvOutput );
             life.add( csvOutput );
         }