S4-130 Update and document parameters initialization from nodes

- applicable to csv metrics logging configuration
- also: license update for some xml files

subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java

@@ -19,16 +19,20 @@
 
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.s4.core.util.ArchiveFetchException;
 import org.apache.s4.core.util.ParametersInjectionModule;
+import org.apache.s4.core.util.ParsingUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.common.io.Resources;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -68,10 +72,15 @@ public void uncaughtException(Thread t, Throwable e) {
             }
         });
 
+        // inject parameter from the command line, including zk string
+        Map<String, String> inlineParameters = Maps.newHashMap(ParsingUtils
+                .convertListArgsToMap(nodeArgs.extraNamedParameters));
+        inlineParameters.put("s4.cluster.zk_address", nodeArgs.zkConnectionString);
+
         Injector injector = Guice.createInjector(Modules.override(
                 new BaseModule(Resources.getResource("default.s4.base.properties").openStream(), nodeArgs.clusterName))
-                .with(new ParametersInjectionModule(ImmutableMap.of("s4.cluster.zk_address",
-                        nodeArgs.zkConnectionString))));
+                .with(new ParametersInjectionModule(inlineParameters)));
+
         S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
         try {
             bootstrap.start(injector);
@@ -96,5 +105,11 @@ public static class S4NodeArgs {
         @Parameter(names = "-zk", description = "Zookeeper connection string", required = false)
         String zkConnectionString = "localhost:2181";
 
+        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of "
+                + "inline configuration parameters, taking precedence over homonymous configuration parameters from "
+                + "configuration files. Syntax: '-p=name1=value1,name2=value2 '. "
+                + "NOTE: application parameters should be injected in the application configuration/deployment step."
+                + "Only parameters relevant to the node should be injected here, e.g. metrics logging configuration", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
+        List<String> extraNamedParameters = new ArrayList<String>();
     }
 }

subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s4.core.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * Utilities for parsing command line parameters
+ */
+public class ParsingUtils {
+
+    public static Map<String, String> convertListArgsToMap(List<String> args) {
+        Map<String, String> result = Maps.newHashMap();
+        for (String arg : args) {
+            String[] split = arg.split("[=]");
+            if (!(split.length == 2)) {
+                throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
+            }
+            result.put(split[0], split[1]);
+        }
+        return result;
+    }
+
+    public static class InlineConfigParameterConverter implements IStringConverter<String> {
+
+        private static Logger logger = LoggerFactory.getLogger(InlineConfigParameterConverter.class);
+
+        @Override
+        public String convert(String arg) {
+            Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
+            logger.info("processing inline configuration parameter {}", arg);
+            Matcher parameterMatcher = parameterPattern.matcher(arg);
+            if (!parameterMatcher.find()) {
+                throw new IllegalArgumentException("Cannot understand parameter " + arg);
+            }
+            return parameterMatcher.group(1);
+        }
+    }
+
+}

subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License.-->
 <diezelImplementation xmlns="http://diezel.ericaro.net/2.0.0/">
 	<package>org.apache.s4.edsl</package>
 	<name>Builder</name>

subprojects/s4-edsl/src/main/diezel/s4/s4.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License. -->
 <diezel xmlns="http://diezel.ericaro.net/2.0.0/">
 	<package>org.apache.s4.edsl</package>
 	<name>S4DSL</name>

subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java

@@ -23,24 +23,20 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.I0Itec.zkclient.ZkClient;
 import org.apache.s4.comm.topology.ZNRecordSerializer;
 import org.apache.s4.core.util.AppConfig;
+import org.apache.s4.core.util.ParsingUtils;
 import org.apache.s4.deploy.DeploymentUtils;
 import org.gradle.tooling.BuildLauncher;
 import org.gradle.tooling.GradleConnector;
 import org.gradle.tooling.ProgressListener;
 import org.gradle.tooling.ProjectConnection;
 import org.slf4j.LoggerFactory;
 
-import com.beust.jcommander.IStringConverter;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Maps;
 import com.google.common.base.Strings;
 
 /**
@@ -86,8 +82,8 @@ public static void main(String[] args) {
                             .appURI(s4rURI == null ? null : s4rURI.toString())
                             .customModulesNames(deployArgs.modulesClassesNames)
                             .customModulesURIs(deployArgs.modulesURIs).appClassName(deployArgs.appClass)
-                            .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
-                    deployArgs.clusterName, false, deployArgs.zkConnectionString);
+                            .namedParameters(ParsingUtils.convertListArgsToMap(deployArgs.extraNamedParameters))
+                            .build(), deployArgs.clusterName, false, deployArgs.zkConnectionString);
             // Explicitly shutdown the JVM since Gradle leaves non-daemon threads running that delay the termination
             if (!deployArgs.testMode) {
                 System.exit(0);
@@ -98,18 +94,6 @@ public static void main(String[] args) {
 
     }
 
-    private static Map<String, String> convertListArgsToMap(List<String> args) {
-        Map<String, String> result = Maps.newHashMap();
-        for (String arg : args) {
-            String[] split = arg.split("[=]");
-            if (!(split.length == 2)) {
-                throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
-            }
-            result.put(split[0], split[1]);
-        }
-        return result;
-    }
-
     @Parameters(commandNames = "s4 deploy", commandDescription = "Package and deploy application to S4 cluster", separators = "=")
     static class DeployAppArgs extends S4ArgsBase {
 
@@ -137,7 +121,7 @@ static class DeployAppArgs extends S4ArgsBase {
         @Parameter(names = { "-modulesClasses", "-emc", "-mc" }, description = "Fully qualified class names of custom modules")
         List<String> modulesClassesNames = new ArrayList<String>();
 
-        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = InlineConfigParameterConverter.class)
+        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
         List<String> extraNamedParameters = new ArrayList<String>();
 
         @Parameter(names = "-testMode", description = "Special mode for regression testing", hidden = true)
@@ -147,24 +131,6 @@ static class DeployAppArgs extends S4ArgsBase {
         boolean debug = false;
     }
 
-    /**
-     * Parameters parsing utility.
-     * 
-     */
-    public static class InlineConfigParameterConverter implements IStringConverter<String> {
-
-        @Override
-        public String convert(String arg) {
-            Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
-            logger.info("processing inline configuration parameter {}", arg);
-            Matcher parameterMatcher = parameterPattern.matcher(arg);
-            if (!parameterMatcher.find()) {
-                throw new IllegalArgumentException("Cannot understand parameter " + arg);
-            }
-            return parameterMatcher.group(1);
-        }
-    }
-
     static class ExecGradle {
 
         public static void exec(File buildFile, String taskName, String[] params, boolean debug) throws Exception {

website/content/doc/0.6.0/metrics.md

@@ -24,19 +24,22 @@ You can also monitor your own PEs. Simply add new probes (`Meter`, `Gauge`, etc.
 
 By default, metrics are exposed by each node through JMX.
 
-The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as an application parameter, and must match the following regular expression: 
+The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as a node or application parameter [^1], and must match the following regular expression: 
 
 	(csv:.+|console):(\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)
 
 Examples:
 
 	# dump metrics to csv files to /path/to/directory every 10 seconds
-	# (recommendation: use a clean directory)
+	# (recommendation: use a different and clean directory for each of the nodes)
 	csv:/path/to/directory:10:SECONDS
 
 	# dump metrics to the console every minute
 	console:1:MINUTES
 
 
 
-Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
+Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
+
+
+[^1]: csv-based metrics logging are usually specified as a node configuration, and must point to a clean directory, different for each node. Otherwise the metrics logging system will try to create multiple new files with the same name, and depending on your setup, that may not be possible.