Refine member metrics configuration

This change refines the member-side metrics configuration to address two
issues with the previously added config:
- It was not clear that `retention-seconds` was for MC only
- Further improving the MC and JMX configs was not supported by the
attributes

To address these, the configuration changes as follows.

Before this change:
```
<metrics enabled="false" mc-enabled="false" jmx-enabled="false">
    <collection-interval-seconds>10</collection-interval-seconds>
    <retention-seconds>30</retention-seconds>
    <metrics-for-data-structures>true</metrics-for-data-structures>
    <minimum-level>DEBUG</minimum-level>
</metrics>
```

After this change:
```
<metrics enabled="false">
    <management-center enabled="false">
        <retention-seconds>30</retention-seconds>
    </management-center>
    <jmx enabled="false"/>
    <collection-frequency-seconds>10</collection-frequency-seconds>
    <data-structure-metrics-enabled>true</data-structure-metrics-enabled>
    <level>DEBUG</level>
</metrics>
```

The programmatic and YAML configuration follows the same structure.

It is also a requirement that all metrics configurations should
overrideable via system properties. This is also implemented in this
change by overriding the configuration from the properties during the
`Node` instances is created. The concrete system properties are added to
the javadocs of the metrics config classes and to the xsd too.

Besides refining the changes above, the following gaps were also closed:
- Extends javadoc and xsd documentation for the metrics config elements
- Add Spring XML configuration
- Add full-example config comments
- Add metrics config to hazelcast-fullconfig.[xml|yaml]
- Add metrics metrics config to hazelcast-default.[xml|yaml]

hazelcast-spring/src/main/java/com/hazelcast/spring/HazelcastConfigBeanDefinitionParser.java

@@ -19,6 +19,7 @@
 import com.hazelcast.config.AdvancedNetworkConfig;
 import com.hazelcast.config.AliasedDiscoveryConfig;
 import com.hazelcast.config.AliasedDiscoveryConfigUtils;
+import com.hazelcast.config.AttributeConfig;
 import com.hazelcast.config.CRDTReplicationConfig;
 import com.hazelcast.config.CachePartitionLostListenerConfig;
 import com.hazelcast.config.CacheSimpleConfig;
@@ -52,7 +53,6 @@
 import com.hazelcast.config.LoginModuleConfig;
 import com.hazelcast.config.MCMutualAuthConfig;
 import com.hazelcast.config.ManagementCenterConfig;
-import com.hazelcast.config.AttributeConfig;
 import com.hazelcast.config.MapConfig;
 import com.hazelcast.config.MapPartitionLostListenerConfig;
 import com.hazelcast.config.MapStoreConfig;
@@ -65,6 +65,9 @@
 import com.hazelcast.config.MemcacheProtocolConfig;
 import com.hazelcast.config.MergePolicyConfig;
 import com.hazelcast.config.MerkleTreeConfig;
+import com.hazelcast.config.MetricsConfig;
+import com.hazelcast.config.MetricsJmxConfig;
+import com.hazelcast.config.MetricsManagementCenterConfig;
 import com.hazelcast.config.MultiMapConfig;
 import com.hazelcast.config.MulticastConfig;
 import com.hazelcast.config.NativeMemoryConfig;
@@ -110,6 +113,9 @@
 import com.hazelcast.config.WanReplicationConfig;
 import com.hazelcast.config.WanReplicationRef;
 import com.hazelcast.config.WanSyncConfig;
+import com.hazelcast.config.cp.CPSubsystemConfig;
+import com.hazelcast.config.cp.FencedLockConfig;
+import com.hazelcast.config.cp.RaftAlgorithmConfig;
 import com.hazelcast.config.cp.SemaphoreConfig;
 import com.hazelcast.config.security.JaasAuthenticationConfig;
 import com.hazelcast.config.security.LdapAuthenticationConfig;
@@ -120,18 +126,15 @@
 import com.hazelcast.config.security.TokenEncoding;
 import com.hazelcast.config.security.TokenIdentityConfig;
 import com.hazelcast.config.security.UsernamePasswordIdentityConfig;
-import com.hazelcast.config.cp.CPSubsystemConfig;
-import com.hazelcast.config.cp.FencedLockConfig;
-import com.hazelcast.config.cp.RaftAlgorithmConfig;
 import com.hazelcast.instance.EndpointQualifier;
 import com.hazelcast.instance.ProtocolType;
+import com.hazelcast.internal.nio.ClassLoaderUtil;
 import com.hazelcast.internal.services.ServiceConfigurationParser;
+import com.hazelcast.internal.util.ExceptionUtil;
 import com.hazelcast.map.eviction.MapEvictionPolicy;
 import com.hazelcast.memory.MemorySize;
 import com.hazelcast.memory.MemoryUnit;
-import com.hazelcast.internal.nio.ClassLoaderUtil;
 import com.hazelcast.splitbrainprotection.SplitBrainProtectionOn;
-import com.hazelcast.internal.util.ExceptionUtil;
 import org.springframework.beans.factory.config.BeanDefinition;
 import org.springframework.beans.factory.support.AbstractBeanDefinition;
 import org.springframework.beans.factory.support.BeanDefinitionBuilder;
@@ -338,6 +341,8 @@ public void handleConfig(Element element) {
                         handlePNCounter(node);
                     } else if ("cp-subsystem".equals(nodeName)) {
                         handleCPSubSystem(node);
+                    } else if ("metrics".equals(nodeName)) {
+                        handleMetrics(node);
                     }
                 }
             }
@@ -2159,5 +2164,35 @@ private void handleMemcacheProtocol(Node node, BeanDefinitionBuilder networkConf
         private EndpointQualifier createEndpointQualifier(ProtocolType type, Node node) {
             return EndpointQualifier.resolve(type, getAttribute(node, "name"));
         }
+
+        private void handleMetrics(Node node) {
+            BeanDefinitionBuilder metricsConfigBuilder = createBeanBuilder(MetricsConfig.class);
+
+            fillValues(node, metricsConfigBuilder, "managementCenter", "jmx");
+
+            NamedNodeMap attributes = node.getAttributes();
+            Node attrEnabled = attributes.getNamedItem("enabled");
+            boolean enabled = attrEnabled != null && getBooleanValue(getTextContent(attrEnabled));
+            metricsConfigBuilder.addPropertyValue("enabled", enabled);
+
+            for (Node child : childElements(node)) {
+                String nodeName = cleanNodeName(child);
+                if ("management-center".equals(nodeName)) {
+                    BeanDefinitionBuilder metricsMcConfigBuilder = createBeanBuilder(MetricsManagementCenterConfig.class);
+                    fillValues(child, metricsMcConfigBuilder);
+
+                    metricsConfigBuilder.addPropertyValue("managementCenterConfig",
+                            metricsMcConfigBuilder.getBeanDefinition());
+                } else if ("jmx".equals(nodeName)) {
+                    BeanDefinitionBuilder metricsJmxConfigBuilder = createBeanBuilder(MetricsJmxConfig.class);
+                    fillValues(child, metricsJmxConfigBuilder);
+
+                    metricsConfigBuilder.addPropertyValue("jmxConfig",
+                            metricsJmxConfigBuilder.getBeanDefinition());
+                }
+            }
+
+            configBuilder.addPropertyValue("metricsConfig", metricsConfigBuilder.getBeanDefinition());
+        }
     }
 }

hazelcast-spring/src/main/resources/hazelcast-spring-4.0.xsd

@@ -1310,6 +1310,7 @@
                             </xs:complexType>
                         </xs:element>
                         <xs:element name="cp-subsystem" type="cp-subsystem" minOccurs="0" maxOccurs="1"/>
+                        <xs:element name="metrics" type="metrics" minOccurs="0" maxOccurs="1"/>
                     </xs:choice>
                 </xs:extension>
             </xs:complexContent>
@@ -4709,6 +4710,114 @@
         </xs:all>
     </xs:complexType>
 
+    <xs:complexType name="metrics">
+        <xs:all>
+            <xs:element name="management-center" type="metrics-management-center" minOccurs="0"/>
+            <xs:element name="jmx" type="metrics-jmx" minOccurs="0"/>
+            <xs:element name="collection-frequency-seconds" type="xs:unsignedInt" default="5" minOccurs="0">
+                <xs:annotation>
+                    <xs:documentation>
+                        Sets the metrics collection frequency in seconds.
+                        By default, metrics are collected every 5 seconds.
+                        May be overridden by 'hazelcast.metrics.collection.frequency'
+                        system property.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:element>
+            <xs:element name="data-structure-metrics-enabled" type="xs:boolean" default="false" minOccurs="0">
+                <xs:annotation>
+                    <xs:documentation>
+                        Controls if metrics should be collected for the
+                        distributed data structures (maps, caches etc).
+                        It is disabled by default.
+                        May be overridden by 'hazelcast.metrics.datastructures'
+                        system property.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:element>
+            <xs:element name="level" type="metric-level" default="INFO" minOccurs="0">
+                <xs:annotation>
+                    <xs:documentation>
+                        Sets the minimum metrics level to be collected. By default
+                        it is set to INFO so metrics collect at INFO and MANDATORY
+                        levels are collected, DEBUG metrics are not.
+                        May be overridden by 'hazelcast.metrics.level'
+                        system property.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:element>
+        </xs:all>
+        <xs:attribute name="enabled" type="xs:boolean" default="true">
+            <xs:annotation>
+                <xs:documentation>
+                    Master-switch for the metrics system. Controls whether
+                    the metrics are collected and publishers are enabled.
+                    May be overridden by 'hazelcast.metrics.enabled'
+                    system property.
+                </xs:documentation>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:complexType>
+
+    <xs:complexType name="metrics-management-center">
+        <xs:all>
+            <xs:element name="retention-seconds" type="xs:unsignedInt" default="5" minOccurs="0">
+                <xs:annotation>
+                    <xs:documentation>
+                        Sets the number of seconds the metrics will be retained on the
+                        instance. By default, metrics are retained for 5 seconds (that is for
+                        one collection of metrics values, if default "collection-frequency-seconds"
+                        collection frequency is used). More retention means more heap memory, but
+                        allows for longer client hiccups without losing a value (for example to
+                        restart the Management Center).
+                        May be overridden by 'hazelcast.metrics.mc.retention'
+                        system property.
+                    </xs:documentation>
+                </xs:annotation>
+            </xs:element>
+        </xs:all>
+        <xs:attribute name="enabled" type="xs:boolean" default="true">
+            <xs:annotation>
+                <xs:documentation>
+                    Controls whether the metrics collected are exposed to
+                    Hazelcast Management Center. It is enabled by default.
+                    Please note that the metrics are polled by the
+                    Hazelcast Management Center, hence the members need to
+                    buffer the collected metrics between two polls. The aim
+                    for this switch is to reduce memory consumption of the
+                    metrics system if the Hazelcast Management Center is not
+                    used.
+                    In order to expose the metrics, the metrics system need
+                    to be enabled via the enabled master-switch attribute.
+                    May be overridden by 'hazelcast.metrics.mc.enabled'
+                    system property.
+                </xs:documentation>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:complexType>
+
+    <xs:complexType name="metrics-jmx">
+        <xs:attribute name="enabled" type="xs:boolean" default="true">
+            <xs:annotation>
+                <xs:documentation>
+                    Controls whether the metrics collected are exposed to
+                    through JMX. It is enabled by default.
+                    In order to expose the metrics, the metrics system need
+                    to be enabled via the enabled master-switch attribute.
+                    May be overridden by 'hazelcast.metrics.jmx.enabled'
+                    system property.
+                </xs:documentation>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:complexType>
+
+    <xs:simpleType name="metric-level">
+        <xs:restriction base="non-space-string">
+            <xs:enumeration value="MANDATORY"/>
+            <xs:enumeration value="INFO"/>
+            <xs:enumeration value="DEBUG"/>
+        </xs:restriction>
+    </xs:simpleType>
 
     <xs:complexType name="realms">
         <xs:sequence>

hazelcast-spring/src/test/java/com/hazelcast/spring/TestFullApplicationContext.java

@@ -68,6 +68,7 @@
 import com.hazelcast.config.MemcacheProtocolConfig;
 import com.hazelcast.config.MergePolicyConfig;
 import com.hazelcast.config.MetadataPolicy;
+import com.hazelcast.config.MetricsConfig;
 import com.hazelcast.config.MultiMapConfig;
 import com.hazelcast.config.NativeMemoryConfig;
 import com.hazelcast.config.NearCacheConfig;
@@ -102,7 +103,6 @@
 import com.hazelcast.config.WanAcknowledgeType;
 import com.hazelcast.config.WanBatchReplicationPublisherConfig;
 import com.hazelcast.config.WanConsumerConfig;
-import com.hazelcast.wan.WanPublisherState;
 import com.hazelcast.config.WanQueueFullBehavior;
 import com.hazelcast.config.WanReplicationConfig;
 import com.hazelcast.config.WanReplicationRef;
@@ -143,6 +143,7 @@
 import com.hazelcast.test.annotation.QuickTest;
 import com.hazelcast.topic.ITopic;
 import com.hazelcast.topic.TopicOverloadPolicy;
+import com.hazelcast.wan.WanPublisherState;
 import com.hazelcast.wan.WanReplicationPublisher;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -170,6 +171,7 @@
 import java.util.concurrent.ExecutorService;
 
 import static com.hazelcast.config.HotRestartClusterDataRecoveryPolicy.PARTIAL_RECOVERY_MOST_COMPLETE;
+import static com.hazelcast.internal.metrics.ProbeLevel.DEBUG;
 import static com.hazelcast.internal.util.CollectionUtil.isNotEmpty;
 import static com.hazelcast.spi.properties.GroupProperty.MERGE_FIRST_RUN_DELAY_SECONDS;
 import static com.hazelcast.spi.properties.GroupProperty.MERGE_NEXT_RUN_DELAY_SECONDS;
@@ -1410,4 +1412,16 @@ public void testCPSubsystemConfig() {
         assertEquals(1, lockConfig1.getLockAcquireLimit());
         assertEquals(2, lockConfig2.getLockAcquireLimit());
     }
+
+    @Test
+    public void testMetricsConfig() {
+        MetricsConfig metricsConfig = config.getMetricsConfig();
+        assertFalse(metricsConfig.isEnabled());
+        assertFalse(metricsConfig.getManagementCenterConfig().isEnabled());
+        assertEquals(42, metricsConfig.getManagementCenterConfig().getRetentionSeconds());
+        assertFalse(metricsConfig.getJmxConfig().isEnabled());
+        assertEquals(24, metricsConfig.getCollectionFrequencySeconds());
+        assertTrue(metricsConfig.isDataStructureMetricsEnabled());
+        assertEquals(DEBUG, metricsConfig.getLevel());
+    }
 }

hazelcast-spring/src/test/resources/com/hazelcast/spring/fullConfig-applicationContext-hazelcast.xml

@@ -915,6 +915,16 @@
                     </hz:fenced-lock>
                 </hz:locks>
             </hz:cp-subsystem>
+
+            <hz:metrics enabled="false">
+                <hz:management-center enabled="false">
+                    <hz:retention-seconds>42</hz:retention-seconds>
+                </hz:management-center>
+                <hz:jmx enabled="false"/>
+                <hz:collection-frequency-seconds>24</hz:collection-frequency-seconds>
+                <hz:data-structure-metrics-enabled>true</hz:data-structure-metrics-enabled>
+                <hz:level>DEBUG</hz:level>
+            </hz:metrics>
         </hz:config>
     </hz:hazelcast>
 

hazelcast/src/main/java/com/hazelcast/client/impl/clientside/HazelcastClientInstanceImpl.java

@@ -310,7 +310,7 @@ private Diagnostics initDiagnostics() {
     }
 
     private MetricsRegistryImpl initMetricsRegistry() {
-        ProbeLevel probeLevel = config.getMetricsConfig().getMinimumLevel();
+        ProbeLevel probeLevel = config.getMetricsConfig().getLevel();
         ILogger logger = loggingService.getLogger(MetricsRegistryImpl.class);
         MetricsRegistryImpl metricsRegistry = new MetricsRegistryImpl(getName(), logger, probeLevel);
         return metricsRegistry;

hazelcast/src/main/java/com/hazelcast/config/ConfigXmlGenerator.java

@@ -1550,13 +1550,15 @@ private static void cpSubsystemConfig(XmlGenerator gen, Config config) {
     private static void metricsConfig(XmlGenerator gen, Config config) {
         MetricsConfig metricsConfig = config.getMetricsConfig();
         gen.open("metrics",
-                "enabled", metricsConfig.isEnabled(),
-                "mc-enabled", metricsConfig.isMcEnabled(),
-                "jmx-enabled", metricsConfig.isJmxEnabled())
-           .node("collection-interval-seconds", metricsConfig.getCollectionIntervalSeconds())
-           .node("retention-seconds", metricsConfig.getRetentionSeconds())
-           .node("metrics-for-data-structures", metricsConfig.isMetricsForDataStructuresEnabled())
-           .node("minimum-level", metricsConfig.getMinimumLevel())
+                "enabled", metricsConfig.isEnabled())
+           .open("management-center", "enabled", metricsConfig.getManagementCenterConfig().isEnabled())
+           .node("retention-seconds", metricsConfig.getManagementCenterConfig().getRetentionSeconds())
+           .close()
+           .open("jmx", "enabled", metricsConfig.getJmxConfig().isEnabled())
+           .close()
+           .node("collection-frequency-seconds", metricsConfig.getCollectionFrequencySeconds())
+           .node("data-structure-metrics-enabled", metricsConfig.isDataStructureMetricsEnabled())
+           .node("level", metricsConfig.getLevel())
            .close();
     }