Warn about skewed results when using node-stats telemetry device

and also adjust docs.

Closes #608
Relates #627

docs/telemetry.rst

@@ -75,7 +75,8 @@ node-stats
 
 .. warning::
 
-    This telemetry device will record a lot of metrics and likely skew your measurement results.
+    Using this telemetry device will skew your results because the node-stats API triggers additional refreshes.
+    Additionally a lot of metrics get recorded impacting the measurement results even further.
 
 The node-stats telemetry devices regularly calls the `cluster node-stats API <https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html>`_ and records metrics from the following sections:
 

esrally/mechanic/telemetry.py

@@ -443,14 +443,18 @@ def record_stats_per_index(self, name, stats):
 
 
 class NodeStats(TelemetryDevice):
+    """
+    Gathers different node stats.
+    """
+
     internal = False
     command = "node-stats"
     human_name = "Node Stats"
     help = "Regularly samples node stats"
-
-    """
-    Gathers different node stats.
+    warning = """You have enabled the node-stats telemetry device, but requests to the _nodes/stats Elasticsearch endpoint
+          trigger additional refreshes and WILL SKEW results.
     """
+
     def __init__(self, telemetry_params, clients, metrics_store):
         super().__init__()
         self.telemetry_params = telemetry_params
@@ -464,6 +468,8 @@ def attach_to_cluster(self, cluster):
         super().attach_to_cluster(cluster)
 
     def on_benchmark_start(self):
+        console.warn(NodeStats.warning, logger=self.logger)
+
         recorder = []
         for cluster_name in self.specified_cluster_names:
             recorder = NodeStatsRecorder(self.telemetry_params, cluster_name, self.clients[cluster_name], self.metrics_store)

tests/mechanic/telemetry_test.py

@@ -24,6 +24,7 @@
 from esrally import config, metrics, exceptions
 from esrally.mechanic import telemetry, team, cluster
 from esrally.metrics import MetaInfoScope
+from esrally.utils import console
 
 
 def create_config():
@@ -624,6 +625,30 @@ def test_stores_filtered_ccr_stats(self, metrics_store_put_doc):
         )
 
 
+class NodeStatsTests(TestCase):
+    warning = """You have enabled the node-stats telemetry device, but requests to the _nodes/stats Elasticsearch endpoint
+          trigger additional refreshes and WILL SKEW results.
+    """
+
+    @mock.patch("esrally.mechanic.telemetry.NodeStatsRecorder", mock.Mock())
+    @mock.patch("esrally.mechanic.telemetry.SamplerThread", mock.Mock())
+    def test_prints_warning_using_node_stats(self):
+        clients = {"default": Client()}
+        cfg = create_config()
+        metrics_store = metrics.EsMetricsStore(cfg)
+        telemetry_params = {
+            "node-stats-sample-interval": random.randint(1, 100)
+        }
+        t = telemetry.NodeStats(telemetry_params, clients, metrics_store)
+
+        with mock.patch.object(console, "warn") as mocked_console_warn:
+            t.on_benchmark_start()
+        mocked_console_warn.assert_called_once_with(
+            NodeStatsTests.warning,
+            logger=t.logger
+        )
+
+
 class NodeStatsRecorderTests(TestCase):
     node_stats_response = {
         "cluster_name": "elasticsearch",