Add datastore cluster tag (#7603)

* Add tag for vsphere datastore cluster

* Add new tag, in addition to legacy 'folder' tag

* Add option to disable datastore cluster name as vsphere folder tag

* Remove duplicated test file

* Update vsphere/tests/test_check.py

Co-authored-by: Florian Veaux <florian.veaux@datadoghq.com>

Co-authored-by: Florian Veaux <florian.veaux@datadoghq.com>

.stubs/pyVmomi/vim/__init__.pyi

@@ -108,6 +108,7 @@ class Datastore(ManagedEntity): ...
 class ClusterComputeResource(ManagedEntity): ...
 class ComputeResource(ManagedEntity): ...
 class Folder(ManagedEntity): ...
+class StoragePod(ManagedEntity): ...
 
 class VirtualMachinePowerState(Enum):
     poweredOff: int

vsphere/assets/configuration/spec.yaml

@@ -401,4 +401,14 @@ files:
         value:
           type: integer
           example: 1800
+      - name: include_datastore_cluster_folder_tag
+        description: |
+          If a datastore is part of a datastore cluster, the tag "vsphere_datastore_cluster" indicating the datastore
+          cluster name is sent along with datastore metrics.
+          However, for compatibility reasons, this name is also sent as tag "vsphere_folder".
+          You can disable this by setting this property to false.
+          The default value is true.
+        value:
+          type: boolean
+          example: true
       - template: instances/default  

vsphere/datadog_checks/vsphere/config.py

@@ -86,7 +86,7 @@ def __init__(self, instance, log):
         self.collect_per_instance_filters = self._parse_metric_regex_filters(
             instance.get("collect_per_instance_filters", {})
         )
-
+        self.include_datastore_cluster_folder_tag = instance.get("include_datastore_cluster_folder_tag", True)
         self.validate_config()
 
     def is_historical(self):

vsphere/datadog_checks/vsphere/data/conf.yaml.example

@@ -318,6 +318,15 @@ instances:
     #
     # refresh_metrics_metadata_cache_interval: 1800
 
+    ## @param include_datastore_cluster_folder_tag - boolean - optional - default: true
+    ## If a datastore is part of a datastore cluster, the tag "vsphere_datastore_cluster" indicating the datastore
+    ## cluster name is sent along with datastore metrics.
+    ## However, for compatibility reasons, this name is also sent as tag "vsphere_folder".
+    ## You can disable this by setting this property to false.
+    ## The default value is true.
+    #
+    # include_datastore_cluster_folder_tag: true
+
     ## @param tags - list of strings - optional
     ## A list of tags to attach to every metric and service check emitted by this instance.
     ##

vsphere/datadog_checks/vsphere/types.py

@@ -39,6 +39,7 @@
         'resource_filters': List[ResourceFilterConfig],
         'metric_filters': MetricFilterConfig,
         'collect_per_instance_filters': MetricFilterConfig,
+        'include_datastore_cluster_folder_tag': bool,
     },
 )
 

vsphere/datadog_checks/vsphere/utils.py

@@ -85,8 +85,8 @@ def is_metric_excluded_by_filters(metric_name, mor_type, metric_filters):
     return True
 
 
-def get_parent_tags_recursively(mor, infrastructure_data):
-    # type: (vim.ManagedEntity, InfrastructureData) -> List[str]
+def get_parent_tags_recursively(mor, infrastructure_data, config):
+    # type: (vim.ManagedEntity, InfrastructureData, VSphereConfig) -> List[str]
     """Go up the resources hierarchy from the given mor. Note that a host running a VM is not considered to be a
     parent of that VM.
 
@@ -108,7 +108,13 @@ def get_parent_tags_recursively(mor, infrastructure_data):
         if isinstance(parent, vim.HostSystem):
             tags.append('vsphere_host:{}'.format(parent_name))
         elif isinstance(parent, vim.Folder):
-            tags.append('vsphere_folder:{}'.format(parent_name))
+            if isinstance(parent, vim.StoragePod):
+                tags.append('vsphere_datastore_cluster:{}'.format(parent_name))
+                # Legacy mode: keep it as "folder"
+                if config.include_datastore_cluster_folder_tag:
+                    tags.append('vsphere_folder:{}'.format(parent_name))
+            else:
+                tags.append('vsphere_folder:{}'.format(parent_name))
         elif isinstance(parent, vim.ComputeResource):
             if isinstance(parent, vim.ClusterComputeResource):
                 tags.append('vsphere_cluster:{}'.format(parent_name))
@@ -118,7 +124,7 @@ def get_parent_tags_recursively(mor, infrastructure_data):
         elif isinstance(parent, vim.Datastore):
             tags.append('vsphere_datastore:{}'.format(parent_name))
 
-        parent_tags = get_parent_tags_recursively(parent, infrastructure_data)
+        parent_tags = get_parent_tags_recursively(parent, infrastructure_data, config)
         parent_tags.extend(tags)
         return parent_tags
     return []

vsphere/datadog_checks/vsphere/vsphere.py

@@ -243,7 +243,7 @@ def refresh_infrastructure_cache(self):
             else:
                 tags.append('vsphere_{}:{}'.format(mor_type_str, mor_name))
 
-            tags.extend(get_parent_tags_recursively(mor, infrastructure_data))
+            tags.extend(get_parent_tags_recursively(mor, infrastructure_data, self.config))
             tags.append('vsphere_type:{}'.format(mor_type_str))
 
             # Attach tags from fetched attributes.

vsphere/tests/fixtures/metrics_historical_values.json

@@ -103,6 +103,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 1",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -116,6 +117,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 2",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -177,6 +179,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 1",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -190,6 +193,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 2",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -251,6 +255,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 1",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -264,6 +269,7 @@
             "vsphere_datacenter:Dätacenter",
             "vsphere_datastore:Datastore 2",
             "vsphere_folder:Datacenters",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_folder:datastore",
             "vsphere_type:datastore"
@@ -1598,6 +1604,7 @@
             "vsphere_folder:Datacenters",
             "vsphere_datacenter:Dätacenter",
             "vsphere_folder:datastore",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_type:datastore"
         ]
@@ -1611,6 +1618,7 @@
             "vsphere_folder:Datacenters",
             "vsphere_datacenter:Dätacenter",
             "vsphere_folder:datastore",
+            "vsphere_datastore_cluster:Datastores",
             "vsphere_folder:Datastores",
             "vsphere_type:datastore"
         ]

vsphere/tests/test_check.py

@@ -53,6 +53,28 @@ def test_historical_metrics(aggregator, dd_run_check, historical_instance):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.usefixtures("mock_type", "mock_threadpool", "mock_api")
+def test_historical_metrics_no_dsc_folder(aggregator, dd_run_check, historical_instance):
+    """This test does the same check than test_historical_events, but deactivate the option to get datastore cluster
+    folder in metrics tags"""
+    check = VSphereCheck('vsphere', {}, [historical_instance])
+    check.config.include_datastore_cluster_folder_tag = False
+    dd_run_check(check)
+
+    fixture_file = os.path.join(HERE, 'fixtures', 'metrics_historical_values.json')
+
+    with open(fixture_file, 'r') as f:
+        data = json.load(f)
+        for metric in data:
+            all_tags = metric.get('tags')
+            if all_tags is not None:
+                # The tag 'vsphere_folder:Datastores' is not supposed to be there anymore!
+                all_tags = [tag for tag in all_tags if tag != 'vsphere_folder:Datastores']
+            aggregator.assert_metric(metric['name'], metric.get('value'), tags=all_tags)
+
+    aggregator.assert_all_metrics_covered()
+
+
 @pytest.mark.usefixtures('mock_type', 'mock_threadpool', 'mock_api', 'mock_rest_api')
 def test_events_only(aggregator, events_only_instance):
     check = VSphereCheck('vsphere', {}, [events_only_instance])