Merge new glance metrics into master (#17492)

* [PLINT-366] Adding glance image up and size metrics (#17454) * adding metric openstack.glance.image.up Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding metric openstack.glance.image.bytes Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding image id as a tag Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * lint Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * removing openstack.glance.image.bytes Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding openstack.glance.image.size Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * removing redundant disable glance metrics test Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * implementing suggestions Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * addressing more comments Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> --------- Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * [PLINT-374] Adding glance image task metrics (#17477) * adding metric openstack.glance.image.up Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding metric openstack.glance.image.bytes Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding image id as a tag Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * lint Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * removing openstack.glance.image.bytes Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding openstack.glance.image.size Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding openstack.glance.image.task.count Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding another image to test Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * adding a test where tasks is specifically disabled Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * fixing metadata.csv description Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> --------- Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * [PLINT-376] Adding glance image member metrics (#17487) * adding metric openstack.glance.image.member.count Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * testing member metrics Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> --------- Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * added a changelog Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> * fixed changelog number Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com> --------- Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>
DataDog · May 1, 2024 · b493c8f · b493c8f
1 parent e3f334d
commit b493c8f
Show file tree

Hide file tree

Showing 17 changed files with 535 additions and 38 deletions.
diff --git a/openstack_controller/assets/configuration/spec.yaml b/openstack_controller/assets/configuration/spec.yaml
@@ -629,7 +629,14 @@ files:
                   - type: object
                     properties:
                       - name: images
-                        type: boolean
+                        anyOf:
+                          - type: boolean
+                          - type: object
+                            properties:
+                              - name: members
+                                type: boolean
+                              - name: tasks
+                                type: boolean
             example:
               compute: false
         - name: projects

diff --git a/openstack_controller/changelog.d/17492.added b/openstack_controller/changelog.d/17492.added
@@ -0,0 +1 @@
+Adding new glance image metrics
diff --git a/openstack_controller/datadog_checks/openstack_controller/api/api_rest.py b/openstack_controller/datadog_checks/openstack_controller/api/api_rest.py
@@ -542,3 +542,17 @@ def get_glance_images(self):
             'id',
             next_signifier='next',
         )
+
+    def get_glance_members(self, image_id):
+        response = self.http.get(
+            '{}/v2/images/{}/members'.format(self._catalog.get_endpoint_by_type(Component.Types.IMAGE.value), image_id)
+        )
+        response.raise_for_status()
+        return response.json().get('members', [])
+
+    def get_glance_tasks(self, image_id):
+        response = self.http.get(
+            '{}/v2/images/{}/tasks'.format(self._catalog.get_endpoint_by_type(Component.Types.IMAGE.value), image_id)
+        )
+        response.raise_for_status()
+        return response.json().get('tasks', [])
diff --git a/openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py b/openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py
@@ -336,3 +336,9 @@ def get_glance_images(self):
             image.to_dict(original_names=True)
             for image in self.call_paginated_api(self.connection.image.images, limit=self.config.paginated_limit)
         ]
+
+    def get_glance_members(self, image_id):
+        return [member.to_dict(original_names=True) for member in self.connection.image.members(image_id)]
+
+    def get_glance_tasks(self, image_id):
+        return [task.to_dict(original_names=True) for task in self.connection.image.tasks(image_id)]
diff --git a/openstack_controller/datadog_checks/openstack_controller/components/image.py b/openstack_controller/datadog_checks/openstack_controller/components/image.py
@@ -6,10 +6,17 @@
 from datadog_checks.openstack_controller.components.component import Component
 from datadog_checks.openstack_controller.metrics import (
     GLANCE_IMAGE_COUNT,
+    GLANCE_IMAGE_METRICS,
+    GLANCE_IMAGE_PREFIX,
     GLANCE_IMAGE_TAGS,
-    GLANCE_METRICS_PREFIX,
+    GLANCE_MEMBER_COUNT,
+    GLANCE_MEMBER_PREFIX,
+    GLANCE_MEMBER_TAGS,
     GLANCE_RESPONSE_TIME,
     GLANCE_SERVICE_CHECK,
+    GLANCE_TASK_COUNT,
+    GLANCE_TASK_PREFIX,
+    GLANCE_TASK_TAGS,
     get_metrics_and_tags,
 )
 
@@ -40,8 +47,45 @@ def _report_images(self, config, tags):
                 image = get_metrics_and_tags(
                     item,
                     tags=GLANCE_IMAGE_TAGS,
-                    prefix=GLANCE_METRICS_PREFIX,
-                    metrics=[GLANCE_IMAGE_COUNT],
+                    prefix=GLANCE_IMAGE_PREFIX,
+                    metrics=GLANCE_IMAGE_METRICS,
+                    lambda_name=lambda key: 'up' if key == 'status' else key,
+                    lambda_value=lambda key, value, item=item: (
+                        item['status'] == 'active' if key == 'status' else value
+                    ),
                 )
                 self.check.log.debug("image: %s", image)
                 self.check.gauge(GLANCE_IMAGE_COUNT, 1, tags=tags + image['tags'])
+                for metric, value in image['metrics'].items():
+                    self.check.gauge(metric, value, tags=tags + image['tags'])
+                self.check.log.debug("reporting tasks and members for image: %s", item['id'])
+                self._report_members(config, tags, item['id'])
+                self._report_tasks(config, tags, item['id'])
+
+    @Component.http_error()
+    def _report_members(self, config, tags, image_id):
+        report_members = config.get('members', True)
+        if report_members:
+            data = self.check.api.get_glance_members(image_id)
+            for item in data:
+                member = get_metrics_and_tags(
+                    item,
+                    tags=GLANCE_MEMBER_TAGS,
+                    prefix=GLANCE_MEMBER_PREFIX,
+                    metrics=[GLANCE_MEMBER_COUNT],
+                )
+                self.check.gauge(GLANCE_MEMBER_COUNT, 1, tags=tags + member['tags'])
+
+    @Component.http_error()
+    def _report_tasks(self, config, tags, image_id):
+        report_tasks = config.get('tasks', True)
+        if report_tasks:
+            data = self.check.api.get_glance_tasks(image_id)
+            for item in data:
+                task = get_metrics_and_tags(
+                    item,
+                    tags=GLANCE_TASK_TAGS,
+                    prefix=GLANCE_TASK_PREFIX,
+                    metrics=[GLANCE_TASK_COUNT],
+                )
+                self.check.gauge(GLANCE_TASK_COUNT, 1, tags=tags + task['tags'])
diff --git a/openstack_controller/datadog_checks/openstack_controller/config_models/instance.py b/openstack_controller/datadog_checks/openstack_controller/config_models/instance.py
@@ -129,12 +129,21 @@ class IdentityItem(BaseModel):
     users: Optional[bool] = None
 
 
+class Image(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    members: Optional[bool] = None
+    tasks: Optional[bool] = None
+
+
 class ImageItem(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
         frozen=True,
     )
-    images: Optional[bool] = None
+    images: Optional[Union[bool, Image]] = None
 
 
 class IncludeItem3(BaseModel):

diff --git a/openstack_controller/datadog_checks/openstack_controller/metrics.py b/openstack_controller/datadog_checks/openstack_controller/metrics.py
@@ -560,12 +560,33 @@
 GLANCE_METRICS_PREFIX = "openstack.glance"
 GLANCE_SERVICE_CHECK = f"{GLANCE_METRICS_PREFIX}.api.up"
 GLANCE_RESPONSE_TIME = f"{GLANCE_METRICS_PREFIX}.response_time"
-GLANCE_IMAGE_COUNT = f"{GLANCE_METRICS_PREFIX}.image.count"
+GLANCE_IMAGE_PREFIX = f"{GLANCE_METRICS_PREFIX}.image"
+GLANCE_IMAGE_COUNT = f"{GLANCE_IMAGE_PREFIX}.count"
+GLANCE_IMAGE_METRICS = {
+    f"{GLANCE_IMAGE_PREFIX}.up": {},
+    f"{GLANCE_IMAGE_PREFIX}.size": {},
+}
 GLANCE_IMAGE_TAGS = {
+    'id': 'image_id',
     'name': 'image_name',
     'status': 'status',
     'container_format': 'container_format',
 }
+GLANCE_MEMBER_PREFIX = f"{GLANCE_IMAGE_PREFIX}.member"
+GLANCE_MEMBER_COUNT = f"{GLANCE_MEMBER_PREFIX}.count"
+GLANCE_MEMBER_TAGS = {
+    'member_id': 'member_id',
+    'image_id': 'image_id',
+    'status': 'status',
+}
+GLANCE_TASK_PREFIX = f"{GLANCE_IMAGE_PREFIX}.task"
+GLANCE_TASK_COUNT = f"{GLANCE_TASK_PREFIX}.count"
+GLANCE_TASK_TAGS = {
+    'id': 'task_id',
+    'image_id': 'image_id',
+    'type': 'type',
+    'status': 'status',
+}
 
 
 def is_interface_metric(label):

diff --git a/openstack_controller/metadata.csv b/openstack_controller/metadata.csv
@@ -13,6 +13,10 @@ openstack.cinder.volume.count,gauge,,,,Number of cinder volumes,0,openstack_cont
 openstack.cinder.volume.size,gauge,,,,Size of cinder volumes,0,openstack_controller,cinder volume size,
 openstack.cinder.volume.transfer.count,gauge,,,,Number of cinder volume transfers,0,openstack_controller,cinder volume transfer ct,
 openstack.glance.image.count,gauge,,,,Number of public virtual machine images,0,openstack_controller,glance image ct,
+openstack.glance.image.member.count,gauge,,,,Number of members associated with an image,0,openstack_controller,glance image member ct,
+openstack.glance.image.size,gauge,,,,Size of image file in bytes,0,openstack_controller,glance image size,
+openstack.glance.image.task.count,gauge,,,,Number of tasks associated with an image,0,openstack_controller,glance image task ct,
+openstack.glance.image.up,gauge,,,,Whether a glance image is up,0,openstack_controller,glance image up,
 openstack.glance.response_time,gauge,,millisecond,,Duration that an HTTP request takes to complete when making a request to glance endpoint,0,openstack_controller,glance response time,
 openstack.ironic.conductor.count,gauge,,,,Number of ironic conductors,0,openstack_controller,ironic conductor ct,
 openstack.ironic.conductor.up,gauge,,,,Whether an ironic conductor is up,0,openstack_controller,ironic conductor up,

diff --git a/openstack_controller/tests/conftest.py b/openstack_controller/tests/conftest.py
@@ -664,7 +664,35 @@ def images(limit=None):
             for node in mock_responses('GET', '/image/v2/images')['images']
         ]
 
-    return mock.MagicMock(images=mock.MagicMock(side_effect=images))
+    def members(image_id):
+        if http_error and 'members' in http_error:
+            raise requests.exceptions.HTTPError(response=http_error['members'])
+        return [
+            mock.MagicMock(
+                to_dict=mock.MagicMock(
+                    return_value=node,
+                )
+            )
+            for node in mock_responses('GET', f'/image/v2/images/{image_id}/members')['members']
+        ]
+
+    def tasks(image_id):
+        if http_error and 'tasks' in http_error:
+            raise requests.exceptions.HTTPError(response=http_error['tasks'])
+        return [
+            mock.MagicMock(
+                to_dict=mock.MagicMock(
+                    return_value=node,
+                )
+            )
+            for node in mock_responses('GET', f'/image/v2/images/{image_id}/tasks')['tasks']
+        ]
+
+    return mock.MagicMock(
+        images=mock.MagicMock(side_effect=images),
+        members=mock.MagicMock(side_effect=members),
+        tasks=mock.MagicMock(side_effect=tasks),
+    )
 
 
 @pytest.fixture

diff --git a/...s/fixtures/GET/image/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5/members/response.json b/...s/fixtures/GET/image/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5/members/response.json
@@ -0,0 +1,21 @@
+{
+    "members": [
+        {
+            "created_at": "2013-10-07T17:58:03Z",
+            "image_id": "28c293db-48c5-4c8b-a711-09b2c11fa8e5",
+            "member_id": "123456789",
+            "schema": "/v2/schemas/member",
+            "status": "pending",
+            "updated_at": "2013-10-07T17:58:03Z"
+        },
+        {
+            "created_at": "2013-10-07T17:58:55Z",
+            "image_id": "28c293db-48c5-4c8b-a711-09b2c11fa8e5",
+            "member_id": "987654321",
+            "schema": "/v2/schemas/member",
+            "status": "accepted",
+            "updated_at": "2013-10-08T12:08:55Z"
+        }
+    ],
+    "schema": "/v2/schemas/members"
+}
diff --git a/...sts/fixtures/GET/image/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5/tasks/response.json b/...sts/fixtures/GET/image/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5/tasks/response.json
@@ -0,0 +1,70 @@
+{
+    "tasks": [
+        {
+            "id": "ee22890e-8948-4ea6-9668-831f973c84f5",
+            "image_id": "28c293db-48c5-4c8b-a711-09b2c11fa8e5",
+            "request-id": "rrrrrrr-rrrr-rrrr-rrrr-rrrrrrrrrrrr",
+            "user": "uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu",
+            "type": "api_image_import",
+            "status": "processing",
+            "owner": "64f0efc9955145aeb06f297a8a6fe402",
+            "expires_at": null,
+            "created_at": "2020-12-18T05:20:38.000000",
+            "updated_at": "2020-12-18T05:25:39.000000",
+            "deleted_at": null,
+            "deleted": false,
+            "input": {
+                "image_id": "829c729b-ebc4-4cc7-a164-6f43f1149b17",
+                "import_req": {
+                    "method": {
+                        "name": "copy-image"
+                    },
+                    "all_stores": true,
+                    "all_stores_must_succeed": false
+                },
+                "backend": [
+                    "fast",
+                    "cheap",
+                    "slow",
+                    "reliable",
+                    "common"
+                ]
+            },
+            "result": null,
+            "message": "Copied 15 MiB"
+        },
+        {
+            "id": "de22890e-8948-4ea6-9668-831f973c84f5",
+            "image_id": "38c293db-48c5-4c8b-a711-09b2c11fa8e5",
+            "request-id": "rrrrrrr-rrrr-rrrr-rrrr-rrrrrrrrrrrr",
+            "user": "uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu",
+            "type": "api_image_import",
+            "status": "processing",
+            "owner": "64f0efc9955145aeb06f297a8a6fe402",
+            "expires_at": null,
+            "created_at": "2020-12-18T05:20:38.000000",
+            "updated_at": "2020-12-18T05:25:39.000000",
+            "deleted_at": null,
+            "deleted": false,
+            "input": {
+                "image_id": "829c729b-ebc4-4cc7-a164-6f43f1149b17",
+                "import_req": {
+                    "method": {
+                        "name": "copy-image"
+                    },
+                    "all_stores": true,
+                    "all_stores_must_succeed": false
+                },
+                "backend": [
+                    "fast",
+                    "cheap",
+                    "slow",
+                    "reliable",
+                    "common"
+                ]
+            },
+            "result": null,
+            "message": "Copied 15 MiB"
+        }
+    ]
+}
diff --git a/...res/GET/image/v2/images/limit=1/marker=28c293db-48c5-4c8b-a711-09b2c11fa8e5/response.json b/...res/GET/image/v2/images/limit=1/marker=28c293db-48c5-4c8b-a711-09b2c11fa8e5/response.json
@@ -1,28 +1,28 @@
 {
     "images": [
         {
-            "hw_rng_model": "virtio",
-            "owner_specified.openstack.md5": "",
-            "owner_specified.openstack.object": "images/cirros-0.5.2-x86_64-disk",
+            "owner_specified.openstack.object": "images/cirros-1.5.2-x86_64-disk",
             "owner_specified.openstack.sha256": "",
-            "name": "cirros-0.5.2-x86_64-disk",
+            "owner_specified.openstack.md5": "",
+            "hw_rng_model": "virtio",
+            "name": "cirros-1.5.2-x86_64-disk",
             "disk_format": "qcow2",
             "container_format": "bare",
             "visibility": "public",
-            "size": 16300544,
-            "virtual_size": 117440512,
+            "size": 46310543,
+            "virtual_size": 217940522,
             "status": "active",
-            "checksum": "b874c39491a2377b8490f5f1e89761a4",
+            "checksum": "b873c39491a1377c8490f5f1e89261a9",
             "protected": false,
             "min_ram": 0,
             "min_disk": 0,
-            "owner": "c09ccb52462d49c89595d2194a367198",
+            "owner": "c09ccb52463d49c89595d2194a367298",
             "os_hidden": false,
             "os_hash_algo": "sha512",
-            "os_hash_value": "6b813aa46bb90b4da216a4d19376593fa3f4fc7e617f03a92b7fe11e9a3981cbe8f0959dbebe36225e5f53dc4492341a4863cac4ed1ee0909f3fc78ef9c3e869",
+            "os_hash_value": "7b813aa46bb90b4da216a4d19376593fa3f3fc7e617f03a92b7fe11e9a3981cbe9f0959dbebe36225e5f53dc4492341a4863cac4ed1ee0909f3fc78ef9c3e861",
             "id": "a4dcf3f7-e4d6-45d5-be2b-a939f2bbedfc",
-            "created_at": "2023-10-18T15:03:09Z",
-            "updated_at": "2023-10-18T15:03:10Z",
+            "created_at": "2023-10-28T15:03:09Z",
+            "updated_at": "2023-10-28T15:03:10Z",
             "tags": [],
             "self": "/v2/images/a4dcf3f7-e4d6-45d5-be2b-a939f2bbedfc",
             "file": "/v2/images/a4dcf3f7-e4d6-45d5-be2b-a939f2bbedfc/file",

diff --git a/openstack_controller/tests/fixtures/GET/image/v2/images/limit=1/response.json b/openstack_controller/tests/fixtures/GET/image/v2/images/limit=1/response.json
@@ -1,27 +1,31 @@
 {
     "images": [
         {
-            "name": "cirros-1.5.2-x86_64-disk",
+            "owner_specified.openstack.object": "images/cirros-0.5.2-x86_64-disk",
+            "owner_specified.openstack.sha256": "",
+            "owner_specified.openstack.md5": "",
+            "hw_rng_model": "virtio",
+            "name": "cirros-0.5.2-x86_64-disk",
             "disk_format": "qcow2",
             "container_format": "bare",
             "visibility": "public",
-            "size": 16338944,
+            "size": 16300544,
             "virtual_size": 117440512,
             "status": "active",
-            "checksum": "1d3062cd89af34e419f7100277f38b2b",
+            "checksum": "b874c39491a2377b8490f5f1e89761a4",
             "protected": false,
             "min_ram": 0,
             "min_disk": 0,
-            "owner": "29b43a58176646e3870a9ea0b60fb7ad",
+            "owner": "c09ccb52462d49c89595d2194a367198",
             "os_hidden": false,
             "os_hash_algo": "sha512",
-            "os_hash_value": "553d220ed58cfee7dafe003c446a9f197ab5edf8ffc09396c74187cf83873c877e7ae041cb80f3b91489acf687183adcd689b53b38e3ddd22e627e7f98a09c46",
+            "os_hash_value": "6b813aa46bb90b4da216a4d19376593fa3f4fc7e617f03a92b7fe11e9a3981cbe8f0959dbebe36225e5f53dc4492341a4863cac4ed1ee0909f3fc78ef9c3e869",
             "id": "28c293db-48c5-4c8b-a711-09b2c11fa8e5",
-            "created_at": "2024-02-28T20:31:08Z",
-            "updated_at": "2024-02-28T20:31:09Z",
+            "created_at": "2023-10-18T15:03:09Z",
+            "updated_at": "2023-10-18T15:03:10Z",
             "tags": [],
-            "self": "/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5",
-            "file": "/v2/images/28c293db-48c5-4c8b-a711-09b2c11fa8e5/file",
+            "self": "/v2/images/a4dcf3f7-e4d6-45d5-be2b-a939f2bbedfc",
+            "file": "/v2/images/a4dcf3f7-e4d6-45d5-be2b-a939f2bbedfc/file",
             "schema": "/v2/schemas/image"
         }
     ],