Merge branch 'main' into COST-4389-fix-parquet-masu

.baseimagedigest

@@ -1,2 +1,2 @@
 sha256:87bcbfedfd70e67aab3875fff103bade460aeff510033ebb36b7efa009ab6639
-696dc6b8af41edae51204a3fcaca22760caea8ced3db440706f1110ff58730e1  -
+5869bead574dd90c800c62a1aafb8ef0516ed1e8b23874b8ed4c7755968614ee  -

docs/specs/openapi.json

@@ -5593,7 +5593,9 @@
                         }
                     }
                 }
-            },
+            }
+        },
+        "/settings/cost/groups/add/": {
             "put": {
                 "tags": [
                     "Settings",
@@ -5619,14 +5621,19 @@
                         }
                     }
                 }
-            },
-            "delete": {
+            }
+        },
+        "/settings/cost-groups/remove/": {
+            "put": {
                 "tags": [
                     "Settings",
                     "Cost Groups"
                 ],
-                "summary": "Remove projects from a coust group",
-                "operationId": "deleteSettingsCostGroups",
+                "summary": "Remove projects from a cost group",
+                "operationId": "putSettingsCostGroupsRemove",
+                "requestBody": {
+                    "$ref": "#/components/requestBodies/CostGroupsBody"
+                },
                 "responses": {
                     "204": {
                         "description": "Cost groups updated"

koku/api/report/queries.py

@@ -205,6 +205,11 @@ def is_aws(self):
         """Determine if we are working with an AWS API."""
         return "aws" in self.parameters.request.path
 
+    @property
+    def is_azure(self):
+        """Determine if we are working with an Azure API."""
+        return "azure" in self.parameters.request.path
+
     def initialize_totals(self):
         """Initialize the total response column values."""
         query_sum = {}
@@ -1123,6 +1128,8 @@ def _group_by_ranks(self, query, data):  # noqa: C901
         )
         if self.is_aws and "account" in self.parameters.url_data:
             ranks = ranks.annotate(**{"account_alias": F("account_alias__account_alias")})
+        if self.is_azure and "subscription_guid" in self.parameters.url_data:
+            ranks = ranks.annotate(**{"subscription_name": F("subscription_name")})
         if self.is_openshift:
             ranks = ranks.annotate(clusters=ArrayAgg(Coalesce("cluster_alias", "cluster_id"), distinct=True))
 
@@ -1168,6 +1175,10 @@ def _ranked_list(self, data_list, ranks, rank_fields=None):  # noqa C901
             drop_columns.add("account_alias")
         if self.is_aws and "account" not in group_by:
             rank_data_frame.drop(columns=["account_alias"], inplace=True, errors="ignore")
+        if self.is_azure and "subscription_guid" in group_by:
+            drop_columns.add("subscription_name")
+        if self.is_azure and "subscription_guid" not in group_by:
+            rank_data_frame.drop(columns=["subscription_name"], errors="ignore")
 
         agg_fields = {}
         for col in [col for col in self.report_annotations if "units" in col]:
@@ -1227,7 +1238,6 @@ def _ranked_list(self, data_list, ranks, rank_fields=None):  # noqa C901
 
         # Finally replace any remaining NaN with None for JSON compatibility
         data_frame = data_frame.replace({np.nan: None})
-
         return data_frame.to_dict("records")
 
     def _aggregate_ranks_over_limit(self, data_frame, group_by):
@@ -1265,6 +1275,8 @@ def _aggregate_ranks_over_limit(self, data_frame, group_by):
                     others_data_frame["default_project"] = "False"
         if self.is_aws and "account" in group_by:
             others_data_frame["account_alias"] = other_str
+        elif self.is_azure and "subscription_guid" in group_by:
+            others_data_frame["subscription_name"] = other_str
         elif "gcp_project" in group_by:
             others_data_frame["gcp_project_alias"] = other_str
 

koku/api/report/test/azure/test_queries.py

@@ -0,0 +1,200 @@
+#
+# Copyright 2023 Red Hat Inc.
+#
+from unittest.mock import patch
+
+from api.iam.test.iam_test_case import IamTestCase
+from api.report.azure.query_handler import AzureReportQueryHandler
+from api.report.azure.view import AzureCostView
+from api.report.queries import ReportQueryHandler
+
+
+class AzureReportQueryTest(IamTestCase):
+    """Tests the Azure report queries."""
+
+    def test_null_subscription_name_azure_accounts(self):
+        with patch.object(ReportQueryHandler, "is_azure", return_value=True):
+            """Test if Azure reports have not null subscription_name value."""
+
+            url = "?group_by[subscription_guid]=*&filter[limit]=100"  # noqa: E501
+            query_params = self.mocked_query_params(url, AzureCostView)
+            handler = AzureReportQueryHandler(query_params)
+
+            ranks = [
+                {
+                    "subscription_guid": "9999991",
+                    "cost_total": "99",
+                    "rank": 1,
+                    "source_uuid": ["UUID('8888881')"],
+                    "subscription_name": "azure_single_source_subs_name",
+                },
+                {
+                    "subscription_guid": "9999992",
+                    "cost_total": "88",
+                    "rank": 2,
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "9999992",
+                },
+            ]
+
+            data_list = [
+                {
+                    "subscription_guid": "9999991",
+                    "date": "2023-11-22",
+                    "infra_total": "99",
+                    "infra_raw": "99",
+                    "infra_usage": "0",
+                    "infra_markup": "99",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "99",
+                    "cost_raw": "99",
+                    "cost_usage": "0",
+                    "cost_markup": "99",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888881')"],
+                    "subscription_name": "test_subscription_name_9999991",
+                },
+                {
+                    "subscription_guid": "9999992",
+                    "date": "2023-11-13",
+                    "infra_total": "88",
+                    "infra_raw": "88",
+                    "infra_usage": "0",
+                    "infra_markup": "88",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "88",
+                    "cost_raw": "88",
+                    "cost_usage": "0",
+                    "cost_markup": "44",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "test_subscription_name_9999992",
+                },
+                {
+                    "subscription_guid": "9999993",
+                    "date": "2023-11-14",
+                    "infra_total": "Decimal('36.350995153')",
+                    "infra_raw": "Decimal('36.350995153')",
+                    "infra_usage": "0",
+                    "infra_markup": "Decimal('0E-9')",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "Decimal('36.350995153')",
+                    "cost_raw": "Decimal('36.350995153')",
+                    "cost_usage": "0",
+                    "cost_markup": "Decimal('0E-9')",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "test_subscription_name_9999993",
+                },
+            ]
+
+            ranked_list = handler._ranked_list(data_list, ranks)
+            for data in ranked_list:
+                self.assertIsNotNone(data["subscription_name"])
+
+    def test_cost_endpoint_with_group_by_subscription_guid(self):
+        with patch.object(ReportQueryHandler, "is_azure", return_value=True):
+            """Test if Azure reports works properly without subscription_guid in group_by clause."""
+
+            url = "?group_by[service_name]=a&filter[limit]=100"  # noqa: E501
+            query_params = self.mocked_query_params(url, AzureCostView)
+            handler = AzureReportQueryHandler(query_params)
+
+            ranks = [
+                {
+                    "subscription_guid": "9999991",
+                    "cost_total": "99",
+                    "rank": 1,
+                    "source_uuid": ["UUID('8888881')"],
+                    "subscription_name": "azure_single_source_subs_name",
+                    "service_name": "abc",
+                },
+                {
+                    "subscription_guid": "9999992",
+                    "cost_total": "88",
+                    "rank": 2,
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "9999992",
+                },
+            ]
+
+            data_list = [
+                {
+                    "subscription_guid": "9999991",
+                    "date": "2023-11-22",
+                    "infra_total": "99",
+                    "infra_raw": "99",
+                    "infra_usage": "0",
+                    "infra_markup": "99",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "99",
+                    "cost_raw": "99",
+                    "cost_usage": "0",
+                    "cost_markup": "99",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888881')"],
+                    "subscription_name": "test_subscription_name_9999991",
+                },
+                {
+                    "subscription_guid": "9999992",
+                    "date": "2023-11-13",
+                    "infra_total": "88",
+                    "infra_raw": "88",
+                    "infra_usage": "0",
+                    "infra_markup": "88",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "88",
+                    "cost_raw": "88",
+                    "cost_usage": "0",
+                    "cost_markup": "44",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "test_subscription_name_9999992",
+                    "service_name": "abc",
+                },
+                {
+                    "subscription_guid": "9999993",
+                    "date": "2023-11-14",
+                    "infra_total": "Decimal('36.350995153')",
+                    "infra_raw": "Decimal('36.350995153')",
+                    "infra_usage": "0",
+                    "infra_markup": "Decimal('0E-9')",
+                    "sup_raw": "0",
+                    "sup_usage": "0",
+                    "sup_markup": "0",
+                    "sup_total": "0",
+                    "cost_total": "Decimal('36.350995153')",
+                    "cost_raw": "Decimal('36.350995153')",
+                    "cost_usage": "0",
+                    "cost_markup": "Decimal('0E-9')",
+                    "cost_units": "USD",
+                    "source_uuid": ["UUID('8888882')"],
+                    "subscription_name": "test_subscription_name_9999993",
+                },
+            ]
+
+            ranked_list = handler._ranked_list(data_list, ranks)
+            self.assertIsNotNone(ranked_list)
+
+    def test_group_by_ranks(self):
+        with patch.object(ReportQueryHandler, "is_azure", return_value=True):
+            url = "?group_by[subscription_guid]=*&filter[limit]=100"
+            query_params = self.mocked_query_params(url, AzureCostView)
+            handler = AzureReportQueryHandler(query_params)
+            query_output = handler.execute_query()
+            self.assertIsNotNone(query_output.get("data"))

koku/api/settings/cost_groups/query_handler.py

@@ -55,32 +55,33 @@ def _remove_default_projects(projects: list[dict[str, str]]) -> list[dict[str, s
 def put_openshift_namespaces(projects: list[dict[str, str]]) -> list[dict[str, str]]:
     projects = _remove_default_projects(projects)
 
-    # Build mapping of cost groups to cost category IDs in order to easiy get
+    # Build mapping of cost groups to cost category IDs in order to easily get
     # the ID of the cost group to update
     cost_groups = {item["name"]: item["id"] for item in OpenshiftCostCategory.objects.values("name", "id")}
 
-    namespaces_to_create = [
-        OpenshiftCostCategoryNamespace(
-            namespace=new_project["project"],
-            system_default=False,
-            cost_category_id=cost_groups[new_project["group"]],
-        )
-        for new_project in projects
-    ]
-    try:
-        # Perform bulk create
-        OpenshiftCostCategoryNamespace.objects.bulk_create(namespaces_to_create)
-    except IntegrityError as e:
-        # Handle IntegrityError (e.g., if a unique constraint is violated)
-        LOG.warning(f"IntegrityError: {e}")
+    # TODO: With Django 4.2, we can move back to using bulk_updates() since it allows updating conflicts
+    #       https://docs.djangoproject.com/en/4.2/ref/models/querysets/#bulk-create
+    for new_project in projects:
+        try:
+            OpenshiftCostCategoryNamespace.objects.update_or_create(
+                namespace=new_project["project"],
+                system_default=False,
+                cost_category_id=cost_groups[new_project["group"]],
+            )
+        except IntegrityError as e:
+            # The project already exists. Move it to a different cost group.
+            LOG.warning(f"IntegrityError: {e}")
+            OpenshiftCostCategoryNamespace.objects.filter(namespace=new_project["project"]).update(
+                cost_category_id=cost_groups[new_project["group"]]
+            )
 
     return projects
 
 
 def delete_openshift_namespaces(projects: list[dict[str, str]]) -> list[dict[str, str]]:
     projects = _remove_default_projects(projects)
     projects_to_delete = [item["project"] for item in projects]
-    deleted_count, _ = (
+    deleted_count, deletions = (
         OpenshiftCostCategoryNamespace.objects.filter(namespace__in=projects_to_delete)
         .exclude(system_default=True)
         .delete()

koku/api/settings/cost_groups/view.py

@@ -70,24 +70,6 @@ def get(self, request: Request, **kwargs) -> Response:
 
         return paginator.paginated_response
 
-    def put(self, request: Request) -> Response:
-        serializer = CostGroupProjectSerializer(data=request.data, many=True)
-        serializer.is_valid(raise_exception=True)
-
-        projects = put_openshift_namespaces(serializer.validated_data)
-        self._summarize_current_month(request.user.customer.schema_name, projects)
-
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-    def delete(self, request: Request) -> Response:
-        serializer = CostGroupProjectSerializer(data=request.data, many=True)
-        serializer.is_valid(raise_exception=True)
-
-        projects = delete_openshift_namespaces(serializer.validated_data)
-        self._summarize_current_month(request.user.customer.schema_name, projects)
-
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
     def _summarize_current_month(self, schema_name: str, projects: list[dict[str, str]]) -> list[str]:
         """Resummarize OCP data for the current month."""
         projects_to_summarize = [proj["project"] for proj in projects]
@@ -111,3 +93,25 @@ def _summarize_current_month(self, schema_name: str, projects: list[dict[str, st
             async_ids.append(str(async_result))
 
         return async_ids
+
+
+class CostGroupsAddView(CostGroupsView):
+    def put(self, request: Request) -> Response:
+        serializer = CostGroupProjectSerializer(data=request.data, many=True)
+        serializer.is_valid(raise_exception=True)
+
+        projects = put_openshift_namespaces(serializer.validated_data)
+        self._summarize_current_month(request.user.customer.schema_name, projects)
+
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class CostGroupsRemoveView(CostGroupsView):
+    def put(self, request: Request) -> Response:
+        serializer = CostGroupProjectSerializer(data=request.data, many=True)
+        serializer.is_valid(raise_exception=True)
+
+        projects = delete_openshift_namespaces(serializer.validated_data)
+        self._summarize_current_month(request.user.customer.schema_name, projects)
+
+        return Response(status=status.HTTP_204_NO_CONTENT)