SKU based chargeback

packages/chargeback/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 0.2.4
+  changes:
+    - description: "Adding sku and cost_type to the billing_cluster_cost_lookup for future utilization"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/14545 
 - version: 0.2.3
   changes:
     - description: "Adding deployment filter, dataview and moving config portion to bottom of dashboard for better usability."

packages/chargeback/elasticsearch/ingest_pipeline/billing.yml

@@ -1,27 +1,53 @@
 ---
-description: 'Chargeback: Set composite_key from @timestamp and deployment_id.'
+description: "Chargeback: Set composite_key from @timestamp and deployment_id."
 processors:
-  - script:
-      ignore_failure: true
-      lang: painless
-      source: |
-        if (ctx['@timestamp'] != null) {
-            ctx.composite_key = ZonedDateTime.parse(ctx['@timestamp']).toLocalDate().toString() + '_' + ctx.deployment_id;
-        }
-      tag: ess_billing
-  - pipeline:
-      description: '[Fleet] Global pipeline for all data streams'
-      ignore_missing_pipeline: true
-      name: global@custom
-  - pipeline:
-      description: '[Fleet] Pipeline for all data streams of type `metrics`'
-      ignore_missing_pipeline: true
-      name: metrics@custom
-  - pipeline:
-      description: '[Fleet] Pipeline for all data streams of type `metrics` defined by the `chargeback` integration'
-      ignore_missing_pipeline: true
-      name: metrics-chargeback.integration@custom
-  - pipeline:
-      description: '[Fleet] Pipeline for the `chargeback.billing` dataset'
-      ignore_missing_pipeline: true
-      name: metrics-chargeback.billing@custom
+    - script:
+        ignore_failure: true
+        lang: painless
+        source: |
+            if (ctx['@timestamp'] != null) {
+                ctx.composite_key = ZonedDateTime.parse(ctx['@timestamp']).toLocalDate().toString() + '_' + ctx.deployment_id;
+            }
+        tag: ess_billing
+    - set:
+        field: cost_type
+        value: 'data-transfer'
+        if: ctx.sku =~ /^(aws|gcp|azure).data-transfer.*/
+    - set:
+        field: cost_type
+        value: 'snapshots'
+        if: ctx.sku =~ /^(aws|gcp|azure).snapshot-storage&/
+    - set:
+        field: cost_type
+        value: 'inference'
+        if: ctx.sku == 'global.inference-chat-input'
+    - grok:
+        field: sku
+        ignore_failure: true
+        patterns:
+            - "%{WORD}.es.%{WORD:cost_type}.%{GREEDYDATA}"
+            - "%{WORD}.%{DATA:cost_type}-vcu_%{GREEDYDATA}"
+            - "%{WORD}.%{WORD:cost_type}.%{GREEDYDATA}"
+            - "%{WORD}.%{DATA:cost_type}.%{GREEDYDATA}"
+        description: retrieve a human readable cost type from the sku field
+        if: ctx.cost_type == null
+    - set:
+        field: cost_type
+        value: 'datahot/datacontent'
+        if: ctx.cost_type == 'datahot'
+    - pipeline:
+        description: "[Fleet] Global pipeline for all data streams"
+        ignore_missing_pipeline: true
+        name: global@custom
+    - pipeline:
+        description: "[Fleet] Pipeline for all data streams of type `metrics`"
+        ignore_missing_pipeline: true
+        name: metrics@custom
+    - pipeline:
+        description: "[Fleet] Pipeline for all data streams of type `metrics` defined by the `chargeback` integration"
+        ignore_missing_pipeline: true
+        name: metrics-chargeback.integration@custom
+    - pipeline:
+        description: "[Fleet] Pipeline for the `chargeback.billing` dataset"
+        ignore_missing_pipeline: true
+        name: metrics-chargeback.billing@custom

packages/chargeback/elasticsearch/transform/billing_cluster_cost/fields/fields.yml

@@ -13,3 +13,9 @@
 - name: composite_key
   type: keyword
   description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and deployment ID.
+- name: sku
+  type: keyword
+  description: Component where the ecu is being spend
+- name: cost_type
+  type: keyword
+  description: Component where the ecu is being spent, derived from SKU field.

packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml

@@ -8,7 +8,7 @@ source:
         gt: 0
 dest:
   index: billing_cluster_cost_lookup
-  pipeline: 0.2.3-billing
+  pipeline: 0.2.4-billing
 frequency: 60m
 sync:
   time:
@@ -26,6 +26,9 @@ pivot:
     deployment_name:
       terms:
         field: ess.billing.deployment_name
+    sku:
+      terms:
+        field: ess.billing.sku
   aggregations:
     total_ecu:
       sum:
@@ -39,4 +42,4 @@ _meta:
   run_as_kibana_system: false
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.2.1
+  fleet_transform_version: 0.2.4

packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml

@@ -4,7 +4,7 @@ source:
     - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only.
 dest:
   index: cluster_datastream_contribution_lookup
-  pipeline: 0.2.3-usage
+  pipeline: 0.2.4-usage
 frequency: 60m
 sync:
   time:

packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml

@@ -4,7 +4,7 @@ source:
     - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only.
 dest:
   index: cluster_deployment_contribution_lookup
-  pipeline: 0.2.3-usage
+  pipeline: 0.2.4-usage
 frequency: 60m
 sync:
   time:

packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml

@@ -4,7 +4,7 @@ source:
     - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only.
 dest:
   index: cluster_tier_and_datastream_contribution_lookup
-  pipeline: 0.2.3-usage
+  pipeline: 0.2.4-usage
 frequency: 60m
 sync:
   time:

packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml

@@ -4,7 +4,7 @@ source:
     - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only.
 dest:
   index: cluster_tier_contribution_lookup
-  pipeline: 0.2.3-usage
+  pipeline: 0.2.4-usage
 frequency: 60m
 sync:
   time:

packages/chargeback/manifest.yml

@@ -1,15 +1,15 @@
 format_version: 3.4.0
 name: chargeback
 title: "Chargeback"
-version: 0.2.3
+version: 0.2.4
 description: "This package calculates chargeback based on billing and consumption data"
 type: integration
 categories:
   - cloud
   - custom
 conditions:
   kibana:
-    version: "9.2.0"
+    version: "^9.2.0"
   elastic:
     subscription: "basic"
 screenshots:

packages/chargeback/pre-setup.sh

@@ -10,7 +10,7 @@ PUT chargeback_conf_lookup
   "mappings": {
     "_meta": {
       "managed": true,
-      "package": { "name": "chargeback", "version": "0.2.3" }
+      "package": { "name": "chargeback", "version": "0.2.4" }
     },
     "properties": {
       "config_join_key": { "type": "keyword" },