Map Qualys cloud fields to cloud ECS fields

packages/qualys_gav/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.4.0"
+  changes:
+    - description: Map Qualys cloud fields to cloud ECS fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15810
+    - description: Fix mapping for field qualys_gav.asset.inventory_list_data.inventory
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15810
 - version: "0.3.1"
   changes:
     - description: Added support for comma separated list of IPv4 addresses in network_interface list data.

packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs

@@ -125,6 +125,9 @@ tags:
 {{#each tags as |tag|}}
   - {{tag}}
 {{/each}}
+{{#if cloud_data}}
+  - {{cloud_data}}
+{{/if}}
 {{#contains "forwarded" tags}}
 publisher_pipeline.disable_host: true
 {{/contains}}

packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml

@@ -23,6 +23,30 @@ processors:
         Removes the fields added by Agentless as metadata,
         as they can collide with ECS fields.
 
+  # Set up tag logic for choosing which cloud metadata to include.
+  - append:
+      field: tags
+      value: ["elastic_cloud_data", "provider_cloud_data"]
+      if: ctx.tags?.contains('both') == true
+  - script:
+      lang: painless
+      if: ctx.tags?.contains('both') == true
+      source: ctx.tags.remove(ctx.tags.indexOf('both'));
+  - script:
+      lang: painless
+      if: ctx.cloud == null && ctx.tags?.contains('elastic_cloud_data') == true
+      source: ctx.tags.remove(ctx.tags.indexOf('elastic_cloud_data'));
+  - set:
+      field: _conf.want_provider_cloud
+      value: true
+      if: ctx.tags?.contains('provider_cloud_data') == true
+  - remove:
+      description: Remove elastic agent-provided cloud metadata fields if not requested.
+      field: cloud
+      if: ctx.tags != null && !ctx.tags.contains('elastic_cloud_data')
+      ignore_missing: true
+      ignore_failure: true
+
   # parse the event JSON
   - rename:
       field: message
@@ -649,6 +673,34 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - date:
+      field: qualys_gav.asset.inventory_list_data.inventory.created
+      target_field: qualys_gav.asset.inventory_list_data.inventory.created
+      tag: date_qualys_gav_asset_inventory_list_data_inventory_created
+      formats:
+        - UNIX_MS
+        - ISO8601
+      if: ctx.qualys_gav?.asset?.inventory_list_data?.inventory?.created != null && ctx.qualys_gav.asset.inventory_list_data.inventory.created != ''
+      on_failure:
+        - remove:
+            field: qualys_gav.asset.inventory.created
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - date:
+      field: qualys_gav.asset.inventory_list_data.inventory.last_updated
+      target_field: qualys_gav.asset.inventory_list_data.inventory.last_updated
+      tag: date_qualys_gav_asset_inventory_last_updated
+      formats:
+        - UNIX_MS
+        - ISO8601
+      if: ctx.qualys_gav?.asset?.inventory_list_data?.inventory?.last_updated != null && ctx.qualys_gav.asset.inventory_list_data.inventory.last_updated != ''
+      on_failure:
+        - remove:
+            field: qualys_gav.asset.inventory_list_data.inventory.last_updated
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - date:
       field: qualys_gav.asset.activity.last_scanned_date
       target_field: qualys_gav.asset.activity.last_scanned_date
@@ -1094,6 +1146,8 @@ processors:
       target_field: cloud.provider
       tag: lowercase_cloud_provider_from_asset_provider
       ignore_missing: true
+
+  # Convert AWS EC2 specific fields
   - convert:
       field: qualys_gav.asset.cloud_provider.aws.ec2.has_agent
       tag: convert_qualys_gav_asset_cloud_provider_aws_ec2_has_agent_to_boolean
@@ -1145,11 +1199,185 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+
+  # Map/Convert AWS EC2 specific fields to ECS fields
+  - set:
+      field: cloud.account.id
+      tag: set_cloud_account_id_from_asset_cloud_provider_aws_ec2_account_id
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.account_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.availability_zone
+      tag: set_cloud_availability_zone_from_asset_cloud_provider_aws_ec2_availability_zone
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.availability_zone
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.id
+      tag: set_cloud_instance_id_from_asset_cloud_provider_aws_ec2_instance_id
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.instance_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.name
+      tag: set_cloud_instance_name_from_asset_cloud_provider_aws_ec2_hostname
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.hostname
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.machine.type
+      tag: set_cloud_machine_type_from_asset_cloud_provider_aws_ec2_instance_type
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.instance_type
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.region
+      tag: set_cloud_region_from_asset_cloud_provider_aws_ec2_region_code
+      copy_from: qualys_gav.asset.cloud_provider.aws.ec2.region.code
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.service.name
+      tag: set_cloud_service_name
+      if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'aws' && ctx.qualys_gav?.asset?.cloud_provider?.aws?.ec2 != null
+      value: ec2
+
+  # Map/Convert Azure specific fields to ECS fields
+  - set:
+      field: cloud.account.id
+      tag: set_cloud_account_id_from_asset_cloud_provider_azure_vm_subscription_id
+      copy_from: qualys_gav.asset.cloud_provider.azure.vm.subscription_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.id
+      tag: set_cloud_instance_id_from_asset_cloud_provider_azure_vm_vm_id
+      copy_from: qualys_gav.asset.cloud_provider.azure.vm.vm_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.name
+      tag: set_cloud_instance_name_from_asset_cloud_provider_azure_vm_name
+      copy_from: qualys_gav.asset.cloud_provider.azure.vm.name
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.machine.type
+      tag: set_cloud_machine_type_from_asset_cloud_provider_azure_vm_size
+      copy_from: qualys_gav.asset.cloud_provider.azure.vm.size
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.region
+      tag: set_cloud_region_from_asset_cloud_provider_azure_vm_location
+      copy_from: qualys_gav.asset.cloud_provider.azure.vm.location
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.service.name
+      tag: set_cloud_service_name
+      if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.azure?.vm != null
+      value: vm
+
+  # Map/Convert GCP specific fields to ECS fields
+  - set:
+      field: cloud.account.id
+      tag: set_cloud_account_id_from_asset_cloud_provider_gcp_compute_project_id
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.project_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.availability_zone
+      tag: set_cloud_availability_zone_from_asset_cloud_provider_gcp_compute_zone
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.zone
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.id
+      tag: set_cloud_instance_id_from_asset_cloud_provider_gcp_compute_instance_id
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.instance_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.name
+      tag: set_cloud_instance_name_from_asset_cloud_provider_gcp_compute_hostname
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.hostname
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.machine.type
+      tag: set_cloud_machine_type_from_asset_cloud_provider_gcp_compute_machine_type
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.machine_type
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.project.id
+      tag: set_cloud_project_id_from_asset_cloud_provider_gcp_compute_project_id
+      copy_from: qualys_gav.asset.cloud_provider.gcp.compute.project_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - grok:
+      field: qualys_gav.asset.cloud_provider.gcp.compute.zone
+      tag: grok_cloud_region_from_asset_cloud_provider_gcp_compute_zone
+      patterns:
+        - '%{GREEDYDATA:cloud.region}-%{WORD}$'
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - set:
+      field: cloud.service.name
+      tag: set_cloud_service_name
+      if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'gcp' && ctx.qualys_gav?.asset?.cloud_provider?.gcp?.compute != null
+      value: compute
+
+  # Convert IBM specific fields
+  - convert:
+      field: qualys_gav.asset.cloud_provider.ibm.virtual_server.datacenter_id
+      tag: convert_qualys_gav_asset_cloud_provider_ibm_virtual_server_datacenter_id_to_string
+      type: string
+      ignore_missing: true
   - convert:
       field: qualys_gav.asset.cloud_provider.ibm.virtual_server.ibm_id
       tag: convert_qualys_gav_asset_cloud_provider_ibm_virtual_server_ibm_id_to_string
       type: string
       ignore_missing: true
+
+  # Map/Convert IBM specific fields to ECS fields
+  - set:
+      field: cloud.account.id
+      tag: set_cloud_account_id_from_asset_cloud_provider_ibm_virtual_server_datacenter_id
+      copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.datacenter_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.id
+      tag: set_cloud_instance_id_from_asset_cloud_provider_ibm_virtual_server_ibm_id
+      copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.ibm_id
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.instance.name
+      tag: set_cloud_instance_name_from_asset_cloud_provider_ibm_virtual_server_device_name
+      copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.device_name
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.availability_zone
+      tag: set_cloud_availability_zone_from_asset_cloud_provider_ibm_virtual_server_location
+      copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.location
+      if: ctx._conf?.want_provider_cloud == true
+      ignore_empty_value: true
+  - set:
+      field: cloud.service.name
+      tag: set_cloud_service_name
+      if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.ibm?.virtual_server != null
+      value: virtual_server
+
+  # Map risk scores
   - convert:
       field: qualys_gav.asset.risk_score
       tag: convert_qualys_gav_asset_risk_score_to_float
@@ -1822,6 +2050,11 @@ processors:
       tag: remove_custom_duplicate_fields
       ignore_missing: true
       if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields')
+  - remove:
+      field:
+        - _conf
+      tag: remove_temporary_fields
+      ignore_missing: true
 
   # Cleanup
   - script:

packages/qualys_gav/data_stream/asset/fields/fields.yml

@@ -394,6 +394,18 @@
               type: date
             - name: source
               type: keyword
+        - name: inventory_list_data
+          type: group
+          fields:
+            - name: inventory
+              type: group
+              fields:
+                - name: source
+                  type: keyword
+                - name: created
+                  type: date
+                - name: last_updated
+                  type: date
         - name: is_container_host
           type: boolean
         - name: isp

packages/qualys_gav/data_stream/asset/manifest.yml

@@ -66,6 +66,23 @@ streams:
         required: true
         show_user: false
         default: 30s
+      - name: cloud_data
+        title: Cloud Metadata Source
+        description: What source to use to populate `cloud.*` fields.
+        type: select
+        multi: false
+        required: true
+        show_user: false
+        options:
+          - text: Elastic Only
+            value: elastic_cloud_data
+          - text: Provider Only
+            value: provider_cloud_data
+          - text: Elastic and Provider
+            value: both
+          - text: None
+            value: ''
+        default: both
       - name: preserve_duplicate_custom_fields
         required: false
         title: Preserve duplicate custom fields