diff --git a/.generator/schemas/v1/openapi.yaml b/.generator/schemas/v1/openapi.yaml index c84d61cfe7..416eca6127 100644 --- a/.generator/schemas/v1/openapi.yaml +++ b/.generator/schemas/v1/openapi.yaml @@ -6331,6 +6331,7 @@ components: - $ref: '#/components/schemas/LogsSpanRemapper' - $ref: '#/components/schemas/LogsArrayProcessor' - $ref: '#/components/schemas/LogsDecoderProcessor' + - $ref: '#/components/schemas/LogsSchemaProcessor' LogsQueryCompute: description: Define computation for a log query. properties: @@ -6392,6 +6393,243 @@ components: periods. type: string type: object + LogsSchemaCategoryMapper: + description: "Use the Schema Category Mapper to categorize log event into enum + fields.\nIn the case of OCSF, they can be used to map sibling fields which + are composed of an ID and a name.\n\n**Notes**:\n\n- The syntax of the query + is the one of Logs Explorer search bar.\n The query can be done on any log + attribute or tag, whether it is a facet or not.\n Wildcards can also be used + inside your query.\n- Categories are executed in order and processing stops + at the first match.\n Make sure categories are properly ordered in case a + log could match multiple queries.\n- Sibling fields always have a numerical + ID field and a human-readable string name.\n- A fallback section handles cases + where the name or ID value matches a specific value.\n If the name matches + \"Other\" or the ID matches 99, the value of the sibling name field will be + pulled from a source field from the original log." + properties: + categories: + description: 'Array of filters to match or not a log and their + + corresponding `name` to assign a custom value to the log.' + example: + - filter: + query: '@eventName:(ConsoleLogin OR ExternalIdPDirectoryLogin OR UserAuthentication + OR Authenticate)' + id: 1 + name: Logon + - filter: + query: '@eventName:*' + id: 99 + name: Other + items: + $ref: '#/components/schemas/LogsSchemaCategoryMapperCategory' + type: array + fallback: + $ref: '#/components/schemas/LogsSchemaCategoryMapperFallback' + name: + description: Name of the logs schema category mapper. + example: activity_id and activity_name + type: string + targets: + $ref: '#/components/schemas/LogsSchemaCategoryMapperTargets' + type: + $ref: '#/components/schemas/LogsSchemaCategoryMapperType' + required: + - categories + - targets + - type + - name + type: object + LogsSchemaCategoryMapperCategory: + description: Object describing the logs filter with corresponding category ID + and name assignment. + properties: + filter: + $ref: '#/components/schemas/LogsFilter' + id: + description: ID to inject into the category. + example: 1 + format: int64 + type: integer + name: + description: Value to assign to target schema field. + example: Password Change + type: string + required: + - filter + - id + - name + type: object + LogsSchemaCategoryMapperFallback: + description: Used to override hardcoded category values with a value pulled + from a source attribute on the log. + properties: + sources: + additionalProperties: + items: + type: string + type: array + description: Fallback sources used to populate value of field. + example: {} + type: object + values: + additionalProperties: + type: string + description: Values that define when the fallback is used. + example: {} + type: object + type: object + LogsSchemaCategoryMapperTargets: + description: Name of the target attributes which value is defined by the matching + category. + properties: + id: + description: ID of the field to map log attributes to. + example: ocsf.activity_id + type: string + name: + description: Name of the field to map log attributes to. + example: ocsf.activity_name + type: string + type: object + LogsSchemaCategoryMapperType: + description: Type of logs schema category mapper. + enum: + - schema-category-mapper + example: schema-category-mapper + type: string + x-enum-varnames: + - SCHEMA_CATEGORY_MAPPER + LogsSchemaData: + description: Configuration of the schema data to use. + properties: + class_name: + description: Class name of the schema to use. + example: Account Change + type: string + class_uid: + description: Class UID of the schema to use. + example: 3001 + format: int64 + type: integer + profiles: + description: Optional list of profiles to modify the schema. + example: + - security_control + - host + items: + type: string + type: array + schema_type: + description: Type of schema to use. + example: ocsf + type: string + version: + description: Version of the schema to use. + example: 1.5.0 + type: string + required: + - schema_type + - version + - class_uid + - class_name + type: object + LogsSchemaMapper: + description: Configuration of the schema processor mapper to use. + oneOf: + - $ref: '#/components/schemas/LogsSchemaRemapper' + - $ref: '#/components/schemas/LogsSchemaCategoryMapper' + LogsSchemaProcessor: + description: A processor that has additional validations and checks for a given + schema. Currently supported schema types include OCSF. + properties: + is_enabled: + default: false + description: Whether or not the processor is enabled. + type: boolean + mappers: + description: The `LogsSchemaProcessor` `mappers`. + example: + - name: Map userIdentity to ocsf.user.uid + sources: + - userIdentity.principalId + target: ocsf.user.uid + type: schema-remapper + items: + $ref: '#/components/schemas/LogsSchemaMapper' + type: array + name: + description: Name of the processor. + example: Map additionalEventData.LoginTo to ocsf.dst_endpoint.svc_name + type: string + schema: + $ref: '#/components/schemas/LogsSchemaData' + type: + $ref: '#/components/schemas/LogsSchemaProcessorType' + required: + - name + - mappers + - type + - schema + type: object + LogsSchemaProcessorType: + default: schema-processor + description: Type of logs schema processor. + enum: + - schema-processor + example: schema-processor + type: string + x-enum-varnames: + - SCHEMA_PROCESSOR + LogsSchemaRemapper: + description: The schema remapper maps source log fields to their correct fields. + properties: + name: + description: Name of the logs schema remapper. + example: Map userIdentity.principalId, responseElements.role.roleId, responseElements.user.userId + to ocsf.user.uid + type: string + override_on_conflict: + default: false + description: Override or not the target element if already set. + type: boolean + preserve_source: + default: false + description: Remove or preserve the remapped source element. + type: boolean + sources: + description: Array of source attributes. + example: + - userIdentity.principalId + - responseElements.role.roleId + - responseElements.user.userId + items: + description: Attribute used as a source to remap its value to the target + attribute. + type: string + type: array + target: + description: Target field to map log source field to. + example: ocsf.user.uid + type: string + target_format: + $ref: '#/components/schemas/TargetFormatType' + type: + $ref: '#/components/schemas/LogsSchemaRemapperType' + required: + - name + - sources + - target + - type + type: object + LogsSchemaRemapperType: + description: Type of logs schema remapper. + enum: + - schema-remapper + example: schema-remapper + type: string + x-enum-varnames: + - SCHEMA_REMAPPER LogsServiceRemapper: description: 'Use this processor if you want to assign one or more attributes as the official service. diff --git a/docs/datadog_api_client.v1.model.rst b/docs/datadog_api_client.v1.model.rst index fd5caf6e30..e2f9fc1e6d 100644 --- a/docs/datadog_api_client.v1.model.rst +++ b/docs/datadog_api_client.v1.model.rst @@ -1964,6 +1964,83 @@ datadog\_api\_client.v1.model.logs\_retention\_sum\_usage module :members: :show-inheritance: +datadog\_api\_client.v1.model.logs\_schema\_category\_mapper module +------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_category_mapper + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_category\_mapper\_category module +----------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_category_mapper_category + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_category\_mapper\_fallback module +----------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_category_mapper_fallback + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_category\_mapper\_targets module +---------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_category_mapper_targets + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_category\_mapper\_type module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_category_mapper_type + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_data module +------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_data + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_mapper module +--------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_mapper + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_processor module +------------------------------------------------------------ + +.. automodule:: datadog_api_client.v1.model.logs_schema_processor + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_processor\_type module +------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v1.model.logs_schema_processor_type + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_remapper module +----------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_remapper + :members: + :show-inheritance: + +datadog\_api\_client.v1.model.logs\_schema\_remapper\_type module +----------------------------------------------------------------- + +.. automodule:: datadog_api_client.v1.model.logs_schema_remapper_type + :members: + :show-inheritance: + datadog\_api\_client.v1.model.logs\_service\_remapper module ------------------------------------------------------------ diff --git a/examples/v1/logs-pipelines/CreateLogsPipeline_1745625064.py b/examples/v1/logs-pipelines/CreateLogsPipeline_1745625064.py new file mode 100644 index 0000000000..1a2a6dfb00 --- /dev/null +++ b/examples/v1/logs-pipelines/CreateLogsPipeline_1745625064.py @@ -0,0 +1,269 @@ +""" +Create a pipeline with Schema Processor and preserve_source true returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v1.api.logs_pipelines_api import LogsPipelinesApi +from datadog_api_client.v1.model.logs_filter import LogsFilter +from datadog_api_client.v1.model.logs_pipeline import LogsPipeline +from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper +from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory +from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback +from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets +from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType +from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData +from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor +from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType +from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper +from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType + +body = LogsPipeline( + filter=LogsFilter( + query="source:python", + ), + name="testSchemaProcessor", + processors=[ + LogsSchemaProcessor( + type=LogsSchemaProcessorType.SCHEMA_PROCESSOR, + is_enabled=True, + name="Apply OCSF schema for 3001", + schema=LogsSchemaData( + schema_type="ocsf", + version="1.5.0", + class_uid=3001, + class_name="Account Change", + profiles=[ + "cloud", + "datetime", + ], + ), + mappers=[ + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="activity_id and activity_name", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Create*)", + ), + name="Create", + id=1, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(ChangePassword OR PasswordUpdated)", + ), + name="Password Change", + id=3, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Attach*)", + ), + name="Attach Policy", + id=7, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Detach* OR *Remove*)", + ), + name="Detach Policy", + id=8, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Delete*)", + ), + name="Delete", + id=6, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Other", + id=99, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.activity_name", + id="ocsf.activity_id", + ), + fallback=LogsSchemaCategoryMapperFallback( + values={ + "ocsf.activity_id": "99", + "ocsf.activity_name": "Other", + }, + sources={ + "ocsf.activity_name": [ + "eventName", + ], + }, + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="status", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="-@errorCode:*", + ), + id=1, + name="Success", + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@errorCode:*", + ), + id=2, + name="Failure", + ), + ], + targets=LogsSchemaCategoryMapperTargets( + id="ocsf.status_id", + name="ocsf.status", + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="Set default severity", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Informational", + id=1, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.severity", + id="ocsf.severity_id", + ), + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userIdentity to ocsf.user.uid", + sources=[ + "userIdentity.principalId", + "responseElements.role.roleId", + "responseElements.user.userId", + ], + target="ocsf.user.uid", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userName to ocsf.user.name", + sources=[ + "requestParameters.userName", + "responseElements.role.roleName", + "requestParameters.roleName", + "responseElements.user.userName", + ], + target="ocsf.user.name", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map api to ocsf.api", + sources=[ + "api", + ], + target="ocsf.api", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map user to ocsf.user", + sources=[ + "user", + ], + target="ocsf.user", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map actor to ocsf.actor", + sources=[ + "actor", + ], + target="ocsf.actor", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map cloud to ocsf.cloud", + sources=[ + "cloud", + ], + target="ocsf.cloud", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map http_request to ocsf.http_request", + sources=[ + "http_request", + ], + target="ocsf.http_request", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map metadata to ocsf.metadata", + sources=[ + "metadata", + ], + target="ocsf.metadata", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map time to ocsf.time", + sources=[ + "time", + ], + target="ocsf.time", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map src_endpoint to ocsf.src_endpoint", + sources=[ + "src_endpoint", + ], + target="ocsf.src_endpoint", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity to ocsf.severity", + sources=[ + "severity", + ], + target="ocsf.severity", + preserve_source=True, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity_id to ocsf.severity_id", + sources=[ + "severity_id", + ], + target="ocsf.severity_id", + preserve_source=True, + ), + ], + ), + ], + tags=[], +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = LogsPipelinesApi(api_client) + response = api_instance.create_logs_pipeline(body=body) + + print(response) diff --git a/examples/v1/logs-pipelines/CreateLogsPipeline_2256674867.py b/examples/v1/logs-pipelines/CreateLogsPipeline_2256674867.py new file mode 100644 index 0000000000..22d14eb842 --- /dev/null +++ b/examples/v1/logs-pipelines/CreateLogsPipeline_2256674867.py @@ -0,0 +1,269 @@ +""" +Create a pipeline with Schema Processor and preserve_source false returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v1.api.logs_pipelines_api import LogsPipelinesApi +from datadog_api_client.v1.model.logs_filter import LogsFilter +from datadog_api_client.v1.model.logs_pipeline import LogsPipeline +from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper +from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory +from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback +from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets +from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType +from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData +from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor +from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType +from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper +from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType + +body = LogsPipeline( + filter=LogsFilter( + query="source:python", + ), + name="testSchemaProcessor", + processors=[ + LogsSchemaProcessor( + type=LogsSchemaProcessorType.SCHEMA_PROCESSOR, + is_enabled=True, + name="Apply OCSF schema for 3001", + schema=LogsSchemaData( + schema_type="ocsf", + version="1.5.0", + class_uid=3001, + class_name="Account Change", + profiles=[ + "cloud", + "datetime", + ], + ), + mappers=[ + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="activity_id and activity_name", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Create*)", + ), + name="Create", + id=1, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(ChangePassword OR PasswordUpdated)", + ), + name="Password Change", + id=3, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Attach*)", + ), + name="Attach Policy", + id=7, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Detach* OR *Remove*)", + ), + name="Detach Policy", + id=8, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Delete*)", + ), + name="Delete", + id=6, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Other", + id=99, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.activity_name", + id="ocsf.activity_id", + ), + fallback=LogsSchemaCategoryMapperFallback( + values={ + "ocsf.activity_id": "99", + "ocsf.activity_name": "Other", + }, + sources={ + "ocsf.activity_name": [ + "eventName", + ], + }, + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="status", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="-@errorCode:*", + ), + id=1, + name="Success", + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@errorCode:*", + ), + id=2, + name="Failure", + ), + ], + targets=LogsSchemaCategoryMapperTargets( + id="ocsf.status_id", + name="ocsf.status", + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="Set default severity", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Informational", + id=1, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.severity", + id="ocsf.severity_id", + ), + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userIdentity to ocsf.user.uid", + sources=[ + "userIdentity.principalId", + "responseElements.role.roleId", + "responseElements.user.userId", + ], + target="ocsf.user.uid", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userName to ocsf.user.name", + sources=[ + "requestParameters.userName", + "responseElements.role.roleName", + "requestParameters.roleName", + "responseElements.user.userName", + ], + target="ocsf.user.name", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map api to ocsf.api", + sources=[ + "api", + ], + target="ocsf.api", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map user to ocsf.user", + sources=[ + "user", + ], + target="ocsf.user", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map actor to ocsf.actor", + sources=[ + "actor", + ], + target="ocsf.actor", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map cloud to ocsf.cloud", + sources=[ + "cloud", + ], + target="ocsf.cloud", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map http_request to ocsf.http_request", + sources=[ + "http_request", + ], + target="ocsf.http_request", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map metadata to ocsf.metadata", + sources=[ + "metadata", + ], + target="ocsf.metadata", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map time to ocsf.time", + sources=[ + "time", + ], + target="ocsf.time", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map src_endpoint to ocsf.src_endpoint", + sources=[ + "src_endpoint", + ], + target="ocsf.src_endpoint", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity to ocsf.severity", + sources=[ + "severity", + ], + target="ocsf.severity", + preserve_source=False, + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity_id to ocsf.severity_id", + sources=[ + "severity_id", + ], + target="ocsf.severity_id", + preserve_source=False, + ), + ], + ), + ], + tags=[], +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = LogsPipelinesApi(api_client) + response = api_instance.create_logs_pipeline(body=body) + + print(response) diff --git a/examples/v1/logs-pipelines/CreateLogsPipeline_501419705.py b/examples/v1/logs-pipelines/CreateLogsPipeline_501419705.py new file mode 100644 index 0000000000..be396064b1 --- /dev/null +++ b/examples/v1/logs-pipelines/CreateLogsPipeline_501419705.py @@ -0,0 +1,257 @@ +""" +Create a pipeline with schema processor +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v1.api.logs_pipelines_api import LogsPipelinesApi +from datadog_api_client.v1.model.logs_filter import LogsFilter +from datadog_api_client.v1.model.logs_pipeline import LogsPipeline +from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper +from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory +from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback +from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets +from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType +from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData +from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor +from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType +from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper +from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType + +body = LogsPipeline( + filter=LogsFilter( + query="source:python", + ), + name="testSchemaProcessor", + processors=[ + LogsSchemaProcessor( + type=LogsSchemaProcessorType.SCHEMA_PROCESSOR, + is_enabled=True, + name="Apply OCSF schema for 3001", + schema=LogsSchemaData( + schema_type="ocsf", + version="1.5.0", + class_uid=3001, + class_name="Account Change", + profiles=[ + "cloud", + "datetime", + ], + ), + mappers=[ + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="activity_id and activity_name", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Create*)", + ), + name="Create", + id=1, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(ChangePassword OR PasswordUpdated)", + ), + name="Password Change", + id=3, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Attach*)", + ), + name="Attach Policy", + id=7, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Detach* OR *Remove*)", + ), + name="Detach Policy", + id=8, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:(*Delete*)", + ), + name="Delete", + id=6, + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Other", + id=99, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.activity_name", + id="ocsf.activity_id", + ), + fallback=LogsSchemaCategoryMapperFallback( + values={ + "ocsf.activity_id": "99", + "ocsf.activity_name": "Other", + }, + sources={ + "ocsf.activity_name": [ + "eventName", + ], + }, + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="status", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="-@errorCode:*", + ), + id=1, + name="Success", + ), + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@errorCode:*", + ), + id=2, + name="Failure", + ), + ], + targets=LogsSchemaCategoryMapperTargets( + id="ocsf.status_id", + name="ocsf.status", + ), + ), + LogsSchemaCategoryMapper( + type=LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER, + name="Set default severity", + categories=[ + LogsSchemaCategoryMapperCategory( + filter=LogsFilter( + query="@eventName:*", + ), + name="Informational", + id=1, + ), + ], + targets=LogsSchemaCategoryMapperTargets( + name="ocsf.severity", + id="ocsf.severity_id", + ), + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userIdentity to ocsf.user.uid", + sources=[ + "userIdentity.principalId", + "responseElements.role.roleId", + "responseElements.user.userId", + ], + target="ocsf.user.uid", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map userName to ocsf.user.name", + sources=[ + "requestParameters.userName", + "responseElements.role.roleName", + "requestParameters.roleName", + "responseElements.user.userName", + ], + target="ocsf.user.name", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map api to ocsf.api", + sources=[ + "api", + ], + target="ocsf.api", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map user to ocsf.user", + sources=[ + "user", + ], + target="ocsf.user", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map actor to ocsf.actor", + sources=[ + "actor", + ], + target="ocsf.actor", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map cloud to ocsf.cloud", + sources=[ + "cloud", + ], + target="ocsf.cloud", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map http_request to ocsf.http_request", + sources=[ + "http_request", + ], + target="ocsf.http_request", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map metadata to ocsf.metadata", + sources=[ + "metadata", + ], + target="ocsf.metadata", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map time to ocsf.time", + sources=[ + "time", + ], + target="ocsf.time", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map src_endpoint to ocsf.src_endpoint", + sources=[ + "src_endpoint", + ], + target="ocsf.src_endpoint", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity to ocsf.severity", + sources=[ + "severity", + ], + target="ocsf.severity", + ), + LogsSchemaRemapper( + type=LogsSchemaRemapperType.SCHEMA_REMAPPER, + name="Map severity_id to ocsf.severity_id", + sources=[ + "severity_id", + ], + target="ocsf.severity_id", + ), + ], + ), + ], + tags=[], +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = LogsPipelinesApi(api_client) + response = api_instance.create_logs_pipeline(body=body) + + print(response) diff --git a/src/datadog_api_client/v1/model/logs_pipeline.py b/src/datadog_api_client/v1/model/logs_pipeline.py index 8485c7e2eb..c7dc54b78b 100644 --- a/src/datadog_api_client/v1/model/logs_pipeline.py +++ b/src/datadog_api_client/v1/model/logs_pipeline.py @@ -35,6 +35,7 @@ from datadog_api_client.v1.model.logs_span_remapper import LogsSpanRemapper from datadog_api_client.v1.model.logs_array_processor import LogsArrayProcessor from datadog_api_client.v1.model.logs_decoder_processor import LogsDecoderProcessor + from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor class LogsPipeline(ModelNormal): @@ -103,6 +104,7 @@ def __init__( LogsSpanRemapper, LogsArrayProcessor, LogsDecoderProcessor, + LogsSchemaProcessor, ] ], UnsetType, diff --git a/src/datadog_api_client/v1/model/logs_pipeline_processor.py b/src/datadog_api_client/v1/model/logs_pipeline_processor.py index c66f8690dd..bf74ba2c2b 100644 --- a/src/datadog_api_client/v1/model/logs_pipeline_processor.py +++ b/src/datadog_api_client/v1/model/logs_pipeline_processor.py @@ -35,6 +35,7 @@ from datadog_api_client.v1.model.logs_span_remapper import LogsSpanRemapper from datadog_api_client.v1.model.logs_array_processor import LogsArrayProcessor from datadog_api_client.v1.model.logs_decoder_processor import LogsDecoderProcessor + from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor class LogsPipelineProcessor(ModelNormal): @@ -89,6 +90,7 @@ def __init__( LogsSpanRemapper, LogsArrayProcessor, LogsDecoderProcessor, + LogsSchemaProcessor, ] ], UnsetType, diff --git a/src/datadog_api_client/v1/model/logs_processor.py b/src/datadog_api_client/v1/model/logs_processor.py index 6baebc4f66..0ad187e001 100644 --- a/src/datadog_api_client/v1/model/logs_processor.py +++ b/src/datadog_api_client/v1/model/logs_processor.py @@ -100,6 +100,12 @@ def __init__(self, **kwargs): :param input_representation: The original representation of input string. :type input_representation: LogsDecoderProcessorInputRepresentation + + :param mappers: The `LogsSchemaProcessor` `mappers`. + :type mappers: [LogsSchemaMapper] + + :param schema: Configuration of the schema data to use. + :type schema: LogsSchemaData """ super().__init__(kwargs) @@ -131,6 +137,7 @@ def _composed_schemas(_): from datadog_api_client.v1.model.logs_span_remapper import LogsSpanRemapper from datadog_api_client.v1.model.logs_array_processor import LogsArrayProcessor from datadog_api_client.v1.model.logs_decoder_processor import LogsDecoderProcessor + from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor return { "oneOf": [ @@ -153,5 +160,6 @@ def _composed_schemas(_): LogsSpanRemapper, LogsArrayProcessor, LogsDecoderProcessor, + LogsSchemaProcessor, ], } diff --git a/src/datadog_api_client/v1/model/logs_schema_category_mapper.py b/src/datadog_api_client/v1/model/logs_schema_category_mapper.py new file mode 100644 index 0000000000..75cbb0719f --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_category_mapper.py @@ -0,0 +1,94 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory + from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback + from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets + from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType + + +class LogsSchemaCategoryMapper(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory + from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback + from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets + from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType + + return { + "categories": ([LogsSchemaCategoryMapperCategory],), + "fallback": (LogsSchemaCategoryMapperFallback,), + "name": (str,), + "targets": (LogsSchemaCategoryMapperTargets,), + "type": (LogsSchemaCategoryMapperType,), + } + + attribute_map = { + "categories": "categories", + "fallback": "fallback", + "name": "name", + "targets": "targets", + "type": "type", + } + + def __init__( + self_, + categories: List[LogsSchemaCategoryMapperCategory], + name: str, + targets: LogsSchemaCategoryMapperTargets, + type: LogsSchemaCategoryMapperType, + fallback: Union[LogsSchemaCategoryMapperFallback, UnsetType] = unset, + **kwargs, + ): + """ + Use the Schema Category Mapper to categorize log event into enum fields. + In the case of OCSF, they can be used to map sibling fields which are composed of an ID and a name. + + **Notes** : + + * The syntax of the query is the one of Logs Explorer search bar. + The query can be done on any log attribute or tag, whether it is a facet or not. + Wildcards can also be used inside your query. + * Categories are executed in order and processing stops at the first match. + Make sure categories are properly ordered in case a log could match multiple queries. + * Sibling fields always have a numerical ID field and a human-readable string name. + * A fallback section handles cases where the name or ID value matches a specific value. + If the name matches "Other" or the ID matches 99, the value of the sibling name field will be pulled from a source field from the original log. + + :param categories: Array of filters to match or not a log and their + corresponding ``name`` to assign a custom value to the log. + :type categories: [LogsSchemaCategoryMapperCategory] + + :param fallback: Used to override hardcoded category values with a value pulled from a source attribute on the log. + :type fallback: LogsSchemaCategoryMapperFallback, optional + + :param name: Name of the logs schema category mapper. + :type name: str + + :param targets: Name of the target attributes which value is defined by the matching category. + :type targets: LogsSchemaCategoryMapperTargets + + :param type: Type of logs schema category mapper. + :type type: LogsSchemaCategoryMapperType + """ + if fallback is not unset: + kwargs["fallback"] = fallback + super().__init__(kwargs) + + self_.categories = categories + self_.name = name + self_.targets = targets + self_.type = type diff --git a/src/datadog_api_client/v1/model/logs_schema_category_mapper_category.py b/src/datadog_api_client/v1/model/logs_schema_category_mapper_category.py new file mode 100644 index 0000000000..a26b612eee --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_category_mapper_category.py @@ -0,0 +1,52 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v1.model.logs_filter import LogsFilter + + +class LogsSchemaCategoryMapperCategory(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v1.model.logs_filter import LogsFilter + + return { + "filter": (LogsFilter,), + "id": (int,), + "name": (str,), + } + + attribute_map = { + "filter": "filter", + "id": "id", + "name": "name", + } + + def __init__(self_, filter: LogsFilter, id: int, name: str, **kwargs): + """ + Object describing the logs filter with corresponding category ID and name assignment. + + :param filter: Filter for logs. + :type filter: LogsFilter + + :param id: ID to inject into the category. + :type id: int + + :param name: Value to assign to target schema field. + :type name: str + """ + super().__init__(kwargs) + + self_.filter = filter + self_.id = id + self_.name = name diff --git a/src/datadog_api_client/v1/model/logs_schema_category_mapper_fallback.py b/src/datadog_api_client/v1/model/logs_schema_category_mapper_fallback.py new file mode 100644 index 0000000000..b9f09c8d0c --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_category_mapper_fallback.py @@ -0,0 +1,48 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Dict, List, Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LogsSchemaCategoryMapperFallback(ModelNormal): + @cached_property + def openapi_types(_): + return { + "sources": ({str: ([str],)},), + "values": ({str: (str,)},), + } + + attribute_map = { + "sources": "sources", + "values": "values", + } + + def __init__( + self_, + sources: Union[Dict[str, List[str]], UnsetType] = unset, + values: Union[Dict[str, str], UnsetType] = unset, + **kwargs, + ): + """ + Used to override hardcoded category values with a value pulled from a source attribute on the log. + + :param sources: Fallback sources used to populate value of field. + :type sources: {str: ([str],)}, optional + + :param values: Values that define when the fallback is used. + :type values: {str: (str,)}, optional + """ + if sources is not unset: + kwargs["sources"] = sources + if values is not unset: + kwargs["values"] = values + super().__init__(kwargs) diff --git a/src/datadog_api_client/v1/model/logs_schema_category_mapper_targets.py b/src/datadog_api_client/v1/model/logs_schema_category_mapper_targets.py new file mode 100644 index 0000000000..66486e59e2 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_category_mapper_targets.py @@ -0,0 +1,43 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LogsSchemaCategoryMapperTargets(ModelNormal): + @cached_property + def openapi_types(_): + return { + "id": (str,), + "name": (str,), + } + + attribute_map = { + "id": "id", + "name": "name", + } + + def __init__(self_, id: Union[str, UnsetType] = unset, name: Union[str, UnsetType] = unset, **kwargs): + """ + Name of the target attributes which value is defined by the matching category. + + :param id: ID of the field to map log attributes to. + :type id: str, optional + + :param name: Name of the field to map log attributes to. + :type name: str, optional + """ + if id is not unset: + kwargs["id"] = id + if name is not unset: + kwargs["name"] = name + super().__init__(kwargs) diff --git a/src/datadog_api_client/v1/model/logs_schema_category_mapper_type.py b/src/datadog_api_client/v1/model/logs_schema_category_mapper_type.py new file mode 100644 index 0000000000..de11446c3b --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_category_mapper_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LogsSchemaCategoryMapperType(ModelSimple): + """ + Type of logs schema category mapper. + + :param value: If omitted defaults to "schema-category-mapper". Must be one of ["schema-category-mapper"]. + :type value: str + """ + + allowed_values = { + "schema-category-mapper", + } + SCHEMA_CATEGORY_MAPPER: ClassVar["LogsSchemaCategoryMapperType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LogsSchemaCategoryMapperType.SCHEMA_CATEGORY_MAPPER = LogsSchemaCategoryMapperType("schema-category-mapper") diff --git a/src/datadog_api_client/v1/model/logs_schema_data.py b/src/datadog_api_client/v1/model/logs_schema_data.py new file mode 100644 index 0000000000..9999c00675 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_data.py @@ -0,0 +1,69 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LogsSchemaData(ModelNormal): + @cached_property + def openapi_types(_): + return { + "class_name": (str,), + "class_uid": (int,), + "profiles": ([str],), + "schema_type": (str,), + "version": (str,), + } + + attribute_map = { + "class_name": "class_name", + "class_uid": "class_uid", + "profiles": "profiles", + "schema_type": "schema_type", + "version": "version", + } + + def __init__( + self_, + class_name: str, + class_uid: int, + schema_type: str, + version: str, + profiles: Union[List[str], UnsetType] = unset, + **kwargs, + ): + """ + Configuration of the schema data to use. + + :param class_name: Class name of the schema to use. + :type class_name: str + + :param class_uid: Class UID of the schema to use. + :type class_uid: int + + :param profiles: Optional list of profiles to modify the schema. + :type profiles: [str], optional + + :param schema_type: Type of schema to use. + :type schema_type: str + + :param version: Version of the schema to use. + :type version: str + """ + if profiles is not unset: + kwargs["profiles"] = profiles + super().__init__(kwargs) + + self_.class_name = class_name + self_.class_uid = class_uid + self_.schema_type = schema_type + self_.version = version diff --git a/src/datadog_api_client/v1/model/logs_schema_mapper.py b/src/datadog_api_client/v1/model/logs_schema_mapper.py new file mode 100644 index 0000000000..93397a0cb8 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_mapper.py @@ -0,0 +1,70 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelComposed, + cached_property, +) + + +class LogsSchemaMapper(ModelComposed): + def __init__(self, **kwargs): + """ + Configuration of the schema processor mapper to use. + + :param name: Name of the logs schema remapper. + :type name: str + + :param override_on_conflict: Override or not the target element if already set. + :type override_on_conflict: bool, optional + + :param preserve_source: Remove or preserve the remapped source element. + :type preserve_source: bool, optional + + :param sources: Array of source attributes. + :type sources: [str] + + :param target: Target field to map log source field to. + :type target: str + + :param target_format: If the `target_type` of the remapper is `attribute`, try to cast the value to a new specific type. + If the cast is not possible, the original type is kept. `string`, `integer`, or `double` are the possible types. + If the `target_type` is `tag`, this parameter may not be specified. + :type target_format: TargetFormatType, optional + + :param type: Type of logs schema remapper. + :type type: LogsSchemaRemapperType + + :param categories: Array of filters to match or not a log and their + corresponding `name` to assign a custom value to the log. + :type categories: [LogsSchemaCategoryMapperCategory] + + :param fallback: Used to override hardcoded category values with a value pulled from a source attribute on the log. + :type fallback: LogsSchemaCategoryMapperFallback, optional + + :param targets: Name of the target attributes which value is defined by the matching category. + :type targets: LogsSchemaCategoryMapperTargets + """ + super().__init__(kwargs) + + @cached_property + def _composed_schemas(_): + # we need this here to make our import statements work + # we must store _composed_schemas in here so the code is only run + # when we invoke this method. If we kept this at the class + # level we would get an error because the class level + # code would be run when this module is imported, and these composed + # classes don't exist yet because their module has not finished + # loading + from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper + from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper + + return { + "oneOf": [ + LogsSchemaRemapper, + LogsSchemaCategoryMapper, + ], + } diff --git a/src/datadog_api_client/v1/model/logs_schema_processor.py b/src/datadog_api_client/v1/model/logs_schema_processor.py new file mode 100644 index 0000000000..6798749ce8 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_processor.py @@ -0,0 +1,81 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v1.model.logs_schema_mapper import LogsSchemaMapper + from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData + from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType + from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper + from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper + + +class LogsSchemaProcessor(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v1.model.logs_schema_mapper import LogsSchemaMapper + from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData + from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType + + return { + "is_enabled": (bool,), + "mappers": ([LogsSchemaMapper],), + "name": (str,), + "schema": (LogsSchemaData,), + "type": (LogsSchemaProcessorType,), + } + + attribute_map = { + "is_enabled": "is_enabled", + "mappers": "mappers", + "name": "name", + "schema": "schema", + "type": "type", + } + + def __init__( + self_, + mappers: List[Union[LogsSchemaMapper, LogsSchemaRemapper, LogsSchemaCategoryMapper]], + name: str, + schema: LogsSchemaData, + type: LogsSchemaProcessorType, + is_enabled: Union[bool, UnsetType] = unset, + **kwargs, + ): + """ + A processor that has additional validations and checks for a given schema. Currently supported schema types include OCSF. + + :param is_enabled: Whether or not the processor is enabled. + :type is_enabled: bool, optional + + :param mappers: The ``LogsSchemaProcessor`` ``mappers``. + :type mappers: [LogsSchemaMapper] + + :param name: Name of the processor. + :type name: str + + :param schema: Configuration of the schema data to use. + :type schema: LogsSchemaData + + :param type: Type of logs schema processor. + :type type: LogsSchemaProcessorType + """ + if is_enabled is not unset: + kwargs["is_enabled"] = is_enabled + super().__init__(kwargs) + + self_.mappers = mappers + self_.name = name + self_.schema = schema + self_.type = type diff --git a/src/datadog_api_client/v1/model/logs_schema_processor_type.py b/src/datadog_api_client/v1/model/logs_schema_processor_type.py new file mode 100644 index 0000000000..a17439d814 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_processor_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LogsSchemaProcessorType(ModelSimple): + """ + Type of logs schema processor. + + :param value: If omitted defaults to "schema-processor". Must be one of ["schema-processor"]. + :type value: str + """ + + allowed_values = { + "schema-processor", + } + SCHEMA_PROCESSOR: ClassVar["LogsSchemaProcessorType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LogsSchemaProcessorType.SCHEMA_PROCESSOR = LogsSchemaProcessorType("schema-processor") diff --git a/src/datadog_api_client/v1/model/logs_schema_remapper.py b/src/datadog_api_client/v1/model/logs_schema_remapper.py new file mode 100644 index 0000000000..d80adfa324 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_remapper.py @@ -0,0 +1,95 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v1.model.target_format_type import TargetFormatType + from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType + + +class LogsSchemaRemapper(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v1.model.target_format_type import TargetFormatType + from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType + + return { + "name": (str,), + "override_on_conflict": (bool,), + "preserve_source": (bool,), + "sources": ([str],), + "target": (str,), + "target_format": (TargetFormatType,), + "type": (LogsSchemaRemapperType,), + } + + attribute_map = { + "name": "name", + "override_on_conflict": "override_on_conflict", + "preserve_source": "preserve_source", + "sources": "sources", + "target": "target", + "target_format": "target_format", + "type": "type", + } + + def __init__( + self_, + name: str, + sources: List[str], + target: str, + type: LogsSchemaRemapperType, + override_on_conflict: Union[bool, UnsetType] = unset, + preserve_source: Union[bool, UnsetType] = unset, + target_format: Union[TargetFormatType, UnsetType] = unset, + **kwargs, + ): + """ + The schema remapper maps source log fields to their correct fields. + + :param name: Name of the logs schema remapper. + :type name: str + + :param override_on_conflict: Override or not the target element if already set. + :type override_on_conflict: bool, optional + + :param preserve_source: Remove or preserve the remapped source element. + :type preserve_source: bool, optional + + :param sources: Array of source attributes. + :type sources: [str] + + :param target: Target field to map log source field to. + :type target: str + + :param target_format: If the ``target_type`` of the remapper is ``attribute`` , try to cast the value to a new specific type. + If the cast is not possible, the original type is kept. ``string`` , ``integer`` , or ``double`` are the possible types. + If the ``target_type`` is ``tag`` , this parameter may not be specified. + :type target_format: TargetFormatType, optional + + :param type: Type of logs schema remapper. + :type type: LogsSchemaRemapperType + """ + if override_on_conflict is not unset: + kwargs["override_on_conflict"] = override_on_conflict + if preserve_source is not unset: + kwargs["preserve_source"] = preserve_source + if target_format is not unset: + kwargs["target_format"] = target_format + super().__init__(kwargs) + + self_.name = name + self_.sources = sources + self_.target = target + self_.type = type diff --git a/src/datadog_api_client/v1/model/logs_schema_remapper_type.py b/src/datadog_api_client/v1/model/logs_schema_remapper_type.py new file mode 100644 index 0000000000..5cf16b7da1 --- /dev/null +++ b/src/datadog_api_client/v1/model/logs_schema_remapper_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LogsSchemaRemapperType(ModelSimple): + """ + Type of logs schema remapper. + + :param value: If omitted defaults to "schema-remapper". Must be one of ["schema-remapper"]. + :type value: str + """ + + allowed_values = { + "schema-remapper", + } + SCHEMA_REMAPPER: ClassVar["LogsSchemaRemapperType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LogsSchemaRemapperType.SCHEMA_REMAPPER = LogsSchemaRemapperType("schema-remapper") diff --git a/src/datadog_api_client/v1/models/__init__.py b/src/datadog_api_client/v1/models/__init__.py index 3b10b8cfa8..001e6874d8 100644 --- a/src/datadog_api_client/v1/models/__init__.py +++ b/src/datadog_api_client/v1/models/__init__.py @@ -316,6 +316,17 @@ from datadog_api_client.v1.model.logs_query_compute import LogsQueryCompute from datadog_api_client.v1.model.logs_retention_agg_sum_usage import LogsRetentionAggSumUsage from datadog_api_client.v1.model.logs_retention_sum_usage import LogsRetentionSumUsage +from datadog_api_client.v1.model.logs_schema_category_mapper import LogsSchemaCategoryMapper +from datadog_api_client.v1.model.logs_schema_category_mapper_category import LogsSchemaCategoryMapperCategory +from datadog_api_client.v1.model.logs_schema_category_mapper_fallback import LogsSchemaCategoryMapperFallback +from datadog_api_client.v1.model.logs_schema_category_mapper_targets import LogsSchemaCategoryMapperTargets +from datadog_api_client.v1.model.logs_schema_category_mapper_type import LogsSchemaCategoryMapperType +from datadog_api_client.v1.model.logs_schema_data import LogsSchemaData +from datadog_api_client.v1.model.logs_schema_mapper import LogsSchemaMapper +from datadog_api_client.v1.model.logs_schema_processor import LogsSchemaProcessor +from datadog_api_client.v1.model.logs_schema_processor_type import LogsSchemaProcessorType +from datadog_api_client.v1.model.logs_schema_remapper import LogsSchemaRemapper +from datadog_api_client.v1.model.logs_schema_remapper_type import LogsSchemaRemapperType from datadog_api_client.v1.model.logs_service_remapper import LogsServiceRemapper from datadog_api_client.v1.model.logs_service_remapper_type import LogsServiceRemapperType from datadog_api_client.v1.model.logs_sort import LogsSort @@ -1387,6 +1398,17 @@ "LogsQueryCompute", "LogsRetentionAggSumUsage", "LogsRetentionSumUsage", + "LogsSchemaCategoryMapper", + "LogsSchemaCategoryMapperCategory", + "LogsSchemaCategoryMapperFallback", + "LogsSchemaCategoryMapperTargets", + "LogsSchemaCategoryMapperType", + "LogsSchemaData", + "LogsSchemaMapper", + "LogsSchemaProcessor", + "LogsSchemaProcessorType", + "LogsSchemaRemapper", + "LogsSchemaRemapperType", "LogsServiceRemapper", "LogsServiceRemapperType", "LogsSort", diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.frozen b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.frozen new file mode 100644 index 0000000000..ac0d5b77e9 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.frozen @@ -0,0 +1 @@ +2025-10-22T19:12:00.030Z \ No newline at end of file diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.yaml b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.yaml new file mode 100644 index 0000000000..76c288e5b1 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor.yaml @@ -0,0 +1,77 @@ +interactions: +- request: + body: '{"filter":{"query":"source:python"},"name":"testSchemaProcessor","processors":[{"is_enabled":true,"mappers":[{"categories":[{"filter":{"query":"@eventName:(*Create*)"},"id":1,"name":"Create"},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"id":3,"name":"Password Change"},{"filter":{"query":"@eventName:(*Attach*)"},"id":7,"name":"Attach + Policy"},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"id":8,"name":"Detach + Policy"},{"filter":{"query":"@eventName:(*Delete*)"},"id":6,"name":"Delete"},{"filter":{"query":"@eventName:*"},"id":99,"name":"Other"}],"fallback":{"sources":{"ocsf.activity_name":["eventName"]},"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"}},"name":"activity_id + and activity_name","targets":{"id":"ocsf.activity_id","name":"ocsf.activity_name"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"-@errorCode:*"},"id":1,"name":"Success"},{"filter":{"query":"@errorCode:*"},"id":2,"name":"Failure"}],"name":"status","targets":{"id":"ocsf.status_id","name":"ocsf.status"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"@eventName:*"},"id":1,"name":"Informational"}],"name":"Set + default severity","targets":{"id":"ocsf.severity_id","name":"ocsf.severity"},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","type":"schema-remapper"},{"name":"Map + api to ocsf.api","sources":["api"],"target":"ocsf.api","type":"schema-remapper"},{"name":"Map + user to ocsf.user","sources":["user"],"target":"ocsf.user","type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","sources":["actor"],"target":"ocsf.actor","type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","sources":["cloud"],"target":"ocsf.cloud","type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","sources":["http_request"],"target":"ocsf.http_request","type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","sources":["metadata"],"target":"ocsf.metadata","type":"schema-remapper"},{"name":"Map + time to ocsf.time","sources":["time"],"target":"ocsf.time","type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","sources":["src_endpoint"],"target":"ocsf.src_endpoint","type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","sources":["severity"],"target":"ocsf.severity","type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","sources":["severity_id"],"target":"ocsf.severity_id","type":"schema-remapper"}],"name":"Apply + OCSF schema for 3001","schema":{"class_name":"Account Change","class_uid":3001,"profiles":["cloud","datetime"],"schema_type":"ocsf","version":"1.5.0"},"type":"schema-processor"}],"tags":[]}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines + response: + body: + string: '{"id":"1unf0vMNQKSSwzsg6BuWMw","type":"pipeline","name":"testSchemaProcessor","is_enabled":false,"is_read_only":false,"filter":{"query":"source:python"},"processors":[{"name":"Apply + OCSF schema for 3001","is_enabled":true,"mappers":[{"name":"activity_id and + activity_name","categories":[{"filter":{"query":"@eventName:(*Create*)"},"name":"Create","id":1},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"name":"Password Change","id":3},{"filter":{"query":"@eventName:(*Attach*)"},"name":"Attach + Policy","id":7},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"name":"Detach + Policy","id":8},{"filter":{"query":"@eventName:(*Delete*)"},"name":"Delete","id":6},{"filter":{"query":"@eventName:*"},"name":"Other","id":99}],"targets":{"name":"ocsf.activity_name","id":"ocsf.activity_id"},"fallback":{"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"},"sources":{"ocsf.activity_name":["eventName"]}},"type":"schema-category-mapper"},{"name":"status","categories":[{"filter":{"query":"-@errorCode:*"},"name":"Success","id":1},{"filter":{"query":"@errorCode:*"},"name":"Failure","id":2}],"targets":{"name":"ocsf.status","id":"ocsf.status_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Set + default severity","categories":[{"filter":{"query":"@eventName:*"},"name":"Informational","id":1}],"targets":{"name":"ocsf.severity","id":"ocsf.severity_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + api to ocsf.api","sources":["api"],"target":"ocsf.api","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + user to ocsf.user","sources":["user"],"target":"ocsf.user","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","sources":["actor"],"target":"ocsf.actor","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","sources":["cloud"],"target":"ocsf.cloud","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","sources":["http_request"],"target":"ocsf.http_request","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","sources":["metadata"],"target":"ocsf.metadata","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + time to ocsf.time","sources":["time"],"target":"ocsf.time","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","sources":["src_endpoint"],"target":"ocsf.src_endpoint","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","sources":["severity"],"target":"ocsf.severity","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","sources":["severity_id"],"target":"ocsf.severity_id","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"}],"schema":{"schema_type":"ocsf","version":"1.5.0","class_name":"Account + Change","class_uid":3001,"extensions":[],"profiles":["cloud","datetime"]},"type":"schema-processor"}],"tags":[]} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +- request: + body: null + headers: + accept: + - '*/*' + method: DELETE + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines/1unf0vMNQKSSwzsg6BuWMw + response: + body: + string: '{} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.frozen b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.frozen new file mode 100644 index 0000000000..bbc78f11a0 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.frozen @@ -0,0 +1 @@ +2025-10-22T19:11:58.774Z \ No newline at end of file diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.yaml b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.yaml new file mode 100644 index 0000000000..fc81a35395 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_false_returns_ok_response.yaml @@ -0,0 +1,77 @@ +interactions: +- request: + body: '{"filter":{"query":"source:python"},"name":"testSchemaProcessor","processors":[{"is_enabled":true,"mappers":[{"categories":[{"filter":{"query":"@eventName:(*Create*)"},"id":1,"name":"Create"},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"id":3,"name":"Password Change"},{"filter":{"query":"@eventName:(*Attach*)"},"id":7,"name":"Attach + Policy"},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"id":8,"name":"Detach + Policy"},{"filter":{"query":"@eventName:(*Delete*)"},"id":6,"name":"Delete"},{"filter":{"query":"@eventName:*"},"id":99,"name":"Other"}],"fallback":{"sources":{"ocsf.activity_name":["eventName"]},"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"}},"name":"activity_id + and activity_name","targets":{"id":"ocsf.activity_id","name":"ocsf.activity_name"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"-@errorCode:*"},"id":1,"name":"Success"},{"filter":{"query":"@errorCode:*"},"id":2,"name":"Failure"}],"name":"status","targets":{"id":"ocsf.status_id","name":"ocsf.status"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"@eventName:*"},"id":1,"name":"Informational"}],"name":"Set + default severity","targets":{"id":"ocsf.severity_id","name":"ocsf.severity"},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","preserve_source":false,"sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","preserve_source":false,"sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","type":"schema-remapper"},{"name":"Map + api to ocsf.api","preserve_source":false,"sources":["api"],"target":"ocsf.api","type":"schema-remapper"},{"name":"Map + user to ocsf.user","preserve_source":false,"sources":["user"],"target":"ocsf.user","type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","preserve_source":false,"sources":["actor"],"target":"ocsf.actor","type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","preserve_source":false,"sources":["cloud"],"target":"ocsf.cloud","type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","preserve_source":false,"sources":["http_request"],"target":"ocsf.http_request","type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","preserve_source":false,"sources":["metadata"],"target":"ocsf.metadata","type":"schema-remapper"},{"name":"Map + time to ocsf.time","preserve_source":false,"sources":["time"],"target":"ocsf.time","type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","preserve_source":false,"sources":["src_endpoint"],"target":"ocsf.src_endpoint","type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","preserve_source":false,"sources":["severity"],"target":"ocsf.severity","type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","preserve_source":false,"sources":["severity_id"],"target":"ocsf.severity_id","type":"schema-remapper"}],"name":"Apply + OCSF schema for 3001","schema":{"class_name":"Account Change","class_uid":3001,"profiles":["cloud","datetime"],"schema_type":"ocsf","version":"1.5.0"},"type":"schema-processor"}],"tags":[]}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines + response: + body: + string: '{"id":"-qkKiJPYTne-113i8XJ_Nw","type":"pipeline","name":"testSchemaProcessor","is_enabled":false,"is_read_only":false,"filter":{"query":"source:python"},"processors":[{"name":"Apply + OCSF schema for 3001","is_enabled":true,"mappers":[{"name":"activity_id and + activity_name","categories":[{"filter":{"query":"@eventName:(*Create*)"},"name":"Create","id":1},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"name":"Password Change","id":3},{"filter":{"query":"@eventName:(*Attach*)"},"name":"Attach + Policy","id":7},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"name":"Detach + Policy","id":8},{"filter":{"query":"@eventName:(*Delete*)"},"name":"Delete","id":6},{"filter":{"query":"@eventName:*"},"name":"Other","id":99}],"targets":{"name":"ocsf.activity_name","id":"ocsf.activity_id"},"fallback":{"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"},"sources":{"ocsf.activity_name":["eventName"]}},"type":"schema-category-mapper"},{"name":"status","categories":[{"filter":{"query":"-@errorCode:*"},"name":"Success","id":1},{"filter":{"query":"@errorCode:*"},"name":"Failure","id":2}],"targets":{"name":"ocsf.status","id":"ocsf.status_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Set + default severity","categories":[{"filter":{"query":"@eventName:*"},"name":"Informational","id":1}],"targets":{"name":"ocsf.severity","id":"ocsf.severity_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + api to ocsf.api","sources":["api"],"target":"ocsf.api","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + user to ocsf.user","sources":["user"],"target":"ocsf.user","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","sources":["actor"],"target":"ocsf.actor","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","sources":["cloud"],"target":"ocsf.cloud","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","sources":["http_request"],"target":"ocsf.http_request","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","sources":["metadata"],"target":"ocsf.metadata","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + time to ocsf.time","sources":["time"],"target":"ocsf.time","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","sources":["src_endpoint"],"target":"ocsf.src_endpoint","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","sources":["severity"],"target":"ocsf.severity","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","sources":["severity_id"],"target":"ocsf.severity_id","preserve_source":false,"override_on_conflict":false,"type":"schema-remapper"}],"schema":{"schema_type":"ocsf","version":"1.5.0","class_name":"Account + Change","class_uid":3001,"extensions":[],"profiles":["cloud","datetime"]},"type":"schema-processor"}],"tags":[]} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +- request: + body: null + headers: + accept: + - '*/*' + method: DELETE + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines/-qkKiJPYTne-113i8XJ_Nw + response: + body: + string: '{} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.frozen b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.frozen new file mode 100644 index 0000000000..bd4fc475d3 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.frozen @@ -0,0 +1 @@ +2025-10-22T19:11:59.195Z \ No newline at end of file diff --git a/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.yaml b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.yaml new file mode 100644 index 0000000000..60594c1958 --- /dev/null +++ b/tests/v1/cassettes/test_scenarios/test_create_a_pipeline_with_schema_processor_and_preserve_source_true_returns_ok_response.yaml @@ -0,0 +1,77 @@ +interactions: +- request: + body: '{"filter":{"query":"source:python"},"name":"testSchemaProcessor","processors":[{"is_enabled":true,"mappers":[{"categories":[{"filter":{"query":"@eventName:(*Create*)"},"id":1,"name":"Create"},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"id":3,"name":"Password Change"},{"filter":{"query":"@eventName:(*Attach*)"},"id":7,"name":"Attach + Policy"},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"id":8,"name":"Detach + Policy"},{"filter":{"query":"@eventName:(*Delete*)"},"id":6,"name":"Delete"},{"filter":{"query":"@eventName:*"},"id":99,"name":"Other"}],"fallback":{"sources":{"ocsf.activity_name":["eventName"]},"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"}},"name":"activity_id + and activity_name","targets":{"id":"ocsf.activity_id","name":"ocsf.activity_name"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"-@errorCode:*"},"id":1,"name":"Success"},{"filter":{"query":"@errorCode:*"},"id":2,"name":"Failure"}],"name":"status","targets":{"id":"ocsf.status_id","name":"ocsf.status"},"type":"schema-category-mapper"},{"categories":[{"filter":{"query":"@eventName:*"},"id":1,"name":"Informational"}],"name":"Set + default severity","targets":{"id":"ocsf.severity_id","name":"ocsf.severity"},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","preserve_source":true,"sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","preserve_source":true,"sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","type":"schema-remapper"},{"name":"Map + api to ocsf.api","preserve_source":true,"sources":["api"],"target":"ocsf.api","type":"schema-remapper"},{"name":"Map + user to ocsf.user","preserve_source":true,"sources":["user"],"target":"ocsf.user","type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","preserve_source":true,"sources":["actor"],"target":"ocsf.actor","type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","preserve_source":true,"sources":["cloud"],"target":"ocsf.cloud","type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","preserve_source":true,"sources":["http_request"],"target":"ocsf.http_request","type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","preserve_source":true,"sources":["metadata"],"target":"ocsf.metadata","type":"schema-remapper"},{"name":"Map + time to ocsf.time","preserve_source":true,"sources":["time"],"target":"ocsf.time","type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","preserve_source":true,"sources":["src_endpoint"],"target":"ocsf.src_endpoint","type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","preserve_source":true,"sources":["severity"],"target":"ocsf.severity","type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","preserve_source":true,"sources":["severity_id"],"target":"ocsf.severity_id","type":"schema-remapper"}],"name":"Apply + OCSF schema for 3001","schema":{"class_name":"Account Change","class_uid":3001,"profiles":["cloud","datetime"],"schema_type":"ocsf","version":"1.5.0"},"type":"schema-processor"}],"tags":[]}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines + response: + body: + string: '{"id":"ReEWRVSbQ-ersoCn0Ibo6g","type":"pipeline","name":"testSchemaProcessor","is_enabled":false,"is_read_only":false,"filter":{"query":"source:python"},"processors":[{"name":"Apply + OCSF schema for 3001","is_enabled":true,"mappers":[{"name":"activity_id and + activity_name","categories":[{"filter":{"query":"@eventName:(*Create*)"},"name":"Create","id":1},{"filter":{"query":"@eventName:(ChangePassword + OR PasswordUpdated)"},"name":"Password Change","id":3},{"filter":{"query":"@eventName:(*Attach*)"},"name":"Attach + Policy","id":7},{"filter":{"query":"@eventName:(*Detach* OR *Remove*)"},"name":"Detach + Policy","id":8},{"filter":{"query":"@eventName:(*Delete*)"},"name":"Delete","id":6},{"filter":{"query":"@eventName:*"},"name":"Other","id":99}],"targets":{"name":"ocsf.activity_name","id":"ocsf.activity_id"},"fallback":{"values":{"ocsf.activity_id":"99","ocsf.activity_name":"Other"},"sources":{"ocsf.activity_name":["eventName"]}},"type":"schema-category-mapper"},{"name":"status","categories":[{"filter":{"query":"-@errorCode:*"},"name":"Success","id":1},{"filter":{"query":"@errorCode:*"},"name":"Failure","id":2}],"targets":{"name":"ocsf.status","id":"ocsf.status_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Set + default severity","categories":[{"filter":{"query":"@eventName:*"},"name":"Informational","id":1}],"targets":{"name":"ocsf.severity","id":"ocsf.severity_id"},"fallback":{"values":{},"sources":{}},"type":"schema-category-mapper"},{"name":"Map + userIdentity to ocsf.user.uid","sources":["userIdentity.principalId","responseElements.role.roleId","responseElements.user.userId"],"target":"ocsf.user.uid","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + userName to ocsf.user.name","sources":["requestParameters.userName","responseElements.role.roleName","requestParameters.roleName","responseElements.user.userName"],"target":"ocsf.user.name","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + api to ocsf.api","sources":["api"],"target":"ocsf.api","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + user to ocsf.user","sources":["user"],"target":"ocsf.user","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + actor to ocsf.actor","sources":["actor"],"target":"ocsf.actor","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + cloud to ocsf.cloud","sources":["cloud"],"target":"ocsf.cloud","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + http_request to ocsf.http_request","sources":["http_request"],"target":"ocsf.http_request","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + metadata to ocsf.metadata","sources":["metadata"],"target":"ocsf.metadata","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + time to ocsf.time","sources":["time"],"target":"ocsf.time","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + src_endpoint to ocsf.src_endpoint","sources":["src_endpoint"],"target":"ocsf.src_endpoint","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity to ocsf.severity","sources":["severity"],"target":"ocsf.severity","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"},{"name":"Map + severity_id to ocsf.severity_id","sources":["severity_id"],"target":"ocsf.severity_id","preserve_source":true,"override_on_conflict":false,"type":"schema-remapper"}],"schema":{"schema_type":"ocsf","version":"1.5.0","class_name":"Account + Change","class_uid":3001,"extensions":[],"profiles":["cloud","datetime"]},"type":"schema-processor"}],"tags":[]} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +- request: + body: null + headers: + accept: + - '*/*' + method: DELETE + uri: https://api.datadoghq.com/api/v1/logs/config/pipelines/ReEWRVSbQ-ersoCn0Ibo6g + response: + body: + string: '{} + + ' + headers: + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v1/features/logs_pipelines.feature b/tests/v1/features/logs_pipelines.feature index cdb150dd6e..5f7dde5db6 100644 --- a/tests/v1/features/logs_pipelines.feature +++ b/tests/v1/features/logs_pipelines.feature @@ -77,6 +77,20 @@ Feature: Logs Pipelines When the request is sent Then the response status is 200 OK + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Schema Processor and preserve_source false returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid", "preserve_source": false}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name", "preserve_source": false}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api", "preserve_source": false}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user", "preserve_source": false}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor", "preserve_source": false}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud", "preserve_source": false}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request", "preserve_source": false}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata", "preserve_source": false}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time", "preserve_source": false}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint", "preserve_source": false}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity", "preserve_source": false}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id", "preserve_source": false}]}], "tags": []} + When the request is sent + Then the response status is 200 OK + + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with Schema Processor and preserve_source true returns "OK" response + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid", "preserve_source": true}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name", "preserve_source": true}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api", "preserve_source": true}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user", "preserve_source": true}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor", "preserve_source": true}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud", "preserve_source": true}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request", "preserve_source": true}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata", "preserve_source": true}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time", "preserve_source": true}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint", "preserve_source": true}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity", "preserve_source": true}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id", "preserve_source": true}]}], "tags": []} + When the request is sent + Then the response status is 200 OK + @team:DataDog/event-platform-experience Scenario: Create a pipeline with Span Id Remapper returns "OK" response Given new "CreateLogsPipeline" request @@ -84,6 +98,13 @@ Feature: Logs Pipelines When the request is sent Then the response status is 200 OK + @team:DataDog/event-platform-experience + Scenario: Create a pipeline with schema processor + Given new "CreateLogsPipeline" request + And body with value {"filter": {"query": "source:python"}, "name": "testSchemaProcessor", "processors": [{"type": "schema-processor", "is_enabled": true, "name": "Apply OCSF schema for 3001", "schema": {"schema_type": "ocsf", "version": "1.5.0", "class_uid": 3001, "class_name": "Account Change", "profiles": ["cloud", "datetime"]}, "mappers": [{"type": "schema-category-mapper", "name": "activity_id and activity_name", "categories": [{"filter": {"query": "@eventName:(*Create*)"}, "name": "Create", "id": 1}, {"filter": {"query": "@eventName:(ChangePassword OR PasswordUpdated)"}, "name": "Password Change", "id": 3}, {"filter": {"query": "@eventName:(*Attach*)"}, "name": "Attach Policy", "id": 7}, {"filter": {"query": "@eventName:(*Detach* OR *Remove*)"}, "name": "Detach Policy", "id": 8}, {"filter": {"query": "@eventName:(*Delete*)"}, "name": "Delete", "id": 6}, {"filter": {"query": "@eventName:*"}, "name": "Other", "id": 99}], "targets": {"name": "ocsf.activity_name", "id": "ocsf.activity_id"}, "fallback": {"values": {"ocsf.activity_id": "99", "ocsf.activity_name": "Other"}, "sources": {"ocsf.activity_name": ["eventName"]}}}, {"type": "schema-category-mapper", "name": "status", "categories": [{"filter": {"query": "-@errorCode:*"}, "id": 1, "name": "Success"}, {"filter": {"query": "@errorCode:*"}, "id": 2, "name": "Failure"}], "targets": {"id": "ocsf.status_id", "name": "ocsf.status"}}, {"type": "schema-category-mapper", "name": "Set default severity", "categories": [{"filter": {"query": "@eventName:*"}, "name": "Informational", "id": 1}], "targets": {"name": "ocsf.severity", "id": "ocsf.severity_id"}}, {"type": "schema-remapper", "name": "Map userIdentity to ocsf.user.uid", "sources": ["userIdentity.principalId", "responseElements.role.roleId", "responseElements.user.userId"], "target": "ocsf.user.uid"}, {"type": "schema-remapper", "name": "Map userName to ocsf.user.name", "sources": ["requestParameters.userName", "responseElements.role.roleName", "requestParameters.roleName", "responseElements.user.userName"], "target": "ocsf.user.name"}, {"type": "schema-remapper", "name": "Map api to ocsf.api", "sources": ["api"], "target": "ocsf.api"}, {"type": "schema-remapper", "name": "Map user to ocsf.user", "sources": ["user"], "target": "ocsf.user"}, {"type": "schema-remapper", "name": "Map actor to ocsf.actor", "sources": ["actor"], "target": "ocsf.actor"}, {"type": "schema-remapper", "name": "Map cloud to ocsf.cloud", "sources": ["cloud"], "target": "ocsf.cloud"}, {"type": "schema-remapper", "name": "Map http_request to ocsf.http_request", "sources": ["http_request"], "target": "ocsf.http_request"}, {"type": "schema-remapper", "name": "Map metadata to ocsf.metadata", "sources": ["metadata"], "target": "ocsf.metadata"}, {"type": "schema-remapper", "name": "Map time to ocsf.time", "sources": ["time"], "target": "ocsf.time"}, {"type": "schema-remapper", "name": "Map src_endpoint to ocsf.src_endpoint", "sources": ["src_endpoint"], "target": "ocsf.src_endpoint"}, {"type": "schema-remapper", "name": "Map severity to ocsf.severity", "sources": ["severity"], "target": "ocsf.severity"}, {"type": "schema-remapper", "name": "Map severity_id to ocsf.severity_id", "sources": ["severity_id"], "target": "ocsf.severity_id"}]}], "tags": []} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/event-platform-experience Scenario: Delete a pipeline returns "Bad Request" response Given new "DeleteLogsPipeline" request