Skip to content

Commit

Permalink
Validation of the YAML structure of query files (#660)
Browse files Browse the repository at this point in the history 
* Update first version of query schema

* Adding JSON Schema library to dependencies

* Add tests to validate the correctness of existing queries

* Fix typos & missing description in queries

* Convert path to Path type

* If path is not a dir, use relative path from config file

* Update validation method to handle expected fails

* Fix path resolution for custom queries
Handle expected failures

* Add missing keys
Ensure parameters are always defined

* Fix incorrect type names

* Remove empty parameters from query files

* Fixing typo

* Adding new metadata fields

* Remove empty parameters
replace tabs with spaces

* Add SQL data environment

* Ignore folder containing Sentinel queries

* Fix call to absolute

* Update JSON validation schema to have an open list of providers

* Replacing tabs with spaces

* s/oneOf/anyOf/

* Rework condition to ignore sentinel_query_import_data

* Simplify comparison for python 3.8

* Increasing the size of description to 1024 characters

* Adding metadata, sources and defaults as mandatory

* Ignore msticpyconfig files

* Remove defaults as a required key

* Created dedicated tests for query validation

* Add multiple sample of valid and invalid queries

* Update test_pkg_config based on commit's comments

---------

Co-authored-by: Florian Bracq <florian.bracq+github@gmail.com>
Co-authored-by: Ian Hellen <ianhelle@microsoft.com>
  • Loading branch information
@FlorianBracq @Furoxu @ianhelle
3 people committed Sep 26, 2023
1 parent ff77bdd commit 6cd72dc
Show file tree
Hide file tree
Showing 38 changed files with 596 additions and 164 deletions.
270 changes: 270 additions & 0 deletions .schemas/queries.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"properties": {
"metadata": {
"$ref": "#/$defs/metadata"
},
"defaults": {
"type": "object",
"properties": {
"metadata": {
"type": "object",
"properties": {
"data_source": {
"type": "string"
},
"data_families": {
"type": "array",
"items": {
"type": "string"
}
},
"pivot": {
"type": "object",
"patternProperties": {
".*": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"top": {
"type": "integer"
}
}
},
"parameters": {
"$ref": "#/$defs/parameter"
}
}
},
"sources": {
"type": "object",
"patternProperties": {
".*": {
"$ref": "#/$defs/query"
}
}
}
},
"required": [
"metadata",
"sources"
],
"$defs": {
"description": {
"type": "string",
"minLength": 5,
"maxLength": 1024
},
"metadata": {
"type": "object",
"properties": {
"version": {
"type": "integer"
},
"description": {
"$ref": "#/$defs/description"
},
"data_environments": {
"type": "array",
"items": {
"anyOf": [
{
"enum": [
"AzureSecurityCenter",
"AzureSentinel",
"Cybereason",
"Elastic",
"Kusto",
"LocalData",
"LogAnalytics",
"M365D",
"MDATP",
"MDE",
"Mordor",
"MSGraph",
"MSSentinel",
"OSQueryLogs",
"OTRF",
"ResourceGraph",
"SecurityGraph",
"Splunk",
"Sumologic"
]
},
{
"type": "string"
}
]
}
},
"data_families": {
"type": "array",
"items": {
"type": "string"
}
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"aliases": {
"oneOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
},
"cluster": {
"type": "string"
},
"clusters": {
"type": "array",
"items": {
"type": "string"
}
},
"cluster_groups": {
"type": "array",
"items": {
"type": "string"
}
},
"database": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"version",
"description",
"data_environments",
"data_families"
]
},
"parameter": {
"type": "object",
"patternProperties": {
".*": {
"type": "object",
"properties": {
"description": {
"$ref": "#/$defs/description"
},
"type": {
"type": "string",
"enum": [
"str",
"datetime",
"int",
"float",
"list"
]
},
"default": {
"oneOf": [
{
"type": "string"
},
{
"type": "array"
},
{
"type": "integer"
}
]
},
"aliases": {
"oneOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
}
},
"additionalProperties": false,
"required": [
"description",
"type"
]
}
}
},
"query": {
"type": "object",
"properties": {
"description": {
"$ref": "#/$defs/description"
},
"metadata": {
"anyOf": [
{
"$ref": "#/$defs/metadata"
},
true
]
},
"parameters": {
"$ref": "#/$defs/parameter"
},
"args": {
"type": "object",
"properties": {
"query": {
"type": "string"
},
"uri": {
"type": "string"
}
},
"required": [
"query"
]
},
"query_macros": {
"patternProperties": {
".*": {
"type": "object",
"properties": {
"description": {
"$ref": "#/$defs/description"
},
"value": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"description",
"value"
]
}
}
}
},
"additionalProperties": false,
"required": [
"description",
"args"
]
}
}
}
4 changes: 2 additions & 2 deletions msticpy/data/queries/cybereason/cybereason_processes.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ sources:
parameters:
command:
description: Command to search for
type: string
type: str
find_process_by_suspicions:
description: Search for process with a specific suspicion
metadata:
Expand Down Expand Up @@ -108,7 +108,7 @@ sources:
parameters:
suspicion:
description: Suspicion that the process should have
type: string
type: str
find_process_by_pid:
description: Search for a process by pid and hostname
metadata:
Expand Down
10 changes: 0 additions & 10 deletions msticpy/data/queries/localdata/local_data.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,68 +7,58 @@ metadata:
defaults:
metadata:
data_source: 'security_alert'
parameters:
sources:
list_alerts:
description: Retrieves list of alerts
metadata:
data_families: [SecurityAlert]
args:
query: alerts_list.pkl
parameters:
list_host_processes:
description: List processes on host
metadata:
data_families: [WindowsSecurity]
args:
query: processes_on_host.pkl
parameters:
list_host_logons:
description: List logons on host
metadata:
data_families: [WindowsSecurity]
args:
query: host_logons.pkl
parameters:
list_host_logon_failures:
description: List logon failures on host
metadata:
data_families: [WindowsSecurity]
args:
query: failed_logons.pkl
parameters:
list_host_events:
description: List events failures on host
metadata:
data_families: [WindowsSecurity]
args:
query: all_events_df.pkl
parameters:
get_process_tree:
description: Get process tree for a process
metadata:
data_families: [WindowsSecurity]
args:
query: process_tree.pkl
parameters:
list_azure_network_flows_by_ip:
description: List Azure Network flows by IP address
metadata:
data_families: [Network]
args:
query: az_net_comms_df.pkl
parameters:
list_azure_network_flows_by_host:
description: List Azure Network flows by host name
metadata:
data_families: [Network]
args:
query: az_net_comms_df.pkl
parameters:
list_all_signins_geo:
description: List all Azure AD logon events
metadata:
data_families: [Azure]
args:
query: aad_logons.pkl
parameters:
4 changes: 1 addition & 3 deletions msticpy/data/queries/m365d/kql_m365_alerts.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ sources:
| where Timestamp <= datetime({end})
{add_query_items}"
uri: None
parameters:
list_alerts_with_evidence:
description: Retrieves list of alerts with their evidence
metadata:
Expand All @@ -50,7 +49,6 @@ sources:
SourceSystem1, Type1
{add_query_items}"
uri: None
parameters:
host_alerts:
description: Lists alerts associated with host/device name
metadata:
Expand Down Expand Up @@ -284,4 +282,4 @@ sources:
parameters:
app_name:
description: The name of the application
type: str
type: str

0 comments on commit 6cd72dc

Please sign in to comment.