diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index 97c0296c..feb0351b 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -51,7 +51,9 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups)) updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros)) updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards)) - updated_conf_files.update(conf_output.writeAppConf()) + updated_conf_files.update(conf_output.writeMiscellaneousAppFiles()) + + #Ensure that the conf file we just generated/update is syntactically valid for conf_file in updated_conf_files: diff --git a/contentctl/actions/deploy_acs.py b/contentctl/actions/deploy_acs.py index 0bc0054f..8451751b 100644 --- a/contentctl/actions/deploy_acs.py +++ b/contentctl/actions/deploy_acs.py @@ -1,38 +1,55 @@ -from dataclasses import dataclass -from contentctl.input.director import DirectorInputDto -from contentctl.output.conf_output import ConfOutput - - -from typing import Union - -@dataclass(frozen=True) -class ACSDeployInputDto: - director_input_dto: DirectorInputDto - splunk_api_username: str - splunk_api_password: str - splunk_cloud_jwt_token: str - splunk_cloud_stack: str - stack_type: str +from contentctl.objects.config import deploy_acs, StackType +from requests import post +import pprint class Deploy: - def execute(self, input_dto: ACSDeployInputDto) -> None: - - conf_output = ConfOutput(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config) + def execute(self, config: deploy_acs, appinspect_token:str) -> None: - appinspect_token = conf_output.inspectAppAPI(input_dto.splunk_api_username, input_dto.splunk_api_password, input_dto.stack_type) + #The following common headers are used by both Clasic and Victoria + headers = { + 'Authorization': f'Bearer {config.splunk_cloud_jwt_token}', + 'ACS-Legal-Ack': 'Y' + } + try: + + with open(config.getPackageFilePath(include_version=False),'rb') as app_data: + #request_data = app_data.read() + if config.stack_type == StackType.classic: + # Classic instead uses a form to store token and package + # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience + address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps" + + form_data = { + 'token': (None, appinspect_token), + 'package': app_data + } + res = post(address, headers=headers, files = form_data) + elif config.stack_type == StackType.victoria: + # Victoria uses the X-Splunk-Authorization Header + # It also uses --data-binary for the app content + # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience + headers.update({'X-Splunk-Authorization': appinspect_token}) + address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria" + res = post(address, headers=headers, data=app_data.read()) + else: + raise Exception(f"Unsupported stack type: '{config.stack_type}'") + except Exception as e: + raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}") - - if input_dto.splunk_cloud_jwt_token is None or input_dto.splunk_cloud_stack is None: - if input_dto.splunk_cloud_jwt_token is None: - raise Exception("Cannot deploy app via ACS, --splunk_cloud_jwt_token was not defined on command line.") - else: - raise Exception("Cannot deploy app via ACS, --splunk_cloud_stack was not defined on command line.") - - conf_output.deploy_via_acs(input_dto.splunk_cloud_jwt_token, - input_dto.splunk_cloud_stack, - appinspect_token, - input_dto.stack_type) - + try: + # Request went through and completed, but may have returned a non-successful error code. + # This likely includes a more verbose response describing the error + res.raise_for_status() + print(res.json()) + except Exception as e: + try: + error_text = res.json() + except Exception as e: + error_text = "No error text - request failed" + formatted_error_text = pprint.pformat(error_text) + print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n" + "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.") + raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}") - \ No newline at end of file + print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!") \ No newline at end of file diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index bfed85a3..ee9e4693 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -13,6 +13,7 @@ from contentctl.objects.macro import Macro from contentctl.objects.lookup import Lookup from contentctl.objects.detection import Detection +from contentctl.objects.data_source import DataSource from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.config import test_common, All, Changes, Selected @@ -67,9 +68,12 @@ def getChanges(self, target_branch:str)->List[Detection]: #Make a filename to content map filepath_to_content_map = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items()} - updated_detections:List[Detection] = [] - updated_macros:List[Macro] = [] - updated_lookups:List[Lookup] =[] + + updated_detections: set[Detection] = set() + updated_macros: set[Macro] = set() + updated_lookups: set[Lookup] = set() + updated_datasources: set[DataSource] = set() + for diff in all_diffs: if type(diff) == pygit2.Patch: @@ -80,16 +84,23 @@ def getChanges(self, target_branch:str)->List[Detection]: if decoded_path.is_relative_to(self.config.path/"detections") and decoded_path.suffix == ".yml": detectionObject = filepath_to_content_map.get(decoded_path, None) if isinstance(detectionObject, Detection): - updated_detections.append(detectionObject) + updated_detections.add(detectionObject) else: raise Exception(f"Error getting detection object for file {str(decoded_path)}") elif decoded_path.is_relative_to(self.config.path/"macros") and decoded_path.suffix == ".yml": macroObject = filepath_to_content_map.get(decoded_path, None) if isinstance(macroObject, Macro): - updated_macros.append(macroObject) + updated_macros.add(macroObject) else: raise Exception(f"Error getting macro object for file {str(decoded_path)}") + + elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml": + datasourceObject = filepath_to_content_map.get(decoded_path, None) + if isinstance(datasourceObject, DataSource): + updated_datasources.add(datasourceObject) + else: + raise Exception(f"Error getting data source object for file {str(decoded_path)}") elif decoded_path.is_relative_to(self.config.path/"lookups"): # We need to convert this to a yml. This means we will catch @@ -98,7 +109,7 @@ def getChanges(self, target_branch:str)->List[Detection]: updatedLookup = filepath_to_content_map.get(decoded_path, None) if not isinstance(updatedLookup,Lookup): raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}") - updated_lookups.append(updatedLookup) + updated_lookups.add(updatedLookup) elif decoded_path.suffix == ".csv": # If the CSV was updated, we want to make sure that we @@ -115,7 +126,6 @@ def getChanges(self, target_branch:str)->List[Detection]: # Detected a changed .mlmodel file. However, since we do not have testing for these detections at # this time, we will ignore this change. updatedLookup = None - else: raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n" @@ -125,7 +135,7 @@ def getChanges(self, target_branch:str)->List[Detection]: if updatedLookup is not None and updatedLookup not in updated_lookups: # It is possible that both the CSV and YML have been modified for the same lookup, # and we do not want to add it twice. - updated_lookups.append(updatedLookup) + updated_lookups.add(updatedLookup) else: pass @@ -136,7 +146,8 @@ def getChanges(self, target_branch:str)->List[Detection]: # If a detection has at least one dependency on changed content, # then we must test it again - changed_macros_and_lookups = updated_macros + updated_lookups + + changed_macros_and_lookups_and_datasources:set[SecurityContentObject] = updated_macros.union(updated_lookups, updated_datasources) for detection in self.director.detections: if detection in updated_detections: @@ -144,16 +155,16 @@ def getChanges(self, target_branch:str)->List[Detection]: # to add it again continue - for obj in changed_macros_and_lookups: + for obj in changed_macros_and_lookups_and_datasources: if obj in detection.get_content_dependencies(): - updated_detections.append(detection) + updated_detections.add(detection) break #Print out the names of all modified/new content modifiedAndNewContentString = "\n - ".join(sorted([d.name for d in updated_detections])) print(f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}") - return updated_detections + return sorted(list(updated_detections)) def getSelected(self, detectionFilenames: List[FilePath]) -> List[Detection]: filepath_to_content_map: dict[FilePath, SecurityContentObject] = { diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 95ebc464..8e816025 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -13,7 +13,7 @@ from shutil import copyfile from typing import Union, Optional -from pydantic import BaseModel, PrivateAttr, Field, dataclasses +from pydantic import ConfigDict, BaseModel, PrivateAttr, Field, dataclasses import requests # type: ignore import splunklib.client as client # type: ignore from splunklib.binding import HTTPError # type: ignore @@ -48,9 +48,9 @@ class SetupTestGroupResults(BaseModel): success: bool = True duration: float = 0 start_time: float - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict( + arbitrary_types_allowed=True + ) class CleanupTestGroupResults(BaseModel): @@ -68,6 +68,15 @@ class CannotRunBaselineException(Exception): # exception pass +class ReplayIndexDoesNotExistOnServer(Exception): + ''' + In order to replay data files into the Splunk Server + for testing, they must be replayed into an index that + exists. If that index does not exist, this error will + be generated and raised before we try to do anything else + with that Data File. + ''' + pass @dataclasses.dataclass(frozen=False) class DetectionTestingManagerOutputDto(): @@ -75,7 +84,7 @@ class DetectionTestingManagerOutputDto(): outputQueue: list[Detection] = Field(default_factory=list) currentTestingQueue: dict[str, Union[Detection, None]] = Field(default_factory=dict) start_time: Union[datetime.datetime, None] = None - replay_index: str = "CONTENTCTL_TESTING_INDEX" + replay_index: str = "contentctl_testing_index" replay_host: str = "CONTENTCTL_HOST" timeout_seconds: int = 60 terminate: bool = False @@ -88,12 +97,13 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC): sync_obj: DetectionTestingManagerOutputDto hec_token: str = "" hec_channel: str = "" + all_indexes_on_server: list[str] = [] _conn: client.Service = PrivateAttr() pbar: tqdm.tqdm = None start_time: Optional[float] = None - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict( + arbitrary_types_allowed=True + ) def __init__(self, **data): super().__init__(**data) @@ -131,6 +141,7 @@ def setup(self): (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + (self.get_all_indexes, "Getting all indexes from server"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), @@ -169,12 +180,11 @@ def configure_hec(self): pass try: - res = self.get_conn().inputs.create( name="DETECTION_TESTING_HEC", kind="http", index=self.sync_obj.replay_index, - indexes=f"{self.sync_obj.replay_index},_internal,_audit", + indexes=",".join(self.all_indexes_on_server), # This allows the HEC to write to all indexes useACK=True, ) self.hec_token = str(res.token) @@ -183,6 +193,23 @@ def configure_hec(self): except Exception as e: raise (Exception(f"Failure creating HEC Endpoint: {str(e)}")) + def get_all_indexes(self) -> None: + """ + Retrieve a list of all indexes in the Splunk instance + """ + try: + # We do not include the replay index because by + # the time we get to this function, it has already + # been created on the server. + indexes = [] + res = self.get_conn().indexes + for index in res.list(): + indexes.append(index.name) + # Retrieve all available indexes on the splunk instance + self.all_indexes_on_server = indexes + except Exception as e: + raise (Exception(f"Failure getting indexes: {str(e)}")) + def get_conn(self) -> client.Service: try: if not self._conn: @@ -265,11 +292,7 @@ def configure_imported_roles( self, imported_roles: list[str] = ["user", "power", "can_delete"], enterprise_security_roles: list[str] = ["ess_admin", "ess_analyst", "ess_user"], - indexes: list[str] = ["_*", "*"], - ): - indexes.append(self.sync_obj.replay_index) - indexes_encoded = ";".join(indexes) - + ): try: # Set which roles should be configured. For Enterprise Security/Integration Testing, # we must add some extra foles. @@ -281,7 +304,7 @@ def configure_imported_roles( self.get_conn().roles.post( self.infrastructure.splunk_app_username, imported_roles=roles, - srchIndexesAllowed=indexes_encoded, + srchIndexesAllowed=";".join(self.all_indexes_on_server), srchIndexesDefault=self.sync_obj.replay_index, ) return @@ -293,19 +316,17 @@ def configure_imported_roles( self.get_conn().roles.post( self.infrastructure.splunk_app_username, imported_roles=imported_roles, - srchIndexesAllowed=indexes_encoded, + srchIndexesAllowed=";".join(self.all_indexes_on_server), srchIndexesDefault=self.sync_obj.replay_index, ) - def configure_delete_indexes(self, indexes: list[str] = ["_*", "*"]): - indexes.append(self.sync_obj.replay_index) + def configure_delete_indexes(self): endpoint = "/services/properties/authorize/default/deleteIndexesAllowed" - indexes_encoded = ";".join(indexes) try: - self.get_conn().post(endpoint, value=indexes_encoded) + self.get_conn().post(endpoint, value=";".join(self.all_indexes_on_server)) except Exception as e: self.pbar.write( - f"Error configuring deleteIndexesAllowed with '{indexes_encoded}': [{str(e)}]" + f"Error configuring deleteIndexesAllowed with '{self.all_indexes_on_server}': [{str(e)}]" ) def wait_for_conf_file(self, app_name: str, conf_file_name: str): @@ -654,8 +675,6 @@ def execute_unit_test( # Set the mode and timeframe, if required kwargs = {"exec_mode": "blocking"} - - # Set earliest_time and latest_time appropriately if FORCE_ALL_TIME is False if not FORCE_ALL_TIME: if test.earliest_time is not None: @@ -1035,8 +1054,8 @@ def retry_search_until_timeout( # Get the start time and compute the timeout search_start_time = time.time() search_stop_time = time.time() + self.sync_obj.timeout_seconds - - # Make a copy of the search string since we may + + # Make a copy of the search string since we may # need to make some small changes to it below search = detection.search @@ -1088,8 +1107,6 @@ def retry_search_until_timeout( # Initialize the collection of fields that are empty that shouldn't be present_threat_objects: set[str] = set() empty_fields: set[str] = set() - - # Filter out any messages in the results for result in results: @@ -1119,7 +1136,7 @@ def retry_search_until_timeout( # not populated and we should throw an error. This can happen if there is a typo # on a field. In this case, the field will appear but will not contain any values current_empty_fields: set[str] = set() - + for field in observable_fields_set: if result.get(field, 'null') == 'null': if field in risk_object_fields_set: @@ -1139,9 +1156,7 @@ def retry_search_until_timeout( if field in threat_object_fields_set: present_threat_objects.add(field) continue - - # If everything succeeded up until now, and no empty fields are found in the # current result, then the search was a success if len(current_empty_fields) == 0: @@ -1155,8 +1170,7 @@ def retry_search_until_timeout( else: empty_fields = empty_fields.union(current_empty_fields) - - + missing_threat_objects = threat_object_fields_set - present_threat_objects # Report a failure if there were empty fields in a threat object in all results if len(missing_threat_objects) > 0: @@ -1172,7 +1186,6 @@ def retry_search_until_timeout( duration=time.time() - search_start_time, ) return - test.result.set_job_content( job.content, @@ -1233,9 +1246,19 @@ def replay_attack_data_file( test_group: TestGroup, test_group_start_time: float, ): - tempfile = mktemp(dir=tmp_dir) - + # Before attempting to replay the file, ensure that the index we want + # to replay into actuall exists. If not, we should throw a detailed + # exception that can easily be interpreted by the user. + if attack_data_file.custom_index is not None and \ + attack_data_file.custom_index not in self.all_indexes_on_server: + raise ReplayIndexDoesNotExistOnServer( + f"Unable to replay data file {attack_data_file.data} " + f"into index '{attack_data_file.custom_index}'. " + "The index does not exist on the Splunk Server. " + f"The only valid indexes on the server are {self.all_indexes_on_server}" + ) + tempfile = mktemp(dir=tmp_dir) if not (str(attack_data_file.data).startswith("http://") or str(attack_data_file.data).startswith("https://")) : if pathlib.Path(str(attack_data_file.data)).is_file(): @@ -1280,7 +1303,6 @@ def replay_attack_data_file( ) ) - # Upload the data self.format_pbar_string( TestReportingType.GROUP, diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py b/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py index 5e2e46c0..cd50d978 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py @@ -1,12 +1,14 @@ -from bottle import template, Bottle, ServerAdapter -from contentctl.actions.detection_testing.views.DetectionTestingView import ( - DetectionTestingView, -) +from threading import Thread +from bottle import template, Bottle, ServerAdapter from wsgiref.simple_server import make_server, WSGIRequestHandler import jinja2 import webbrowser -from threading import Thread +from pydantic import ConfigDict + +from contentctl.actions.detection_testing.views.DetectionTestingView import ( + DetectionTestingView, +) DEFAULT_WEB_UI_PORT = 7999 @@ -100,9 +102,9 @@ def log_exception(*args, **kwargs): class DetectionTestingViewWeb(DetectionTestingView): bottleApp: Bottle = Bottle() server: SimpleWebServer = SimpleWebServer(host="0.0.0.0", port=DEFAULT_WEB_UI_PORT) - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict( + arbitrary_types_allowed=True + ) def setup(self): self.bottleApp.route("/", callback=self.showStatus) diff --git a/contentctl/actions/inspect.py b/contentctl/actions/inspect.py index 38bc2b23..261fd413 100644 --- a/contentctl/actions/inspect.py +++ b/contentctl/actions/inspect.py @@ -297,9 +297,11 @@ def check_detection_metadata(self, config: inspect) -> None: validation_errors[rule_name] = [] # No detections should be removed from build to build if rule_name not in current_build_conf.detection_stanzas: - validation_errors[rule_name].append(DetectionMissingError(rule_name=rule_name)) + if config.suppress_missing_content_exceptions: + print(f"[SUPPRESSED] {DetectionMissingError(rule_name=rule_name).long_message}") + else: + validation_errors[rule_name].append(DetectionMissingError(rule_name=rule_name)) continue - # Pull out the individual stanza for readability previous_stanza = previous_build_conf.detection_stanzas[rule_name] current_stanza = current_build_conf.detection_stanzas[rule_name] @@ -335,7 +337,7 @@ def check_detection_metadata(self, config: inspect) -> None: ) # Convert our dict mapping to a flat list of errors for use in reporting - validation_error_list = [x for inner_list in validation_errors.values() for x in inner_list] + validation_error_list = [x for inner_list in validation_errors.values() for x in inner_list] # Report failure/success print("\nDetection Metadata Validation:") @@ -355,4 +357,4 @@ def check_detection_metadata(self, config: inspect) -> None: raise ExceptionGroup( "Validation errors when comparing detection stanzas in current and previous build:", validation_error_list - ) + ) \ No newline at end of file diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index bca81521..2e451704 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -29,8 +29,7 @@ def buildDetection(self)->dict[str,Any]: answers['date'] = datetime.today().strftime('%Y-%m-%d') answers['author'] = answers['detection_author'] del answers['detection_author'] - answers['data_sources'] = answers['data_source'] - del answers['data_source'] + answers['data_source'] = answers['data_source'] answers['type'] = answers['detection_type'] del answers['detection_type'] answers['status'] = "production" #start everything as production since that's what we INTEND the content to become @@ -50,7 +49,6 @@ def buildDetection(self)->dict[str,Any]: answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')] answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}] answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - answers['tags']['required_fields'] = ['UPDATE'] answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100' answers['tags']['security_domain'] = answers['security_domain'] del answers["security_domain"] diff --git a/contentctl/actions/release_notes.py b/contentctl/actions/release_notes.py index 859fcf87..fe2c90d4 100644 --- a/contentctl/actions/release_notes.py +++ b/contentctl/actions/release_notes.py @@ -114,9 +114,11 @@ def release_notes(self, config:release_notes) -> None: #If a branch name is supplied, compare against develop if config.latest_branch not in repo.branches: raise ValueError(f"latest branch {config.latest_branch} does not exist in the repository. Make sure your branch name is correct") - compare_against = "develop" + if config.compare_against not in repo.branches: + raise ValueError(f"compare_against branch {config.compare_against} does not exist in the repository. Make sure your branch name is correct") + commit1 = repo.commit(config.latest_branch) - commit2 = repo.commit(compare_against) + commit2 = repo.commit(config.compare_against) diff_index = commit2.diff(commit1) modified_files:List[pathlib.Path] = [] @@ -189,7 +191,7 @@ def release_notes(self, config:release_notes) -> None: if config.latest_branch: print(f"Generating release notes - \033[92m{config.latest_branch}\033[0m") - print(f"Compared against - \033[92m{compare_against}\033[0m") + print(f"Compared against - \033[92m{config.compare_against}\033[0m") print("\n## Release notes for ESCU " + config.latest_branch) notes = [self.create_notes(config.path, stories_added, header="New Analytic Story"), diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index dbf434a7..1bb21174 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -19,6 +19,7 @@ from contentctl.actions.reporting import ReportingInputDto, Reporting from contentctl.actions.inspect import Inspect from contentctl.input.yml_reader import YmlReader +from contentctl.actions.deploy_acs import Deploy from contentctl.actions.release_notes import ReleaseNotes # def print_ascii_art(): @@ -95,8 +96,11 @@ def new_func(config:new): def deploy_acs_func(config:deploy_acs): - #This is a bit challenging to get to work with the default values. - raise Exception("deploy acs not yet implemented") + print("Building and inspecting app...") + token = inspect_func(config) + print("App successfully built and inspected.") + print("Deploying app...") + Deploy().execute(config, token) def test_common_func(config:test_common): if type(config) == test: @@ -150,7 +154,7 @@ def main(): else: #The file exists, so load it up! - config_obj = YmlReader().load_file(configFile) + config_obj = YmlReader().load_file(configFile,add_fields=False) t = test.model_validate(config_obj) except Exception as e: print(f"Error validating 'contentctl.yml':\n{str(e)}") diff --git a/contentctl/enrichments/cve_enrichment.py b/contentctl/enrichments/cve_enrichment.py index 748a66b3..66160eda 100644 --- a/contentctl/enrichments/cve_enrichment.py +++ b/contentctl/enrichments/cve_enrichment.py @@ -5,7 +5,7 @@ import shelve import time from typing import Annotated, Any, Union, TYPE_CHECKING -from pydantic import BaseModel,Field, computed_field +from pydantic import ConfigDict, BaseModel,Field, computed_field from decimal import Decimal from requests.exceptions import ReadTimeout from contentctl.objects.annotated_types import CVE_TYPE @@ -32,13 +32,12 @@ def url(self)->str: class CveEnrichment(BaseModel): use_enrichment: bool = True cve_api_obj: Union[CVESearch,None] = None - - class Config: - # Arbitrary_types are allowed to let us use the CVESearch Object - arbitrary_types_allowed = True - frozen = True - + # Arbitrary_types are allowed to let us use the CVESearch Object + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True + ) @staticmethod def getCveEnrichment(config:validate, timeout_seconds:int=10, force_disable_enrichment:bool=True)->CveEnrichment: diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 261ecb64..e0649f2d 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -247,20 +247,6 @@ def validate_git_pull_request(repo_path: str, pr_number: int) -> str: return hash - # @staticmethod - # def check_required_fields( - # thisField: str, definedFields: dict, requiredFields: list[str] - # ): - # missing_fields = [ - # field for field in requiredFields if field not in definedFields - # ] - # if len(missing_fields) > 0: - # raise ( - # ValueError( - # f"Could not validate - please resolve other errors resulting in missing fields {missing_fields}" - # ) - # ) - @staticmethod def verify_file_exists( file_path: str, verbose_print=False, timeout_seconds: int = 10 diff --git a/contentctl/input/new_content_questions.py b/contentctl/input/new_content_questions.py index 0bd227d4..02b20f46 100644 --- a/contentctl/input/new_content_questions.py +++ b/contentctl/input/new_content_questions.py @@ -1,4 +1,5 @@ from typing import Any +from contentctl.objects.enums import DataSource class NewContentQuestions: @@ -48,46 +49,9 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: 'type': 'checkbox', 'message': 'Your data source', 'name': 'data_source', - 'choices': [ - "OSQuery ES Process Events", - "Powershell 4104", - "Sysmon Event ID 1", - "Sysmon Event ID 3", - "Sysmon Event ID 5", - "Sysmon Event ID 6", - "Sysmon Event ID 7", - "Sysmon Event ID 8", - "Sysmon Event ID 9", - "Sysmon Event ID 10", - "Sysmon Event ID 11", - "Sysmon Event ID 13", - "Sysmon Event ID 15", - "Sysmon Event ID 20", - "Sysmon Event ID 21", - "Sysmon Event ID 22", - "Sysmon Event ID 23", - "Windows Security 4624", - "Windows Security 4625", - "Windows Security 4648", - "Windows Security 4663", - "Windows Security 4688", - "Windows Security 4698", - "Windows Security 4703", - "Windows Security 4720", - "Windows Security 4732", - "Windows Security 4738", - "Windows Security 4741", - "Windows Security 4742", - "Windows Security 4768", - "Windows Security 4769", - "Windows Security 4771", - "Windows Security 4776", - "Windows Security 4781", - "Windows Security 4798", - "Windows Security 5136", - "Windows Security 5145", - "Windows System 7045" - ] + #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory + 'choices': sorted(DataSource._value2member_map_ ) + }, { "type": "text", diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 26014886..dc0350d5 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -36,7 +36,7 @@ from contentctl.objects.integration_test import IntegrationTest from contentctl.objects.data_source import DataSource from contentctl.objects.base_test_result import TestResultStatus - +from contentctl.objects.drilldown import Drilldown, DRILLDOWN_SEARCH_PLACEHOLDER from contentctl.objects.enums import ProvidingTechnology from contentctl.enrichments.cve_enrichment import CveEnrichmentObj import datetime @@ -67,6 +67,16 @@ class Detection_Abstract(SecurityContentObject): search: str = Field(...) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) + explanation: None | str = Field( + default=None, + exclude=True, #Don't serialize this value when dumping the object + description="Provide an explanation to be included " + "in the 'Explanation' field of the Detection in " + "the Use Case Library. If this field is not " + "defined in the YML, it will default to the " + "value of the 'description' field when " + "serialized in analyticstories_detections.j2", + ) enabled_by_default: bool = False file_path: FilePath = Field(...) @@ -80,6 +90,7 @@ class Detection_Abstract(SecurityContentObject): test_groups: list[TestGroup] = [] data_source_objects: list[DataSource] = [] + drilldown_searches: list[Drilldown] = Field(default=[], description="A list of Drilldowns that should be included with this search") def get_conf_stanza_name(self, app:CustomApp)->str: stanza_name = CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name) @@ -157,6 +168,7 @@ def adjust_tests_and_groups(self) -> None: the model from the list of unit tests. Also, preemptively skips all manual tests, as well as tests for experimental/deprecated detections and Correlation type detections. """ + # Since ManualTest and UnitTest are not differentiable without looking at the manual_test # tag, Pydantic builds all tests as UnitTest objects. If we see the manual_test flag, we # convert these to ManualTest @@ -275,6 +287,7 @@ def annotations(self) -> dict[str, Union[List[str], int, str]]: annotations_dict["cve"] = self.tags.cve annotations_dict["impact"] = self.tags.impact annotations_dict["type"] = self.type + annotations_dict["type_list"] = [self.type] # annotations_dict["version"] = self.version annotations_dict["data_source"] = self.data_source @@ -552,6 +565,46 @@ def model_post_init(self, __context: Any) -> None: # Derive TestGroups and IntegrationTests, adjust for ManualTests, skip as needed self.adjust_tests_and_groups() + # Ensure that if there is at least 1 drilldown, at least + # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. + # This is presently a requirement when 1 or more drilldowns are added to a detection. + # Note that this is only required for production searches that are not hunting + + if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + #No additional check need to happen on the potential drilldowns. + pass + else: + found_placeholder = False + if len(self.drilldown_searches) < 2: + raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + for drilldown in self.drilldown_searches: + if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: + found_placeholder = True + if not found_placeholder: + raise ValueError("Detection has one or more drilldown_searches, but none of them " + f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + "if drilldown_searches are defined.'") + + # Update the search fields with the original search, if required + for drilldown in self.drilldown_searches: + drilldown.perform_search_substitutions(self) + + #For experimental purposes, add the default drilldowns + #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) + + @property + def drilldowns_in_JSON(self) -> list[dict[str,str]]: + """This function is required for proper JSON + serializiation of drilldowns to occur in savedsearches.conf. + It returns the list[Drilldown] as a list[dict]. + Without this function, the jinja template is unable + to convert list[Drilldown] to JSON + + Returns: + list[dict[str,str]]: List of Drilldowns dumped to dict format + """ + return [drilldown.model_dump() for drilldown in self.drilldown_searches] + @field_validator('lookups', mode="before") @classmethod def getDetectionLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]: @@ -636,6 +689,7 @@ def get_content_dependencies(self) -> list[SecurityContentObject]: objects: list[SecurityContentObject] = [] objects += self.macros objects += self.lookups + objects += self.data_source_objects return objects @field_validator("deployment", mode="before") @@ -778,6 +832,45 @@ def search_observables_exist_validate(self): # Found everything return self + @field_validator("tests", mode="before") + def ensure_yml_test_is_unittest(cls, v:list[dict]): + """The typing for the tests field allows it to be one of + a number of different types of tests. However, ONLY + UnitTest should be allowed to be defined in the YML + file. If part of the UnitTest defined in the YML + is incorrect, such as the attack_data file, then + it will FAIL to be instantiated as a UnitTest and + may instead be instantiated as a different type of + test, such as IntegrationTest (since that requires + less fields) which is incorrect. Ensure that any + raw data read from the YML can actually construct + a valid UnitTest and, if not, return errors right + away instead of letting Pydantic try to construct + it into a different type of test + + Args: + v (list[dict]): list of dicts read from the yml. + Each one SHOULD be a valid UnitTest. If we cannot + construct a valid unitTest from it, a ValueError should be raised + + Returns: + _type_: The input of the function, assuming no + ValueError is raised. + """ + valueErrors:list[ValueError] = [] + for unitTest in v: + #This raises a ValueError on a failed UnitTest. + try: + UnitTest.model_validate(unitTest) + except ValueError as e: + valueErrors.append(e) + if len(valueErrors): + raise ValueError(valueErrors) + # All of these can be constructred as UnitTests with no + # Exceptions, so let the normal flow continue + return v + + @field_validator("tests") def tests_validate( cls, diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index f93602f1..af1b3674 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -33,8 +33,7 @@ # TODO (#266): disable the use_enum_values configuration class SecurityContentObject_Abstract(BaseModel, abc.ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True) - + model_config = ConfigDict(use_enum_values=True,validate_default=True,extra="forbid") name: str = Field(...,max_length=99) author: str = Field(...,max_length=255) date: datetime.date = Field(...) diff --git a/contentctl/objects/alert_action.py b/contentctl/objects/alert_action.py index f2f745d4..d2855292 100644 --- a/contentctl/objects/alert_action.py +++ b/contentctl/objects/alert_action.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import BaseModel, model_serializer +from pydantic import BaseModel, model_serializer, ConfigDict from typing import Optional from contentctl.objects.deployment_email import DeploymentEmail @@ -9,6 +9,7 @@ from contentctl.objects.deployment_phantom import DeploymentPhantom class AlertAction(BaseModel): + model_config = ConfigDict(extra="forbid") email: Optional[DeploymentEmail] = None notable: Optional[DeploymentNotable] = None rba: Optional[DeploymentRBA] = DeploymentRBA() diff --git a/contentctl/objects/atomic.py b/contentctl/objects/atomic.py index a723304d..7e79227c 100644 --- a/contentctl/objects/atomic.py +++ b/contentctl/objects/atomic.py @@ -41,6 +41,7 @@ class InputArgumentType(StrEnum): Url = "Url" class AtomicExecutor(BaseModel): + model_config = ConfigDict(extra="forbid") name: str elevation_required: Optional[bool] = False #Appears to be optional command: Optional[str] = None diff --git a/contentctl/objects/base_test.py b/contentctl/objects/base_test.py index 20e681cf..8a377dfc 100644 --- a/contentctl/objects/base_test.py +++ b/contentctl/objects/base_test.py @@ -2,7 +2,7 @@ from typing import Union from abc import ABC, abstractmethod -from pydantic import BaseModel +from pydantic import BaseModel,ConfigDict from contentctl.objects.base_test_result import BaseTestResult @@ -21,6 +21,7 @@ def __str__(self) -> str: # TODO (#224): enforce distinct test names w/in detections class BaseTest(BaseModel, ABC): + model_config = ConfigDict(extra="forbid") """ A test case for a detection """ diff --git a/contentctl/objects/base_test_result.py b/contentctl/objects/base_test_result.py index 1e1b287c..d29f93cb 100644 --- a/contentctl/objects/base_test_result.py +++ b/contentctl/objects/base_test_result.py @@ -1,8 +1,8 @@ from typing import Union, Any from enum import Enum -from pydantic import BaseModel -from splunklib.data import Record +from pydantic import ConfigDict, BaseModel +from splunklib.data import Record # type: ignore from contentctl.helper.utils import Utils @@ -53,11 +53,11 @@ class BaseTestResult(BaseModel): # The Splunk endpoint URL sid_link: Union[None, str] = None - class Config: - validate_assignment = True - - # Needed to allow for embedding of Exceptions in the model - arbitrary_types_allowed = True + # Needed to allow for embedding of Exceptions in the model + model_config = ConfigDict( + validate_assignment=True, + arbitrary_types_allowed=True + ) @property def passed(self) -> bool: diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index 5dc59d8f..a41acbb4 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import Annotated, Optional, List,Any -from pydantic import field_validator, ValidationInfo, Field, model_serializer +from typing import Annotated, List,Any +from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.enums import DataModel @@ -15,7 +15,6 @@ class Baseline(SecurityContentObject): name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) type: Annotated[str,Field(pattern="^Baseline$")] = Field(...) - datamodel: Optional[List[DataModel]] = None search: str = Field(..., min_length=4) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) @@ -34,6 +33,10 @@ def get_conf_stanza_name(self, app:CustomApp)->str: def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment: return Deployment.getDeployment(v,info) + @computed_field + @property + def datamodel(self) -> List[DataModel]: + return [dm for dm in DataModel if dm.value in self.search] @model_serializer def serialize_model(self): diff --git a/contentctl/objects/baseline_tags.py b/contentctl/objects/baseline_tags.py index ea979664..db5f8048 100644 --- a/contentctl/objects/baseline_tags.py +++ b/contentctl/objects/baseline_tags.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer +from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer, ConfigDict from typing import List, Any, Union from contentctl.objects.story import Story @@ -12,12 +12,12 @@ class BaselineTags(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: list[Story] = Field(...) #deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION') # TODO (#223): can we remove str from the possible types here? detections: List[Union[Detection,str]] = Field(...) product: List[SecurityContentProductName] = Field(...,min_length=1) - required_fields: List[str] = Field(...,min_length=1) security_domain: SecurityDomain = Field(...) @@ -33,7 +33,6 @@ def serialize_model(self): "analytic_story": [story.name for story in self.analytic_story], "detections": [detection.name for detection in self.detections if isinstance(detection,Detection)], "product": self.product, - "required_fields":self.required_fields, "security_domain":self.security_domain, "deployments": None } diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 0b262c55..c41b93ea 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -35,7 +35,7 @@ # TODO (#266): disable the use_enum_values configuration class App_Base(BaseModel,ABC): - model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) + model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True, extra='forbid') uid: Optional[int] = Field(default=None) title: str = Field(description="Human-readable name used by the app. This can have special characters.") appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. " @@ -159,8 +159,6 @@ def getApp(self, config:test, stage_file=True)->str: verbose_print=True) return str(destination) - - # TODO (#266): disable the use_enum_values configuration class Config_Base(BaseModel): model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) @@ -288,7 +286,6 @@ def getAPIPath(self)->pathlib.Path: def getAppTemplatePath(self)->pathlib.Path: return self.path/"app_template" - class StackType(StrEnum): @@ -297,6 +294,7 @@ class StackType(StrEnum): class inspect(build): + splunk_api_username: str = Field( description="Splunk API username used for appinspect and Splunkbase downloads." ) @@ -311,6 +309,16 @@ class inspect(build): "should be enabled." ) ) + suppress_missing_content_exceptions: bool = Field( + default=False, + description=( + "Suppress exceptions during metadata validation if a detection that existed in " + "the previous build does not exist in this build. This is to ensure that content " + "is not accidentally removed. In order to support testing both public and private " + "content, this warning can be suppressed. If it is suppressed, it will still be " + "printed out as a warning." + ) + ) enrichments: bool = Field( default=True, description=( @@ -952,15 +960,15 @@ def check_environment_variable_for_config(cls, v:List[Infrastructure]): index+=1 - class release_notes(Config_Base): old_tag:Optional[str] = Field(None, description="Name of the tag to diff against to find new content. " "If it is not supplied, then it will be inferred as the " "second newest tag at runtime.") new_tag:Optional[str] = Field(None, description="Name of the tag containing new content. If it is not supplied," " then it will be inferred as the newest tag at runtime.") - latest_branch:Optional[str] = Field(None, description="Branch for which we are generating release notes") - + latest_branch:Optional[str] = Field(None, description="Branch name for which we are generating release notes for") + compare_against:Optional[str] = Field(default="develop", description="Branch name for which we are comparing the files changes against") + def releaseNotesFilename(self, filename:str)->pathlib.Path: #Assume that notes are written to dist/. This does not respect build_dir since that is #only a member of build @@ -1034,6 +1042,4 @@ def ensureNewTagOrLatestBranch(self): # raise ValueError("The latest_branch '{self.latest_branch}' was not found in the repository") - # return self - - + # return self \ No newline at end of file diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index a0b25da9..c64eed6b 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -1,10 +1,11 @@ import logging import time import json -from typing import Union, Optional, Any +from typing import Any from enum import Enum +from functools import cached_property -from pydantic import BaseModel, validator, Field, PrivateAttr +from pydantic import ConfigDict, BaseModel, computed_field, Field, PrivateAttr from splunklib.results import JSONResultsReader, Message # type: ignore from splunklib.binding import HTTPError, ResponseReader # type: ignore import splunklib.client as splunklib # type: ignore @@ -15,7 +16,7 @@ from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.integration_test_result import IntegrationTestResult from contentctl.actions.detection_testing.progress_bar import ( - format_pbar_string, + format_pbar_string, # type: ignore TestReportingType, TestingStates ) @@ -178,13 +179,14 @@ class PbarData(BaseModel): :param fq_test_name: the fully qualifed (fq) test name (":") used for logging :param start_time: the start time used for logging """ - pbar: tqdm + pbar: tqdm # type: ignore fq_test_name: str start_time: float - class Config: - # needed to support the tqdm type - arbitrary_types_allowed = True + # needed to support the tqdm type + model_config = ConfigDict( + arbitrary_types_allowed=True + ) class CorrelationSearch(BaseModel): @@ -197,143 +199,110 @@ class CorrelationSearch(BaseModel): :param pbar_data: the encapsulated info needed for logging w/ pbar :param test_index: the index attack data is forwarded to for testing (optionally used in cleanup) """ - ## The following three fields are explicitly needed at instantiation # noqa: E266 - # the detection associated with the correlation search (e.g. "Windows Modify Registry EnableLinkedConnections") - detection: Detection + detection: Detection = Field(...) # a Service instance representing a connection to a Splunk instance - service: splunklib.Service + service: splunklib.Service = Field(...) # the encapsulated info needed for logging w/ pbar - pbar_data: PbarData - - ## The following field is optional for instantiation # noqa: E266 + pbar_data: PbarData = Field(...) # The index attack data is sent to; can be None if we are relying on the caller to do our # cleanup of this index - test_index: Optional[str] = Field(default=None, min_length=1) - - ## All remaining fields can be derived from other fields or have intentional defaults that # noqa: E266 - ## should not be changed (validators should prevent instantiating some of these fields directly # noqa: E266 - ## to prevent undefined behavior) # noqa: E266 + test_index: str | None = Field(default=None, min_length=1) # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not # to conflict w/ tqdm) - logger: logging.Logger = Field(default_factory=get_logger) - - # The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule") - name: Optional[str] = None - - # The path to the saved search on the Splunk instance - splunk_path: Optional[str] = None - - # A model of the saved search as provided by splunklib - saved_search: Optional[splunklib.SavedSearch] = None + logger: logging.Logger = Field(default_factory=get_logger, init=False) # The set of indexes to clear on cleanup - indexes_to_purge: set[str] = set() + indexes_to_purge: set[str] = Field(default=set(), init=False) # The risk analysis adaptive response action (if defined) - risk_analysis_action: Union[RiskAnalysisAction, None] = None + _risk_analysis_action: RiskAnalysisAction | None = PrivateAttr(default=None) # The notable adaptive response action (if defined) - notable_action: Union[NotableAction, None] = None + _notable_action: NotableAction | None = PrivateAttr(default=None) # The list of risk events found - _risk_events: Optional[list[RiskEvent]] = PrivateAttr(default=None) + _risk_events: list[RiskEvent] | None = PrivateAttr(default=None) # The list of notable events found - _notable_events: Optional[list[NotableEvent]] = PrivateAttr(default=None) + _notable_events: list[NotableEvent] | None = PrivateAttr(default=None) - class Config: - # needed to allow fields w/ types like SavedSearch - arbitrary_types_allowed = True - # We want to have more ridgid typing - extra = 'forbid' + # Need arbitrary types to allow fields w/ types like SavedSearch; we also want to forbid + # unexpected fields + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra='forbid' + ) - @validator("name", always=True) - @classmethod - def _convert_detection_to_search_name(cls, v, values) -> str: - """ - Validate name and derive if None - """ - if "detection" not in values: - raise ValueError("detection missing; name is dependent on detection") + def model_post_init(self, __context: Any) -> None: + super().model_post_init(__context) - expected_name = f"ESCU - {values['detection'].name} - Rule" - if v is not None and v != expected_name: - raise ValueError( - "name must be derived from detection; leave as None and it will be derived automatically" - ) - return expected_name + # Parse the initial values for the risk/notable actions + self._parse_risk_and_notable_actions() - @validator("splunk_path", always=True) - @classmethod - def _derive_splunk_path(cls, v, values) -> str: + @computed_field + @cached_property + def name(self) -> str: """ - Validate splunk_path and derive if None + The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule") + + :returns: the search name + :rtype: str """ - if "name" not in values: - raise ValueError("name missing; splunk_path is dependent on name") + return f"ESCU - {self.detection.name} - Rule" - expected_path = f"saved/searches/{values['name']}" - if v is not None and v != expected_path: - raise ValueError( - "splunk_path must be derived from name; leave as None and it will be derived automatically" - ) - return f"saved/searches/{values['name']}" + @computed_field + @cached_property + def splunk_path(self) -> str: + """ + The path to the saved search on the Splunk instance - @validator("saved_search", always=True) - @classmethod - def _instantiate_saved_search(cls, v, values) -> str: + :returns: the search path + :rtype: str """ - Ensure saved_search was initialized as None and derive + return f"saved/searches/{self.name}" + + @computed_field + @cached_property + def saved_search(self) -> splunklib.SavedSearch: """ - if "splunk_path" not in values or "service" not in values: - raise ValueError("splunk_path or service missing; saved_search is dependent on both") + A model of the saved search as provided by splunklib - if v is not None: - raise ValueError( - "saved_search must be derived from the service and splunk_path; leave as None and it will be derived " - "automatically" - ) + :returns: the SavedSearch object + :rtype: :class:`splunklib.client.SavedSearch` + """ return splunklib.SavedSearch( - values['service'], - values['splunk_path'], + self.service, + self.splunk_path, ) - @validator("risk_analysis_action", always=True) - @classmethod - def _init_risk_analysis_action(cls, v, values) -> Optional[RiskAnalysisAction]: - """ - Initialize risk_analysis_action + # TODO (cmcginley): need to make this refreshable + @computed_field + @property + def risk_analysis_action(self) -> RiskAnalysisAction | None: """ - if "saved_search" not in values: - raise ValueError("saved_search missing; risk_analysis_action is dependent on saved_search") + The risk analysis adaptive response action (if defined) - if v is not None: - raise ValueError( - "risk_analysis_action must be derived from the saved_search; leave as None and it will be derived " - "automatically" - ) - return CorrelationSearch._get_risk_analysis_action(values['saved_search'].content) - - @validator("notable_action", always=True) - @classmethod - def _init_notable_action(cls, v, values) -> Optional[NotableAction]: + :returns: the RiskAnalysisAction object, if it exists + :rtype: :class:`contentctl.objects.risk_analysis_action.RiskAnalysisAction` | None """ - Initialize notable_action + return self._risk_analysis_action + + # TODO (cmcginley): need to make this refreshable + @computed_field + @property + def notable_action(self) -> NotableAction | None: """ - if "saved_search" not in values: - raise ValueError("saved_search missing; notable_action is dependent on saved_search") + The notable adaptive response action (if defined) - if v is not None: - raise ValueError( - "notable_action must be derived from the saved_search; leave as None and it will be derived " - "automatically" - ) - return CorrelationSearch._get_notable_action(values['saved_search'].content) + :returns: the NotableAction object, if it exists + :rtype: :class:`contentctl.objects.notable_action.NotableAction` | None + """ + return self._notable_action @property def earliest_time(self) -> str: @@ -393,7 +362,7 @@ def has_notable_action(self) -> bool: return self.notable_action is not None @staticmethod - def _get_risk_analysis_action(content: dict[str, Any]) -> Optional[RiskAnalysisAction]: + def _get_risk_analysis_action(content: dict[str, Any]) -> RiskAnalysisAction | None: """ Given the saved search content, parse the risk analysis action :param content: a dict of strings to values @@ -407,7 +376,7 @@ def _get_risk_analysis_action(content: dict[str, Any]) -> Optional[RiskAnalysisA return None @staticmethod - def _get_notable_action(content: dict[str, Any]) -> Optional[NotableAction]: + def _get_notable_action(content: dict[str, Any]) -> NotableAction | None: """ Given the saved search content, parse the notable action :param content: a dict of strings to values @@ -431,10 +400,6 @@ def _get_relevant_observables(observables: list[Observable]) -> list[Observable] relevant.append(observable) return relevant - # TODO (PEX-484): ideally, we could handle this and the following init w/ a call to - # model_post_init, so that all the logic is encapsulated w/in _parse_risk_and_notable_actions - # but that is a pydantic v2 feature (see the init validators for risk/notable actions): - # https://docs.pydantic.dev/latest/api/base_model/#pydantic.main.BaseModel.model_post_init def _parse_risk_and_notable_actions(self) -> None: """Parses the risk/notable metadata we care about from self.saved_search.content @@ -445,12 +410,12 @@ def _parse_risk_and_notable_actions(self) -> None: unpacked to be anything other than a singleton """ # grab risk details if present - self.risk_analysis_action = CorrelationSearch._get_risk_analysis_action( + self._risk_analysis_action = CorrelationSearch._get_risk_analysis_action( self.saved_search.content # type: ignore ) # grab notable details if present - self.notable_action = CorrelationSearch._get_notable_action(self.saved_search.content) # type: ignore + self._notable_action = CorrelationSearch._get_notable_action(self.saved_search.content) # type: ignore def refresh(self) -> None: """Refreshes the metadata in the SavedSearch entity, and re-parses the fields we care about @@ -738,7 +703,7 @@ def validate_risk_events(self) -> None: # TODO (#250): Re-enable and refactor code that validates the specific risk counts # Validate risk events in aggregate; we should have an equal amount of risk events for each # relevant observable, and the total count should match the total number of events - # individual_count: Optional[int] = None + # individual_count: int | None = None # total_count = 0 # for observable_str in observable_counts: # self.logger.debug( @@ -802,7 +767,7 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP.value, raise_on_exc: boo ) # initialize result as None - result: Optional[IntegrationTestResult] = None + result: IntegrationTestResult | None = None # keep track of time slept and number of attempts for exponential backoff (base 2) elapsed_sleep_time = 0 diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index 868bdd51..ed8a8f86 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -1,8 +1,7 @@ from __future__ import annotations from typing import Optional, Any -from pydantic import Field, HttpUrl, model_serializer, BaseModel +from pydantic import Field, HttpUrl, model_serializer, BaseModel, ConfigDict from contentctl.objects.security_content_object import SecurityContentObject -from contentctl.objects.event_source import EventSource class TA(BaseModel): @@ -10,15 +9,16 @@ class TA(BaseModel): url: HttpUrl | None = None version: str class DataSource(SecurityContentObject): + model_config = ConfigDict(extra="forbid") source: str = Field(...) sourcetype: str = Field(...) separator: Optional[str] = None configuration: Optional[str] = None supported_TA: list[TA] = [] - fields: Optional[list] = None - field_mappings: Optional[list] = None - convert_to_log_source: Optional[list] = None - example_log: Optional[str] = None + fields: None | list = None + field_mappings: None | list = None + convert_to_log_source: None | list = None + example_log: None | str = None @model_serializer diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index 832c048d..8fd264b6 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt +from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt, ConfigDict from typing import Any import uuid import datetime @@ -11,6 +11,7 @@ class Deployment(SecurityContentObject): + model_config = ConfigDict(extra="forbid") #id: str = None #date: str = None #author: str = None @@ -72,7 +73,6 @@ def serialize_model(self): "tags": self.tags } - #Combine fields from this model with fields from parent model.update(super_fields) diff --git a/contentctl/objects/deployment_email.py b/contentctl/objects/deployment_email.py index a607502c..1d1269fe 100644 --- a/contentctl/objects/deployment_email.py +++ b/contentctl/objects/deployment_email.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentEmail(BaseModel): + model_config = ConfigDict(extra="forbid") message: str subject: str to: str \ No newline at end of file diff --git a/contentctl/objects/deployment_notable.py b/contentctl/objects/deployment_notable.py index b6e2c463..7f064b43 100644 --- a/contentctl/objects/deployment_notable.py +++ b/contentctl/objects/deployment_notable.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from typing import List class DeploymentNotable(BaseModel): + model_config = ConfigDict(extra="forbid") rule_description: str rule_title: str nes_fields: List[str] \ No newline at end of file diff --git a/contentctl/objects/deployment_phantom.py b/contentctl/objects/deployment_phantom.py index 11df2feb..1d4a9975 100644 --- a/contentctl/objects/deployment_phantom.py +++ b/contentctl/objects/deployment_phantom.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentPhantom(BaseModel): + model_config = ConfigDict(extra="forbid") cam_workers : str label : str phantom_server : str diff --git a/contentctl/objects/deployment_rba.py b/contentctl/objects/deployment_rba.py index b3412b3f..58917c70 100644 --- a/contentctl/objects/deployment_rba.py +++ b/contentctl/objects/deployment_rba.py @@ -1,6 +1,7 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentRBA(BaseModel): + model_config = ConfigDict(extra="forbid") enabled: bool = False \ No newline at end of file diff --git a/contentctl/objects/deployment_scheduling.py b/contentctl/objects/deployment_scheduling.py index 6c5a75a8..b21673d8 100644 --- a/contentctl/objects/deployment_scheduling.py +++ b/contentctl/objects/deployment_scheduling.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentScheduling(BaseModel): + model_config = ConfigDict(extra="forbid") cron_schedule: str earliest_time: str latest_time: str diff --git a/contentctl/objects/deployment_slack.py b/contentctl/objects/deployment_slack.py index 294836e2..03cf5ebb 100644 --- a/contentctl/objects/deployment_slack.py +++ b/contentctl/objects/deployment_slack.py @@ -1,7 +1,8 @@ from __future__ import annotations -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DeploymentSlack(BaseModel): + model_config = ConfigDict(extra="forbid") channel: str message: str \ No newline at end of file diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index c8dce678..185bc190 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -38,13 +38,12 @@ # TODO (#266): disable the use_enum_values configuration class DetectionTags(BaseModel): # detection spec - model_config = ConfigDict(use_enum_values=True, validate_default=False) + model_config = ConfigDict(use_enum_values=True,validate_default=False, extra='forbid') analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) - - confidence: NonNegativeInt = Field(..., le=100) - impact: NonNegativeInt = Field(..., le=100) - + group: list[str] = [] + confidence: NonNegativeInt = Field(...,le=100) + impact: NonNegativeInt = Field(...,le=100) @computed_field @property def risk_score(self) -> int: @@ -74,12 +73,11 @@ def severity(self)->RiskSeverity: observable: List[Observable] = [] message: str = Field(...) product: list[SecurityContentProductName] = Field(..., min_length=1) - required_fields: list[str] = Field(min_length=1) throttling: Optional[Throttling] = None security_domain: SecurityDomain = Field(...) cve: List[CVE_TYPE] = [] atomic_guid: List[AtomicTest] = [] - drilldown_search: Optional[str] = None + # enrichment mitre_attack_enrichments: List[MitreAttackEnrichment] = Field([], validate_default=True) @@ -114,7 +112,7 @@ def cis20(self) -> list[Cis18Value]: # TODO (#268): Validate manual_test has length > 0 if not None manual_test: Optional[str] = None - + # The following validator is temporarily disabled pending further discussions # @validator('message') # def validate_message(cls,v,values): diff --git a/contentctl/objects/drilldown.py b/contentctl/objects/drilldown.py new file mode 100644 index 00000000..3fe41e7c --- /dev/null +++ b/contentctl/objects/drilldown.py @@ -0,0 +1,70 @@ +from __future__ import annotations +from pydantic import BaseModel, Field, model_serializer +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from contentctl.objects.detection import Detection +from contentctl.objects.enums import AnalyticsType +DRILLDOWN_SEARCH_PLACEHOLDER = "%original_detection_search%" +EARLIEST_OFFSET = "$info_min_time$" +LATEST_OFFSET = "$info_max_time$" +RISK_SEARCH = "index = risk starthoursago = 168 endhoursago = 0 | stats count values(search_name) values(risk_message) values(analyticstories) values(annotations._all) values(annotations.mitre_attack.mitre_tactic) " + +class Drilldown(BaseModel): + name: str = Field(..., description="The name of the drilldown search", min_length=5) + search: str = Field(..., description="The text of a drilldown search. This must be valid SPL.", min_length=1) + earliest_offset:None | str = Field(..., + description="Earliest offset time for the drilldown search. " + f"The most common value for this field is '{EARLIEST_OFFSET}', " + "but it is NOT the default value and must be supplied explicitly.", + min_length= 1) + latest_offset:None | str = Field(..., + description="Latest offset time for the driolldown search. " + f"The most common value for this field is '{LATEST_OFFSET}', " + "but it is NOT the default value and must be supplied explicitly.", + min_length= 1) + + @classmethod + def constructDrilldownsFromDetection(cls, detection: Detection) -> list[Drilldown]: + victim_observables = [o for o in detection.tags.observable if o.role[0] == "Victim"] + if len(victim_observables) == 0 or detection.type == AnalyticsType.Hunting: + # No victims, so no drilldowns + return [] + print(f"Adding default drilldowns for [{detection.name}]") + variableNamesString = ' and '.join([f"${o.name}$" for o in victim_observables]) + nameField = f"View the detection results for {variableNamesString}" + appendedSearch = " | search " + ' '.join([f"{o.name} = ${o.name}$" for o in victim_observables]) + search_field = f"{detection.search}{appendedSearch}" + detection_results = cls(name=nameField, earliest_offset=EARLIEST_OFFSET, latest_offset=LATEST_OFFSET, search=search_field) + + + nameField = f"View risk events for the last 7 days for {variableNamesString}" + fieldNamesListString = ', '.join([o.name for o in victim_observables]) + search_field = f"{RISK_SEARCH}by {fieldNamesListString} {appendedSearch}" + risk_events_last_7_days = cls(name=nameField, earliest_offset=None, latest_offset=None, search=search_field) + + return [detection_results,risk_events_last_7_days] + + + def perform_search_substitutions(self, detection:Detection)->None: + """Replaces the field DRILLDOWN_SEARCH_PLACEHOLDER (%original_detection_search%) + with the search contained in the detection. We do this so that the YML does not + need the search copy/pasted from the search field into the drilldown object. + + Args: + detection (Detection): Detection to be used to update the search field of the drilldown + """ + self.search = self.search.replace(DRILLDOWN_SEARCH_PLACEHOLDER, detection.search) + + + @model_serializer + def serialize_model(self) -> dict[str,str]: + #Call serializer for parent + model:dict[str,str] = {} + + model['name'] = self.name + model['search'] = self.search + if self.earliest_offset is not None: + model['earliest_offset'] = self.earliest_offset + if self.latest_offset is not None: + model['latest_offset'] = self.latest_offset + return model \ No newline at end of file diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index d7072ecc..333ef358 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -197,21 +197,21 @@ class KillChainPhase(str, enum.Enum): class DataSource(str,enum.Enum): OSQUERY_ES_PROCESS_EVENTS = "OSQuery ES Process Events" POWERSHELL_4104 = "Powershell 4104" - SYSMON_EVENT_ID_1 = "Sysmon Event ID 1" - SYSMON_EVENT_ID_10 = "Sysmon Event ID 10" - SYSMON_EVENT_ID_11 = "Sysmon Event ID 11" - SYSMON_EVENT_ID_13 = "Sysmon Event ID 13" - SYSMON_EVENT_ID_15 = "Sysmon Event ID 15" - SYSMON_EVENT_ID_20 = "Sysmon Event ID 20" - SYSMON_EVENT_ID_21 = "Sysmon Event ID 21" - SYSMON_EVENT_ID_22 = "Sysmon Event ID 22" - SYSMON_EVENT_ID_23 = "Sysmon Event ID 23" - SYSMON_EVENT_ID_3 = "Sysmon Event ID 3" - SYSMON_EVENT_ID_5 = "Sysmon Event ID 5" - SYSMON_EVENT_ID_6 = "Sysmon Event ID 6" - SYSMON_EVENT_ID_7 = "Sysmon Event ID 7" - SYSMON_EVENT_ID_8 = "Sysmon Event ID 8" - SYSMON_EVENT_ID_9 = "Sysmon Event ID 9" + SYSMON_EVENT_ID_1 = "Sysmon EventID 1" + SYSMON_EVENT_ID_3 = "Sysmon EventID 3" + SYSMON_EVENT_ID_5 = "Sysmon EventID 5" + SYSMON_EVENT_ID_6 = "Sysmon EventID 6" + SYSMON_EVENT_ID_7 = "Sysmon EventID 7" + SYSMON_EVENT_ID_8 = "Sysmon EventID 8" + SYSMON_EVENT_ID_9 = "Sysmon EventID 9" + SYSMON_EVENT_ID_10 = "Sysmon EventID 10" + SYSMON_EVENT_ID_11 = "Sysmon EventID 11" + SYSMON_EVENT_ID_13 = "Sysmon EventID 13" + SYSMON_EVENT_ID_15 = "Sysmon EventID 15" + SYSMON_EVENT_ID_20 = "Sysmon EventID 20" + SYSMON_EVENT_ID_21 = "Sysmon EventID 21" + SYSMON_EVENT_ID_22 = "Sysmon EventID 22" + SYSMON_EVENT_ID_23 = "Sysmon EventID 23" WINDOWS_SECURITY_4624 = "Windows Security 4624" WINDOWS_SECURITY_4625 = "Windows Security 4625" WINDOWS_SECURITY_4648 = "Windows Security 4648" @@ -330,7 +330,6 @@ class SecurityDomain(str, enum.Enum): IDENTITY = "identity" ACCESS = "access" AUDIT = "audit" - CLOUD = "cloud" class AssetType(str, enum.Enum): AWS_ACCOUNT = "AWS Account" diff --git a/contentctl/objects/event_source.py b/contentctl/objects/event_source.py deleted file mode 100644 index 0ed61979..00000000 --- a/contentctl/objects/event_source.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations -from typing import Union, Optional, List -from pydantic import BaseModel, Field - -from contentctl.objects.security_content_object import SecurityContentObject - -class EventSource(SecurityContentObject): - fields: Optional[list[str]] = None - field_mappings: Optional[list[dict]] = None - convert_to_log_source: Optional[list[dict]] = None - example_log: Optional[str] = None diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 293e3331..6e058783 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -16,13 +16,10 @@ class Investigation(SecurityContentObject): model_config = ConfigDict(use_enum_values=True,validate_default=False) type: str = Field(...,pattern="^Investigation$") - datamodel: list[DataModel] = Field(...) name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) search: str = Field(...) how_to_implement: str = Field(...) known_false_positives: str = Field(...) - - tags: InvestigationTags # enrichment @@ -38,6 +35,11 @@ def inputs(self)->List[str]: return inputs + @computed_field + @property + def datamodel(self) -> List[DataModel]: + return [dm for dm in DataModel if dm.value in self.search] + @computed_field @property def lowercase_name(self)->str: diff --git a/contentctl/objects/investigation_tags.py b/contentctl/objects/investigation_tags.py index 6db99eff..c4b812e6 100644 --- a/contentctl/objects/investigation_tags.py +++ b/contentctl/objects/investigation_tags.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import List -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer +from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer,ConfigDict from contentctl.objects.story import Story from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain class InvestigationTags(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: List[Story] = Field([],min_length=1) product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1) - required_fields: List[str] = Field(min_length=1) security_domain: SecurityDomain = Field(...) @@ -23,7 +23,6 @@ def serialize_model(self): model= { "analytic_story": [story.name for story in self.analytic_story], "product": self.product, - "required_fields": self.required_fields, "security_domain": self.security_domain, } diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index e37e60e9..bc699f49 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -33,6 +33,7 @@ class Lookup(SecurityContentObject): default_match: Optional[bool] = None match_type: Optional[str] = None min_matches: Optional[int] = None + max_matches: Optional[int] = None case_sensitive_match: Optional[bool] = None # TODO: Add id field to all lookup ymls id: uuid.UUID = Field(default_factory=uuid.uuid4) @@ -52,6 +53,7 @@ def serialize_model(self): "default_match": "true" if self.default_match is True else "false", "match_type": self.match_type, "min_matches": self.min_matches, + "max_matches": self.max_matches, "case_sensitive_match": "true" if self.case_sensitive_match is True else "false", "collection": self.collection, "fields_list": self.fields_list diff --git a/contentctl/objects/macro.py b/contentctl/objects/macro.py index 48daf602..ba5faa8f 100644 --- a/contentctl/objects/macro.py +++ b/contentctl/objects/macro.py @@ -10,7 +10,6 @@ from contentctl.input.director import DirectorOutputDto from contentctl.objects.security_content_object import SecurityContentObject - #The following macros are included in commonly-installed apps. #As such, we will ignore if they are missing from our app. #Included in @@ -55,10 +54,15 @@ def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[st #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here```` #then there is a small edge case where the regex below does not work properly. If that is #the case, we edit the search slightly to insert a space - text_field = re.sub(r"\`\`\`\`", r"` ```", text_field) - text_field = re.sub(r"\`\`\`.*?\`\`\`", " ", text_field) - + if re.findall(r"\`\`\`\`", text_field): + raise ValueError("Search contained four or more '`' characters in a row which is invalid SPL" + "This may have occurred when a macro was commented out.\n" + "Please ammend your search to remove the substring '````'") + # replace all the macros with a space + text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) + + macros_to_get = re.findall(r'`([^\s]+)`', text_field) #If macros take arguments, stop at the first argument. We just want the name of the macro macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get]) @@ -68,4 +72,3 @@ def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[st macros_to_get -= macros_to_ignore return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director) - \ No newline at end of file diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index 401774e9..85df2c4b 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -85,7 +85,7 @@ def standardize_contributors(cls, contributors:list[str] | None) -> list[str]: # TODO (#266): disable the use_enum_values configuration class MitreAttackEnrichment(BaseModel): - ConfigDict(use_enum_values=True) + ConfigDict(use_enum_values=True,extra='forbid') mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...) mitre_attack_technique: str = Field(...) mitre_attack_tactics: List[MitreTactics] = Field(...) diff --git a/contentctl/objects/notable_event.py b/contentctl/objects/notable_event.py index d28d4a62..51b9715d 100644 --- a/contentctl/objects/notable_event.py +++ b/contentctl/objects/notable_event.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel from contentctl.objects.detection import Detection @@ -11,10 +11,11 @@ class NotableEvent(BaseModel): # The search ID that found that generated this risk event orig_sid: str - class Config: - # Allowing fields that aren't explicitly defined to be passed since some of the risk event's - # fields vary depending on the SPL which generated them - extra = 'allow' + # Allowing fields that aren't explicitly defined to be passed since some of the risk event's + # fields vary depending on the SPL which generated them + model_config = ConfigDict( + extra='allow' + ) def validate_against_detection(self, detection: Detection) -> None: raise NotImplementedError() diff --git a/contentctl/objects/observable.py b/contentctl/objects/observable.py index daf7a70b..81b04922 100644 --- a/contentctl/objects/observable.py +++ b/contentctl/objects/observable.py @@ -1,8 +1,9 @@ -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, ConfigDict from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, RBA_OBSERVABLE_ROLE_MAPPING class Observable(BaseModel): + model_config = ConfigDict(extra="forbid") name: str type: str role: list[str] diff --git a/contentctl/objects/playbook_tags.py b/contentctl/objects/playbook_tags.py index fd4a21e6..10d90ac1 100644 --- a/contentctl/objects/playbook_tags.py +++ b/contentctl/objects/playbook_tags.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, Optional, List -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field,ConfigDict import enum from contentctl.objects.detection import Detection @@ -36,6 +36,7 @@ class DefendTechnique(str,enum.Enum): D3_SRA = "D3-SRA" D3_RUAA = "D3-RUAA" class PlaybookTag(BaseModel): + model_config = ConfigDict(extra="forbid") analytic_story: Optional[list] = None detections: Optional[list] = None platform_tags: list[str] = Field(...,min_length=0) @@ -46,5 +47,8 @@ class PlaybookTag(BaseModel): use_cases: list[PlaybookUseCase] = Field([],min_length=0) defend_technique_id: Optional[List[DefendTechnique]] = None + labels:list[str] = [] + playbook_outputs:list[str] = [] + detection_objects: list[Detection] = [] \ No newline at end of file diff --git a/contentctl/objects/risk_analysis_action.py b/contentctl/objects/risk_analysis_action.py index e29939d3..2fa295e4 100644 --- a/contentctl/objects/risk_analysis_action.py +++ b/contentctl/objects/risk_analysis_action.py @@ -1,7 +1,7 @@ from typing import Any import json -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from contentctl.objects.risk_object import RiskObject from contentctl.objects.threat_object import ThreatObject @@ -21,11 +21,11 @@ class RiskAnalysisAction(BaseModel): risk_objects: list[RiskObject] message: str - @validator("message", always=True, pre=True) + @field_validator("message", mode="before") @classmethod - def _validate_message(cls, v, values) -> str: + def _validate_message(cls, v: Any) -> str: """ - Validate splunk_path and derive if None + Validate message and derive if None """ if v is None: raise ValueError( diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index 7d30d324..de98bd0b 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -1,7 +1,7 @@ import re +from functools import cached_property -from pydantic import BaseModel, Field, PrivateAttr, field_validator, computed_field - +from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field from contentctl.objects.errors import ValidationFailed from contentctl.objects.detection import Detection from contentctl.objects.observable import Observable @@ -85,10 +85,11 @@ class RiskEvent(BaseModel): # Private attribute caching the observable this RiskEvent is mapped to _matched_observable: Observable | None = PrivateAttr(default=None) - class Config: - # Allowing fields that aren't explicitly defined to be passed since some of the risk event's - # fields vary depending on the SPL which generated them - extra = "allow" + # Allowing fields that aren't explicitly defined to be passed since some of the risk event's + # fields vary depending on the SPL which generated them + model_config = ConfigDict( + extra="allow" + ) @field_validator("annotations_mitre_attack", "analyticstories", mode="before") @classmethod @@ -103,7 +104,7 @@ def _convert_str_value_to_singleton(cls, v: str | list[str]) -> list[str]: return [v] @computed_field - @property + @cached_property def source_field_name(self) -> str: """ A cached derivation of the source field name the risk event corresponds to in the relevant diff --git a/contentctl/objects/test_attack_data.py b/contentctl/objects/test_attack_data.py index 2c53df0b..5d5f9c80 100644 --- a/contentctl/objects/test_attack_data.py +++ b/contentctl/objects/test_attack_data.py @@ -1,8 +1,9 @@ from __future__ import annotations -from pydantic import BaseModel, HttpUrl, FilePath, Field +from pydantic import BaseModel, HttpUrl, FilePath, Field, ConfigDict class TestAttackData(BaseModel): + model_config = ConfigDict(extra="forbid") data: HttpUrl | FilePath = Field(...) # TODO - should source and sourcetype should be mapped to a list # of supported source and sourcetypes in a given environment? diff --git a/contentctl/objects/unit_test_baseline.py b/contentctl/objects/unit_test_baseline.py index 9ba49336..66a60594 100644 --- a/contentctl/objects/unit_test_baseline.py +++ b/contentctl/objects/unit_test_baseline.py @@ -1,9 +1,10 @@ -from pydantic import BaseModel +from pydantic import BaseModel,ConfigDict from typing import Union class UnitTestBaseline(BaseModel): + model_config = ConfigDict(extra="forbid") name: str file: str pass_condition: str diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 0d00cf64..e53aeba0 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -57,19 +57,26 @@ def writeHeaders(self) -> set[pathlib.Path]: pass - def writeAppConf(self)->set[pathlib.Path]: + + + def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: written_files:set[pathlib.Path] = set() - for output_app_path, template_name in [ ("default/app.conf", "app.conf.j2"), - ("default/content-version.conf", "content-version.j2")]: - written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, - self.config, - [self.config.app])) + + written_files.add(ConfWriter.writeConfFile(pathlib.Path("default/content-version.conf"), + "content-version.j2", + self.config, + [self.config.app])) written_files.add(ConfWriter.writeManifestFile(pathlib.Path("app.manifest"), "app.manifest.j2", self.config, [self.config.app])) + + written_files.add(ConfWriter.writeServerConf(self.config)) + + written_files.add(ConfWriter.writeAppConf(self.config)) + + return written_files diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 4d2e0490..410ce4f6 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -12,6 +12,76 @@ from contentctl.objects.config import build import xml.etree.ElementTree as ET +# This list is not exhaustive of all default conf files, but should be +# sufficient for our purposes. +DEFAULT_CONF_FILES = [ + "alert_actions.conf", + "app.conf", + "audit.conf", + "authentication.conf", + "authorize.conf", + "bookmarks.conf", + "checklist.conf", + "collections.conf", + "commands.conf", + "conf.conf", + "datamodels.conf", + "datatypesbnf.conf", + "default-mode.conf", + "deploymentclient.conf", + "distsearch.conf", + "event_renderers.conf", + "eventdiscoverer.conf", + "eventtypes.conf", + "federated.conf", + "fields.conf", + "global-banner.conf", + "health.conf", + "indexes.conf", + "inputs.conf", + "limits.conf", + "literals.conf", + "livetail.conf", + "macros.conf", + "messages.conf", + "metric_alerts.conf", + "metric_rollups.conf", + "multikv.conf", + "outputs.conf", + "passwords.conf", + "procmon-filters.conf", + "props.conf", + "pubsub.conf", + "restmap.conf", + "rolling_upgrade.conf", + "savedsearches.conf", + "searchbnf.conf", + "segmenters.conf", + "server.conf", + "serverclass.conf", + "serverclass.seed.xml.conf", + "source-classifier.conf", + "sourcetypes.conf", + "tags.conf", + "telemetry.conf", + "times.conf", + "transactiontypes.conf", + "transforms.conf", + "ui-prefs.conf", + "ui-tour.conf", + "user-prefs.conf", + "user-seed.conf", + "viewstates.conf", + "visualizations.conf", + "web-features.conf", + "web.conf", + "wmi.conf", + "workflow_actions.conf", + "workload_policy.conf", + "workload_pools.conf", + "workload_rules.conf", +] + class ConfWriter(): @staticmethod @@ -57,6 +127,52 @@ def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib. ConfWriter.validateConfFile(output_path) return output_path + @staticmethod + def getCustomConfFileStems(config:build)->list[str]: + # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if + # they are custom conf files + default_path = config.getPackageDirectoryPath()/"default" + conf_files = default_path.glob("*.conf") + + custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES] + return sorted(custom_conf_file_stems) + + @staticmethod + def writeServerConf(config: build) -> pathlib.Path: + app_output_path = pathlib.Path("default/server.conf") + template_name = "server.conf.j2" + + j2_env = ConfWriter.getJ2Environment() + template = j2_env.get_template(template_name) + + output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config)) + + output_path = config.getPackageDirectoryPath()/app_output_path + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'a') as f: + output = output.encode('utf-8', 'ignore').decode('utf-8') + f.write(output) + return output_path + + + @staticmethod + def writeAppConf(config: build) -> pathlib.Path: + app_output_path = pathlib.Path("default/app.conf") + template_name = "app.conf.j2" + + j2_env = ConfWriter.getJ2Environment() + template = j2_env.get_template(template_name) + + output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), + app=config.app) + + output_path = config.getPackageDirectoryPath()/app_output_path + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'a') as f: + output = output.encode('utf-8', 'ignore').decode('utf-8') + f.write(output) + return output_path + @staticmethod def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() @@ -70,6 +186,7 @@ def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: output = output.encode('utf-8', 'ignore').decode('utf-8') f.write(output) return output_path + @staticmethod @@ -218,8 +335,3 @@ def validateManifestFile(path:pathlib.Path): _ = json.load(manifestFile) except Exception as e: raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}") - - - - - diff --git a/contentctl/output/data_source_writer.py b/contentctl/output/data_source_writer.py index 97967a72..1a6e4f95 100644 --- a/contentctl/output/data_source_writer.py +++ b/contentctl/output/data_source_writer.py @@ -1,6 +1,5 @@ import csv from contentctl.objects.data_source import DataSource -from contentctl.objects.event_source import EventSource from typing import List import pathlib diff --git a/contentctl/output/templates/analyticstories_detections.j2 b/contentctl/output/templates/analyticstories_detections.j2 index 2f9a1318..e97f82a8 100644 --- a/contentctl/output/templates/analyticstories_detections.j2 +++ b/contentctl/output/templates/analyticstories_detections.j2 @@ -7,7 +7,7 @@ type = detection asset_type = {{ detection.tags.asset_type.value }} confidence = medium -explanation = {{ detection.description | escapeNewlines() }} +explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }} {% if detection.how_to_implement is defined %} how_to_implement = {{ detection.how_to_implement | escapeNewlines() }} {% else %} diff --git a/contentctl/output/templates/app.conf.j2 b/contentctl/output/templates/app.conf.j2 index 51734792..63310711 100644 --- a/contentctl/output/templates/app.conf.j2 +++ b/contentctl/output/templates/app.conf.j2 @@ -4,31 +4,33 @@ is_configured = false state = enabled state_change_requires_restart = false -build = {{ objects[0].build }} +build = {{ app.build }} [triggers] -reload.analytic_stories = simple -reload.usage_searches = simple -reload.use_case_library = simple -reload.correlationsearches = simple -reload.analyticstories = simple -reload.governance = simple -reload.managed_configurations = simple -reload.postprocess = simple -reload.content-version = simple -reload.es_investigations = simple +{% for custom_conf_file in custom_conf_files%} +reload.{{custom_conf_file}} = simple +{% endfor %} [launcher] -author = {{ objects[0].author_company }} -version = {{ objects[0].version }} -description = {{ objects[0].description | escapeNewlines() }} +author = {{ app.author_company }} +version = {{ app.version }} +description = {{ app.description | escapeNewlines() }} [ui] is_visible = true -label = {{ objects[0].title }} +label = {{ app.title }} [package] -id = {{ objects[0].appid }} +id = {{ app.appid }} +{% if app.uid == 3449 %} +check_for_updates = true +{% else %} +check_for_updates = false +{% endif %} + +[id] +version = {{ app.version }} +name = {{ app.appid }} diff --git a/contentctl/output/templates/app.manifest.j2 b/contentctl/output/templates/app.manifest.j2 index 7891f52b..408eb1ef 100644 --- a/contentctl/output/templates/app.manifest.j2 +++ b/contentctl/output/templates/app.manifest.j2 @@ -1,5 +1,6 @@ { - "schemaVersion": "1.0.0", + "schemaVersion": "1.0.0", + "targetWorkloads": ["_search_heads"], "info": { "title": "{{ objects[0].title }}", "id": { diff --git a/contentctl/output/templates/doc_detections.j2 b/contentctl/output/templates/doc_detections.j2 index 5430b0ed..60f0282f 100644 --- a/contentctl/output/templates/doc_detections.j2 +++ b/contentctl/output/templates/doc_detections.j2 @@ -162,11 +162,6 @@ The SPL above uses the following Lookups: {% endfor %} {% endif -%} -#### Required field -{% for field in object.tags.required_fields -%} -* {{ field }} -{% endfor %} - #### How To Implement {{ object.how_to_implement}} diff --git a/contentctl/output/templates/savedsearches_detections.j2 b/contentctl/output/templates/savedsearches_detections.j2 index f2f345aa..396bb2c6 100644 --- a/contentctl/output/templates/savedsearches_detections.j2 +++ b/contentctl/output/templates/savedsearches_detections.j2 @@ -112,7 +112,8 @@ alert.suppress.fields = {{ detection.tags.throttling.conf_formatted_fields() }} alert.suppress.period = {{ detection.tags.throttling.period }} {% endif %} search = {{ detection.search | escapeNewlines() }} - +action.notable.param.drilldown_searches = {{ detection.drilldowns_in_JSON | tojson | escapeNewlines() }} {% endif %} + {% endfor %} ### END {{ app.label }} DETECTIONS ### diff --git a/contentctl/output/templates/server.conf.j2 b/contentctl/output/templates/server.conf.j2 new file mode 100644 index 00000000..28a78f98 --- /dev/null +++ b/contentctl/output/templates/server.conf.j2 @@ -0,0 +1,4 @@ +[shclustering] +{% for custom_conf_file in custom_conf_files%} +conf_replication_include.{{custom_conf_file}} = true +{% endfor %} \ No newline at end of file diff --git a/contentctl/output/yml_output.py b/contentctl/output/yml_output.py index 93eae5dc..b4da5412 100644 --- a/contentctl/output/yml_output.py +++ b/contentctl/output/yml_output.py @@ -43,7 +43,6 @@ def writeDetections(self, objects: list, output_path : str) -> None: "kill_chain_phases:": True, "observable": True, "product": True, - "required_fields": True, "risk_score": True, "security_domain": True }, diff --git a/contentctl/templates/app_template/default/app.conf b/contentctl/templates/app_template/default/app.conf deleted file mode 100644 index c6991ff5..00000000 --- a/contentctl/templates/app_template/default/app.conf +++ /dev/null @@ -1,30 +0,0 @@ -## Splunk app configuration file - -[install] -is_configured = false -state = enabled -state_change_requires_restart = false -build = 16367 - -[triggers] -reload.analytic_stories = simple -reload.use_case_library = simple -reload.correlationsearches = simple -reload.analyticstories = simple -reload.governance = simple -reload.managed_configurations = simple -reload.postprocess = simple -reload.content-version = simple -reload.es_investigations = simple - -[launcher] -author = Splunk -version = 4.9.0 -description = Explore the Analytic Stories included with ES Content Updates. - -[ui] -is_visible = true -label = ES Content Updates - -[package] -id = DA-ESS-ContentUpdate diff --git a/contentctl/templates/app_template/default/content-version.conf b/contentctl/templates/app_template/default/content-version.conf deleted file mode 100644 index 4bbba5eb..00000000 --- a/contentctl/templates/app_template/default/content-version.conf +++ /dev/null @@ -1,2 +0,0 @@ -[content-version] -version = 4.9.0 diff --git a/contentctl/templates/app_template/metadata/default.meta b/contentctl/templates/app_template/metadata/default.meta index b9b933bf..7d137480 100644 --- a/contentctl/templates/app_template/metadata/default.meta +++ b/contentctl/templates/app_template/metadata/default.meta @@ -1,6 +1,6 @@ ## shared Application-level permissions [] -access = read : [ * ], write : [ admin ] +access = read : [ * ], write : [ admin, sc_admin ] export = system [savedsearches] diff --git a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml index 1a4af7b1..fbf847e1 100644 --- a/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +++ b/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml @@ -29,6 +29,15 @@ references: - https://attack.mitre.org/techniques/T1560/001/ - https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/ - https://thedfirreport.com/2021/01/31/bazar-no-ryuk/ +drilldown_searches: +- name: View the detection results for $user$ and $dest$ + search: '%original_detection_search% | search user = $user$ dest = $dest$' + earliest_offset: $info_min_time$ + latest_offset: $info_max_time$ +- name: View risk events for the last 7 days for $user$ and $dest$ + search: '| from datamodel Risk.All_Risk | search normalized_risk_object IN ($user$, $dest$) starthoursago=168 endhoursago=1 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`' + earliest_offset: $info_min_time$ + latest_offset: $info_max_time$ tags: analytic_story: - Cobalt Strike @@ -62,22 +71,10 @@ tags: - Splunk Enterprise - Splunk Enterprise Security - Splunk Cloud - required_fields: - - _time - - Processes.process_name - - Processes.process - - Processes.dest - - Processes.user - - Processes.parent_process_name - - Processes.process_name - - Processes.parent_process - - Processes.process_id - - Processes.parent_process_id - risk_score: 64 security_domain: endpoint tests: - name: True Positive Test attack_data: - data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1560.001/archive_utility/windows-sysmon.log source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational - sourcetype: xmlwineventlog \ No newline at end of file + sourcetype: xmlwineventlog diff --git a/pyproject.toml b/pyproject.toml index 1db114c9..ed1eebd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.poetry] name = "contentctl" -version = "4.4.0" +version = "4.4.6" + description = "Splunk Content Control Tool" authors = ["STRT "] license = "Apache 2.0" @@ -10,12 +11,12 @@ readme = "README.md" contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] -python = "^3.11" +python = "^3.11,<3.13" pydantic = "^2.8.2" PyYAML = "^6.0.2" requests = "~2.32.3" pycvesearch = "^1.2" -xmltodict = "^0.13.0" +xmltodict = ">=0.13,<0.15" attackcti = "^0.4.0" Jinja2 = "^3.1.4" questionary = "^2.0.1"