From aaac5e26dec589d4d62db9a9bbb2261a68c9d1c0 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 15:49:51 -0600 Subject: [PATCH 1/7] Ruff lint autofixes --- contentctl/actions/build.py | 5 +---- contentctl/actions/deploy_acs.py | 4 ++-- .../actions/detection_testing/DetectionTestingManager.py | 8 ++------ .../DetectionTestingInfrastructureContainer.py | 2 +- .../detection_testing/views/DetectionTestingViewCLI.py | 1 - contentctl/actions/initialize_old.py | 4 ---- contentctl/actions/inspect.py | 2 +- contentctl/actions/new_content.py | 1 - contentctl/actions/release_notes.py | 7 +++---- contentctl/actions/reporting.py | 1 - contentctl/actions/test.py | 2 -- contentctl/enrichments/attack_enrichment.py | 1 - contentctl/enrichments/cve_enrichment.py | 7 +------ contentctl/enrichments/splunk_app_enrichment.py | 5 +---- contentctl/helper/link_validator.py | 5 +++-- contentctl/helper/utils.py | 2 +- .../detection_abstract.py | 1 - .../security_content_object_abstract.py | 1 - contentctl/objects/data_source.py | 2 +- contentctl/objects/deployment.py | 2 +- contentctl/objects/lookup.py | 2 +- contentctl/objects/mitre_attack_enrichment.py | 2 +- contentctl/objects/playbook.py | 2 +- contentctl/objects/playbook_tags.py | 2 +- contentctl/objects/story_tags.py | 2 +- contentctl/objects/unit_test.py | 1 - contentctl/objects/unit_test_result.py | 2 +- contentctl/output/attack_nav_output.py | 1 - contentctl/output/doc_md_output.py | 2 -- contentctl/output/svg_output.py | 1 - 30 files changed, 24 insertions(+), 56 deletions(-) diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index 5e3acdb3..32876fee 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -1,11 +1,8 @@ -import sys import shutil -import os from dataclasses import dataclass -from contentctl.objects.enums import SecurityContentType -from contentctl.input.director import Director, DirectorOutputDto +from contentctl.input.director import DirectorOutputDto from contentctl.output.conf_output import ConfOutput from contentctl.output.conf_writer import ConfWriter from contentctl.output.api_json_output import ApiJsonOutput diff --git a/contentctl/actions/deploy_acs.py b/contentctl/actions/deploy_acs.py index 8451751b..2b07b2f0 100644 --- a/contentctl/actions/deploy_acs.py +++ b/contentctl/actions/deploy_acs.py @@ -42,10 +42,10 @@ def execute(self, config: deploy_acs, appinspect_token:str) -> None: # This likely includes a more verbose response describing the error res.raise_for_status() print(res.json()) - except Exception as e: + except Exception: try: error_text = res.json() - except Exception as e: + except Exception: error_text = "No error text - request failed" formatted_error_text = pprint.pformat(error_text) print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n" diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 13058e2f..80450916 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -3,23 +3,19 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer -from urllib.parse import urlparse -from copy import deepcopy import signal import datetime # from queue import Queue from dataclasses import dataclass # import threading -import ctypes from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( - DetectionTestingInfrastructure, DetectionTestingManagerOutputDto, ) from contentctl.actions.detection_testing.views.DetectionTestingView import ( DetectionTestingView, ) from contentctl.objects.enums import PostTestBehavior -from pydantic import BaseModel, Field +from pydantic import BaseModel from contentctl.objects.detection import Detection import concurrent.futures import docker @@ -133,7 +129,7 @@ def create_DetectionTestingInfrastructureObjects(self): if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)): try: client = docker.from_env() - except Exception as e: + except Exception: raise Exception("Unable to connect to docker. Are you sure that docker is running on this host?") try: diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py index f5887033..0a19003a 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py @@ -141,7 +141,7 @@ def removeContainer(self, removeVolumes: bool = True, forceRemove: bool = True): container: docker.models.containers.Container = ( self.get_docker_client().containers.get(self.get_name()) ) - except Exception as e: + except Exception: # Container does not exist, no need to try and remove it return try: diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index 1675fce4..7b29f048 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -3,7 +3,6 @@ ) import time -import datetime import tqdm diff --git a/contentctl/actions/initialize_old.py b/contentctl/actions/initialize_old.py index 82fa41bf..f5210511 100644 --- a/contentctl/actions/initialize_old.py +++ b/contentctl/actions/initialize_old.py @@ -7,12 +7,8 @@ import sys import questionary import os -from contentctl.objects.enums import LogLevel -import abc -from pydantic import BaseModel, Field -from contentctl.objects.config import Config DEFAULT_FOLDERS = ['detections', 'stories', 'lookups', 'macros', 'baselines', 'dist'] diff --git a/contentctl/actions/inspect.py b/contentctl/actions/inspect.py index 261fd413..9938b300 100644 --- a/contentctl/actions/inspect.py +++ b/contentctl/actions/inspect.py @@ -150,7 +150,7 @@ def inspectAppCLI(self, config: inspect) -> None: from splunk_appinspect.main import ( validate, MODE_OPTION, APP_PACKAGE_ARGUMENT, OUTPUT_FILE_OPTION, LOG_FILE_OPTION, INCLUDED_TAGS_OPTION, EXCLUDED_TAGS_OPTION, - PRECERT_MODE, TEST_MODE) + TEST_MODE) except Exception as e: print(e) # print("******WARNING******") diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index 3d5fa5b6..e57fc982 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass import questionary from typing import Any from contentctl.input.new_content_questions import NewContentQuestions diff --git a/contentctl/actions/release_notes.py b/contentctl/actions/release_notes.py index fe2c90d4..fb42effc 100644 --- a/contentctl/actions/release_notes.py +++ b/contentctl/actions/release_notes.py @@ -1,4 +1,3 @@ -import os from contentctl.objects.config import release_notes from git import Repo import re @@ -54,7 +53,7 @@ def create_notes(self,repo_path:pathlib.Path, file_paths:List[pathlib.Path], hea # Check and create detection link - if 'name' in data and 'id' in data and 'detections' in file_path.parts and not 'ssa_detections' in file_path.parts and 'detections/deprecated' not in file_path.parts: + if 'name' in data and 'id' in data and 'detections' in file_path.parts and 'ssa_detections' not in file_path.parts and 'detections/deprecated' not in file_path.parts: if data['status'] == "production": temp_link = "https://research.splunk.com" + str(file_path).replace(str(repo_path),"") @@ -232,9 +231,9 @@ def printNotes(notes:List[dict[str,Union[List[str], str]]], outfile:Union[pathli with open(outfile,'w') as writer: writer.write(text_blob) - printNotes(notes, config.releaseNotesFilename(f"release_notes.txt")) + printNotes(notes, config.releaseNotesFilename("release_notes.txt")) print("\n\n### Other Updates\n-\n") print("\n## BA Release Notes") printNotes(ba_notes, config.releaseNotesFilename("ba_release_notes.txt")) - print(f"Release notes completed succesfully") \ No newline at end of file + print("Release notes completed succesfully") \ No newline at end of file diff --git a/contentctl/actions/reporting.py b/contentctl/actions/reporting.py index a7997713..db3a278d 100644 --- a/contentctl/actions/reporting.py +++ b/contentctl/actions/reporting.py @@ -1,4 +1,3 @@ -import os from dataclasses import dataclass diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index b3437cef..e45688f7 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -2,10 +2,8 @@ from typing import List from contentctl.objects.config import test_common, Selected, Changes -from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType from contentctl.objects.detection import Detection -from contentctl.input.director import DirectorOutputDto from contentctl.actions.detection_testing.DetectionTestingManager import ( DetectionTestingManager, diff --git a/contentctl/enrichments/attack_enrichment.py b/contentctl/enrichments/attack_enrichment.py index 71e1c955..47ed6d78 100644 --- a/contentctl/enrichments/attack_enrichment.py +++ b/contentctl/enrichments/attack_enrichment.py @@ -1,6 +1,5 @@ from __future__ import annotations -import sys from attackcti import attack_client import logging from pydantic import BaseModel diff --git a/contentctl/enrichments/cve_enrichment.py b/contentctl/enrichments/cve_enrichment.py index 66160eda..d9d71546 100644 --- a/contentctl/enrichments/cve_enrichment.py +++ b/contentctl/enrichments/cve_enrichment.py @@ -1,13 +1,8 @@ from __future__ import annotations from pycvesearch import CVESearch -import functools -import os -import shelve -import time -from typing import Annotated, Any, Union, TYPE_CHECKING +from typing import Annotated, Union, TYPE_CHECKING from pydantic import ConfigDict, BaseModel,Field, computed_field from decimal import Decimal -from requests.exceptions import ReadTimeout from contentctl.objects.annotated_types import CVE_TYPE if TYPE_CHECKING: from contentctl.objects.config import validate diff --git a/contentctl/enrichments/splunk_app_enrichment.py b/contentctl/enrichments/splunk_app_enrichment.py index 9a6ce917..161ec91c 100644 --- a/contentctl/enrichments/splunk_app_enrichment.py +++ b/contentctl/enrichments/splunk_app_enrichment.py @@ -1,11 +1,8 @@ import requests import xmltodict -import json import functools -import pickle import shelve import os -import time SPLUNKBASE_API_URL = "https://apps.splunk.com/api/apps/entriesbyid/" @@ -29,7 +26,7 @@ def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes: req = requests.get(url) req_content = req.content cache[url] = req_content - except Exception as e: + except Exception: raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}")) if isinstance(cache, shelve.Shelf): diff --git a/contentctl/helper/link_validator.py b/contentctl/helper/link_validator.py index 6fbb9fde..13cfabc5 100644 --- a/contentctl/helper/link_validator.py +++ b/contentctl/helper/link_validator.py @@ -1,7 +1,8 @@ from pydantic import BaseModel, model_validator from typing import Union, Callable, Any import requests -import urllib3, urllib3.exceptions +import urllib3 +import urllib3.exceptions import time import abc @@ -74,7 +75,7 @@ def check_reference(cls, data:Any)->Any: data['valid'] = False return data - except Exception as e: + except Exception: resolution_time = time.time() - start_time #print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds") data['status_code'] = 0 diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index e0649f2d..1df47825 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -148,7 +148,7 @@ def validate_git_hash( # If we get here, it does not exist in the given branch raise (Exception("Does not exist in branch")) - except Exception as e: + except Exception: if branch_name is None: branch_name = "ANY_BRANCH" if ALWAYS_PULL: diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index c216eb20..0a0fceff 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -11,7 +11,6 @@ Field, computed_field, model_serializer, - ConfigDict, FilePath ) diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index f231f5f3..2ddb3124 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -5,7 +5,6 @@ from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject from contentctl.input.director import DirectorOutputDto - from contentctl.objects.config import CustomApp from contentctl.objects.enums import AnalyticsType from contentctl.objects.constants import CONTENTCTL_MAX_STANZA_LENGTH diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index 2ed9c80c..920a19e2 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import Optional, Any -from pydantic import Field, HttpUrl, model_serializer, BaseModel, ConfigDict +from pydantic import Field, HttpUrl, model_serializer, BaseModel from contentctl.objects.security_content_object import SecurityContentObject diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index 6e2cc6d2..d268670f 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -1,5 +1,5 @@ from __future__ import annotations -from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt, ConfigDict +from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt from typing import Any import uuid import datetime diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 8d42c55e..9f0b1318 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -2,7 +2,7 @@ from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field, TypeAdapter from enum import StrEnum, auto -from typing import TYPE_CHECKING, Optional, Any, Union, Literal, Annotated, Self +from typing import TYPE_CHECKING, Optional, Any, Literal, Annotated, Self import re import csv import abc diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index 4a09209a..6c532bd8 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -1,6 +1,6 @@ from __future__ import annotations from pydantic import BaseModel, Field, ConfigDict, HttpUrl, field_validator -from typing import List, Annotated +from typing import List from enum import StrEnum import datetime from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE diff --git a/contentctl/objects/playbook.py b/contentctl/objects/playbook.py index f025d227..6d9290bf 100644 --- a/contentctl/objects/playbook.py +++ b/contentctl/objects/playbook.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import TYPE_CHECKING,Self +from typing import Self from pydantic import model_validator, Field, FilePath diff --git a/contentctl/objects/playbook_tags.py b/contentctl/objects/playbook_tags.py index 10d90ac1..299618fd 100644 --- a/contentctl/objects/playbook_tags.py +++ b/contentctl/objects/playbook_tags.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional, List +from typing import Optional, List from pydantic import BaseModel, Field,ConfigDict import enum from contentctl.objects.detection import Detection diff --git a/contentctl/objects/story_tags.py b/contentctl/objects/story_tags.py index e1bd45dc..5f5e4dfb 100644 --- a/contentctl/objects/story_tags.py +++ b/contentctl/objects/story_tags.py @@ -1,6 +1,6 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_serializer, ConfigDict -from typing import List,Set,Optional, Annotated +from typing import List,Set,Optional from enum import Enum diff --git a/contentctl/objects/unit_test.py b/contentctl/objects/unit_test.py index 67dc1d62..a07b532b 100644 --- a/contentctl/objects/unit_test.py +++ b/contentctl/objects/unit_test.py @@ -2,7 +2,6 @@ from pydantic import Field -from contentctl.objects.unit_test_baseline import UnitTestBaseline from contentctl.objects.test_attack_data import TestAttackData from contentctl.objects.unit_test_result import UnitTestResult from contentctl.objects.base_test import BaseTest, TestType diff --git a/contentctl/objects/unit_test_result.py b/contentctl/objects/unit_test_result.py index 8c40da10..2a9f230d 100644 --- a/contentctl/objects/unit_test_result.py +++ b/contentctl/objects/unit_test_result.py @@ -72,7 +72,7 @@ def set_job_content( if self.exception is not None: self.message = f"EXCEPTION: {str(self.exception)}" else: - self.message = f"ERROR with no more specific message available." + self.message = "ERROR with no more specific message available." self.sid_link = NO_SID return self.success diff --git a/contentctl/output/attack_nav_output.py b/contentctl/output/attack_nav_output.py index e6c3e35b..e94abfde 100644 --- a/contentctl/output/attack_nav_output.py +++ b/contentctl/output/attack_nav_output.py @@ -1,4 +1,3 @@ -import os from typing import List,Union import pathlib diff --git a/contentctl/output/doc_md_output.py b/contentctl/output/doc_md_output.py index 90b3dfad..ab3cec66 100644 --- a/contentctl/output/doc_md_output.py +++ b/contentctl/output/doc_md_output.py @@ -1,10 +1,8 @@ import os -import asyncio import sys from pathlib import Path -from contentctl.objects.enums import SecurityContentType from contentctl.output.jinja_writer import JinjaWriter diff --git a/contentctl/output/svg_output.py b/contentctl/output/svg_output.py index 2d0c9d56..055b3128 100644 --- a/contentctl/output/svg_output.py +++ b/contentctl/output/svg_output.py @@ -1,4 +1,3 @@ -import os import pathlib from typing import List, Any From 48977efa20024c1d6f33bfa6c2b5fd2bb06900bc Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 15:54:58 -0600 Subject: [PATCH 2/7] Auto unsafe fixes --- .../actions/detection_testing/DetectionTestingManager.py | 8 ++++---- .../detection_testing/views/DetectionTestingViewCLI.py | 2 +- contentctl/helper/utils.py | 4 ++-- contentctl/input/yml_reader.py | 2 +- contentctl/output/conf_output.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 80450916..3696dbfb 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -82,7 +82,7 @@ def sigint_handler(signum, frame): # Wait for all instances to be set up for future in concurrent.futures.as_completed(future_instances_setup): try: - result = future.result() + future.result() except Exception as e: self.output_dto.terminate = True print(f"Error setting up container: {str(e)}") @@ -97,7 +97,7 @@ def sigint_handler(signum, frame): # Wait for execution to finish for future in concurrent.futures.as_completed(future_instances_execute): try: - result = future.result() + future.result() except Exception as e: self.output_dto.terminate = True print(f"Error running in container: {str(e)}") @@ -110,14 +110,14 @@ def sigint_handler(signum, frame): } for future in concurrent.futures.as_completed(future_views_shutdowner): try: - result = future.result() + future.result() except Exception as e: print(f"Error stopping view: {str(e)}") # Wait for original view-related threads to complete for future in concurrent.futures.as_completed(future_views): try: - result = future.result() + future.result() except Exception as e: print(f"Error running container: {str(e)}") diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index 7b29f048..7ae1c8c3 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -38,7 +38,7 @@ def setup(self): miniters=0, mininterval=0, ) - fmt = self.format_pbar( + self.format_pbar( len(self.sync_obj.outputQueue), len(self.sync_obj.inputQueue) ) diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 1df47825..f003e2fc 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -141,7 +141,7 @@ def validate_git_hash( # Note, of course, that a hash can be in 0, 1, more branches! for branch_string in all_branches_containing_hash: if branch_string.split(" ")[0] == "*" and ( - branch_string.split(" ")[-1] == branch_name or branch_name == None + branch_string.split(" ")[-1] == branch_name or branch_name is None ): # Yes, the hash exists in the branch (or branch_name was None and it existed in at least one branch)! return True @@ -376,7 +376,7 @@ def download_file_from_http( ) try: - download_start_time = default_timer() + default_timer() bytes_written = 0 file_to_download = requests.get(file_path, stream=True) file_to_download.raise_for_status() diff --git a/contentctl/input/yml_reader.py b/contentctl/input/yml_reader.py index 49dfb812..bc3219b5 100644 --- a/contentctl/input/yml_reader.py +++ b/contentctl/input/yml_reader.py @@ -42,7 +42,7 @@ def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING print(exc) sys.exit(1) - if add_fields == False: + if add_fields is False: return yml_obj diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index c5a67673..a59e9fa5 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -53,7 +53,7 @@ def writeHeaders(self) -> set[pathlib.Path]: #The contents of app.manifest are not a conf file, but json. #DO NOT write a header for this file type, simply create the file - with open(self.config.getPackageDirectoryPath() / pathlib.Path('app.manifest'), 'w') as f: + with open(self.config.getPackageDirectoryPath() / pathlib.Path('app.manifest'), 'w'): pass From d051c089a8d259b037d9e5628c7421bff1136f14 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 16:05:10 -0600 Subject: [PATCH 3/7] Manual fixes --- .../actions/detection_testing/GitService.py | 3 +- contentctl/actions/initialize_old.py | 241 ------------------ contentctl/helper/link_validator.py | 2 +- contentctl/objects/config.py | 2 +- 4 files changed, 4 insertions(+), 244 deletions(-) delete mode 100644 contentctl/actions/initialize_old.py diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index be8d3d47..f19b4ac3 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto +from contentctl.input.director import DirectorOutputDto from contentctl.objects.config import All, Changes, Selected, test_common from contentctl.objects.data_source import DataSource from contentctl.objects.detection import Detection @@ -22,7 +23,7 @@ LOGGER = logging.getLogger(__name__) -from contentctl.input.director import DirectorOutputDto + class GitService(BaseModel): diff --git a/contentctl/actions/initialize_old.py b/contentctl/actions/initialize_old.py deleted file mode 100644 index f5210511..00000000 --- a/contentctl/actions/initialize_old.py +++ /dev/null @@ -1,241 +0,0 @@ -''' -Initializes a Splunk Content Project -''' - -from pathlib import Path -import yaml -import sys -import questionary -import os - - - -DEFAULT_FOLDERS = ['detections', 'stories', 'lookups', 'macros', 'baselines', 'dist'] - - -def create_folders(path): - - for folder in DEFAULT_FOLDERS: - folder_path = path + "/" + folder - if not os.path.exists(folder_path): - os.makedirs(folder_path) - - - -def NewContentPack(args, default_config): - """ - new function creates a new configuration file based on the user input on the terminal. - :param config: python dictionary having the configuration - :return: No return value - """ - contentctl_config_file = Path(args.config) - if contentctl_config_file.is_file(): - questions = [ - { - 'type': 'confirm', - 'message': 'File {0} already exist, are you sure you want to continue?\nTHIS WILL OVERWRITE YOUR CURRENT CONFIG!'.format(contentctl_config_file), - 'name': 'continue', - 'default': True, - }, - ] - - answers = questionary.prompt(questions) - if answers['continue']: - print("> continuing with contentctl configuration...") - else: - print("> exiting, to create a unique configuration file in another location use the --config flag") - sys.exit(0) - - - # configuration parameters - if os.path.exists(args.output): - config_path = args.output + "/" + str(contentctl_config_file) - else: - print("ERROR, output folder: {0} does not exist".format(args.output)) - sys.exit(1) - - # deal with skipping configuration - if args.skip_configuration: - print("initializing with default configuration: {0}".format(config_path)) - # write config file - with open(config_path, 'w') as outfile: - yaml.dump(default_config, outfile, default_flow_style=False, sort_keys=False) - - # write folder structure - create_folders(args.output) - sys.exit(0) - - - questions = [ - { - "type": "select", - "message": "Which build format should we use for this content pack? Builds will be created under the dist/ folder.", - "name": "product", - "choices": ["Splunk App", "JSON API Objects", "BA Objects", "All"], - "default": "Splunk App" - }, - { - 'type': 'text', - 'message': 'What should the Splunk App for this content pack be called?', - 'name': 'product_app_name', - 'default': 'Capybara Splunk Content Pack', - 'when': lambda answers: answers['product'] == "Splunk App" or answers['product'] == "All", - - }, - { - 'type': 'confirm', - 'message': 'Should this content pack be deployed to a (Cloud) Splunk Enterprise Server?', - 'name': 'deploy_to_splunk', - 'default': False, - - }, - { - 'type': 'text', - 'message': 'What is the : of the (Cloud) Splunk Enterprise Server?', - 'name': 'deploy_to_splunk_server', - 'default': '127.0.0.1:8089', - 'when': lambda answers: answers['deploy_to_splunk'], - - }, - { - 'type': 'text', - 'message': 'What is the username of the (Cloud) Splunk Enterprise Server?', - 'name': 'deploy_to_splunk_username', - 'default': 'admin', - 'when': lambda answers: answers['deploy_to_splunk'], - - }, - { - 'type': 'text', - 'message': 'What is the password of the (Cloud) Splunk Enterprise Server?', - 'name': 'deploy_to_splunk_password', - 'default': 'xxx', - 'when': lambda answers: answers['deploy_to_splunk'], - - }, - { - 'type': 'text', - 'message': 'How often should analytics run? The schedule is on cron format (https://crontab.guru/).', - 'name': 'scheduling_cron_schedule', - 'default': '0 * * * *', - }, - { - 'type': 'text', - 'message': 'What is the earliest time for analytics? Uses Splunk time modifiers (https://docs.splunk.com/Documentation/SCS/current/Search/Timemodifiers).', - 'name': 'scheduling_earliest_time', - 'default': '-70m@m', - }, - { - 'type': 'text', - 'message': 'What is the latest time for analytics? Uses Splunk time modifiers (https://docs.splunk.com/Documentation/SCS/current/Search/Timemodifiers).', - 'name': 'scheduling_latest_time', - 'default': '-10m@m', - }, - { - 'type': 'checkbox', - 'message': 'What should the default action be when an analytic triggers?', - 'name': 'default_actions', - 'choices': ["notable", "risk_event", "email"], - 'default': 'notable', - }, - { - 'type': 'text', - 'message': 'What email address should we send the alerts to?', - 'name': 'to_email', - 'default': 'geralt@monsterkiller.com', - 'when': lambda answers: 'email' in answers['default_actions'], - }, - { - 'type': 'confirm', - 'message': 'Should we include some example content? This will add a detection and its test with supporting components like lookups and macros.', - 'name': 'pre_populate', - 'default': True, - }, - ] - - answers = questionary.prompt(questions) - - # create a custom config object to store answers - custom_config = default_config - - # remove other product settings - if answers['product'] == 'Splunk App': - # pop other configs out - custom_config['build'].pop('json_objects') - custom_config['build'].pop('ba_objects') - # capture configs - custom_config['build']['splunk_app']['name'] = answers['product_app_name'] - custom_config['build']['splunk_app']['path'] = 'dist/' + answers['product_app_name'].lower().replace(" ", "_") - custom_config['build']['splunk_app']['prefix'] = answers['product_app_name'].upper()[0: 3] - - elif answers['product'] == 'JSON API Objects': - custom_config['build'].pop('splunk_app') - custom_config['build'].pop('ba_objects') - elif answers['product'] == 'BA Objects': - custom_config['build'].pop('splunk_app') - custom_config['build'].pop('json_objects') - else: - # splunk app config - custom_config['build']['splunk_app']['name'] = answers['product_app_name'] - custom_config['build']['splunk_app']['path'] = 'dist/' + answers['product_app_name'].lower().replace(" ", "_") - custom_config['build']['splunk_app']['prefix'] = answers['product_app_name'].upper()[0: 3] - - if answers['deploy_to_splunk']: - custom_config['deploy']['server'] = answers['deploy_to_splunk_server'] - custom_config['deploy']['username'] = answers['deploy_to_splunk_username'] - custom_config['deploy']['password'] = answers['deploy_to_splunk_password'] - else: - custom_config.pop('deploy') - - custom_config['scheduling']['cron_schedule'] = answers['scheduling_cron_schedule'] - custom_config['scheduling']['earliest_time'] = answers['scheduling_earliest_time'] - custom_config['scheduling']['latest_time'] = answers['scheduling_latest_time'] - - if 'notable' in answers['default_actions']: - custom_config['alert_actions']['notable']['rule_description'] = '%description%' - custom_config['alert_actions']['notable']['rule_title'] = '%name%' - custom_config['alert_actions']['notable']['nes_fields'] = ['user','dest','src'] - else: - custom_config['alert_actions'].pop('notable') - if 'risk_event' in answers['default_actions']: - rba = dict() - custom_config['alert_actions']['rba'] = rba - custom_config['alert_actions']['rba']['enabled'] = 'true' - - if 'email' in answers['default_actions']: - email = dict() - custom_config['alert_actions']['email'] = email - custom_config['alert_actions']['email']['subject'] = 'Alert %name% triggered' - custom_config['alert_actions']['email']['message'] = 'The rule %name% triggered base on %description%' - custom_config['alert_actions']['email']['to'] = answers['to_email'] - - - # write config file - with open(config_path, 'w') as outfile: - yaml.dump(custom_config, outfile, default_flow_style=False, sort_keys=False) - print('Content pack configuration created under: {0} .. edit to fine tune details before building'.format(config_path)) - - # write folder structure - create_folders(args.output) - print('The following folders were created: {0} under {1}.\nContent pack has been initialized, please run `new` to create new content.'.format(DEFAULT_FOLDERS, args.output)) - - print("Load the custom_config into the pydantic model we have created") - cfg = ContentPackConfig().parse_obj(custom_config) - import pprint - pprint.pprint(cfg.__dict__) - print("********************") - pprint.pprint(custom_config) - print("done") - - - - - - - - - - - - - diff --git a/contentctl/helper/link_validator.py b/contentctl/helper/link_validator.py index 13cfabc5..4419c32c 100644 --- a/contentctl/helper/link_validator.py +++ b/contentctl/helper/link_validator.py @@ -104,7 +104,7 @@ def initialize_cache(use_file_cache: bool = False): try: LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True) - except: + except Exception: print(f"Failed to create the cache file {LinkValidator.reference_cache_file}. Reference info will not be cached.") LinkValidator.cache = {} diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 02fd4bd9..e72a60ec 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -1162,7 +1162,7 @@ def releaseNotesFilename(self, filename: str) -> pathlib.Path: p = self.path / "dist" try: p.mkdir(exist_ok=True, parents=True) - except Exception: + except Exception as e: raise Exception( f"Error making the directory '{p}' to hold release_notes: {str(e)}" ) From 2d85d317c1dcf59e0614783740a8ba9e161e84b9 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 16:07:37 -0600 Subject: [PATCH 4/7] type comparisons --- .../actions/detection_testing/GitService.py | 2 +- contentctl/contentctl.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index f19b4ac3..bbfb7c2d 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -80,7 +80,7 @@ def getChanges(self, target_branch: str) -> List[Detection]: updated_datasources: set[DataSource] = set() for diff in all_diffs: - if type(diff) == pygit2.Patch: + if type(diff) is pygit2.Patch: if diff.delta.status in ( DeltaStatus.ADDED, DeltaStatus.MODIFIED, diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 753c11f8..d5864eb7 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -113,7 +113,7 @@ def deploy_acs_func(config: deploy_acs): def test_common_func(config: test_common): - if type(config) == test: + if type(config) is test: # construct the container Infrastructure objects config.getContainerInfrastructureObjects() # otherwise, they have already been passed as servers @@ -220,25 +220,25 @@ def main(): with warnings.catch_warnings(action="ignore"): config = tyro.cli(models) - if type(config) == init: + if type(config) is init: t.__dict__.update(config.__dict__) init_func(t) - elif type(config) == validate: + elif type(config) is validate: validate_func(config) - elif type(config) == report: + elif type(config) is report: report_func(config) - elif type(config) == build: + elif type(config) is build: build_func(config) - elif type(config) == new: + elif type(config) is new: new_func(config) - elif type(config) == inspect: + elif type(config) is inspect: inspect_func(config) - elif type(config) == release_notes: + elif type(config) is release_notes: release_notes_func(config) - elif type(config) == deploy_acs: + elif type(config) is deploy_acs: updated_config = deploy_acs.model_validate(config) deploy_acs_func(updated_config) - elif type(config) == test or type(config) == test_servers: + elif type(config) is test or type(config) is test_servers: test_common_func(config) else: raise Exception(f"Unknown command line type '{type(config).__name__}'") From ff87bcaf1741e8ecf15cb8d401438592dfef3ba7 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 16:20:25 -0600 Subject: [PATCH 5/7] Massive reformat --- contentctl/__init__.py | 2 +- contentctl/actions/build.py | 138 +++--- contentctl/actions/deploy_acs.py | 49 +- .../DetectionTestingManager.py | 91 ++-- .../actions/detection_testing/GitService.py | 3 - .../generate_detection_coverage_badge.py | 78 ++-- .../DetectionTestingInfrastructure.py | 227 ++++++---- ...DetectionTestingInfrastructureContainer.py | 75 ++-- .../actions/detection_testing/progress_bar.py | 3 + .../views/DetectionTestingView.py | 33 +- .../views/DetectionTestingViewCLI.py | 5 +- .../views/DetectionTestingViewFile.py | 4 +- .../views/DetectionTestingViewWeb.py | 5 +- contentctl/actions/doc_gen.py | 14 +- contentctl/actions/initialize.py | 78 ++-- contentctl/actions/inspect.py | 179 +++++--- contentctl/actions/new_content.py | 114 +++-- contentctl/actions/release_notes.py | 419 ++++++++++++------ contentctl/actions/reporting.py | 41 +- contentctl/actions/test.py | 54 ++- contentctl/actions/validate.py | 88 ++-- contentctl/api.py | 99 +++-- contentctl/enrichments/attack_enrichment.py | 183 +++++--- contentctl/enrichments/cve_enrichment.py | 55 ++- .../enrichments/splunk_app_enrichment.py | 69 +-- contentctl/helper/link_validator.py | 172 +++---- contentctl/helper/splunk_app.py | 110 +++-- contentctl/helper/utils.py | 91 ++-- contentctl/input/director.py | 106 +++-- contentctl/input/new_content_questions.py | 60 ++- contentctl/input/yml_reader.py | 37 +- .../detection_abstract.py | 375 +++++++++------- .../security_content_object_abstract.py | 93 ++-- contentctl/objects/alert_action.py | 16 +- contentctl/objects/annotated_types.py | 2 +- contentctl/objects/atomic.py | 118 ++--- contentctl/objects/base_test.py | 3 +- contentctl/objects/base_test_result.py | 24 +- contentctl/objects/baseline.py | 71 +-- contentctl/objects/baseline_tags.py | 51 ++- contentctl/objects/constants.py | 51 ++- contentctl/objects/correlation_search.py | 130 +++--- contentctl/objects/dashboard.py | 96 ++-- contentctl/objects/data_source.py | 24 +- contentctl/objects/deployment.py | 81 ++-- contentctl/objects/deployment_email.py | 2 +- contentctl/objects/deployment_notable.py | 3 +- contentctl/objects/deployment_phantom.py | 10 +- contentctl/objects/deployment_rba.py | 2 +- contentctl/objects/deployment_scheduling.py | 2 +- contentctl/objects/deployment_slack.py | 2 +- contentctl/objects/detection.py | 7 +- contentctl/objects/detection_metadata.py | 1 + contentctl/objects/detection_stanza.py | 9 +- contentctl/objects/detection_tags.py | 61 ++- contentctl/objects/drilldown.py | 90 ++-- contentctl/objects/enums.py | 104 +++-- contentctl/objects/errors.py | 40 +- contentctl/objects/integration_test.py | 6 +- contentctl/objects/integration_test_result.py | 1 + contentctl/objects/investigation.py | 67 +-- contentctl/objects/investigation_tags.py | 46 +- contentctl/objects/lookup.py | 270 ++++++----- contentctl/objects/macro.py | 93 ++-- contentctl/objects/manual_test.py | 6 +- contentctl/objects/manual_test_result.py | 1 + contentctl/objects/mitre_attack_enrichment.py | 31 +- contentctl/objects/notable_action.py | 3 +- contentctl/objects/notable_event.py | 4 +- contentctl/objects/observable.py | 12 +- contentctl/objects/playbook.py | 70 +-- contentctl/objects/playbook_tags.py | 36 +- contentctl/objects/rba.py | 22 +- contentctl/objects/risk_analysis_action.py | 26 +- contentctl/objects/risk_event.py | 40 +- contentctl/objects/risk_object.py | 1 + contentctl/objects/savedsearches_conf.py | 16 +- contentctl/objects/security_content_object.py | 7 +- contentctl/objects/story.py | 89 ++-- contentctl/objects/story_tags.py | 100 +++-- contentctl/objects/test_group.py | 7 +- contentctl/objects/threat_object.py | 1 + contentctl/objects/throttling.py | 45 +- contentctl/objects/unit_test.py | 6 +- contentctl/objects/unit_test_baseline.py | 9 +- contentctl/objects/unit_test_result.py | 10 +- contentctl/output/api_json_output.py | 44 +- contentctl/output/attack_nav_output.py | 41 +- contentctl/output/attack_nav_writer.py | 66 ++- contentctl/output/conf_output.py | 372 +++++++++------- contentctl/output/data_source_writer.py | 63 +-- contentctl/output/doc_md_output.py | 78 ++-- contentctl/output/jinja_writer.py | 34 +- contentctl/output/json_writer.py | 28 +- contentctl/output/svg_output.py | 93 ++-- contentctl/output/yml_writer.py | 42 +- tests/test_splunk_contentctl.py | 2 +- 97 files changed, 3565 insertions(+), 2473 deletions(-) diff --git a/contentctl/__init__.py b/contentctl/__init__.py index b794fd40..3dc1f76b 100644 --- a/contentctl/__init__.py +++ b/contentctl/__init__.py @@ -1 +1 @@ -__version__ = '0.1.0' +__version__ = "0.1.0" diff --git a/contentctl/actions/build.py b/contentctl/actions/build.py index 32876fee..1476b817 100644 --- a/contentctl/actions/build.py +++ b/contentctl/actions/build.py @@ -15,86 +15,122 @@ from contentctl.objects.config import build + @dataclass(frozen=True) class BuildInputDto: director_output_dto: DirectorOutputDto - config:build + config: build class Build: - - - def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto: if input_dto.config.build_app: - - updated_conf_files:set[pathlib.Path] = set() + updated_conf_files: set[pathlib.Path] = set() conf_output = ConfOutput(input_dto.config) - # Construct a path to a YML that does not actually exist. # We mock this "fake" path since the YML does not exist. # This ensures the checking for the existence of the CSV is correct - data_sources_fake_yml_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.yml" + data_sources_fake_yml_path = ( + input_dto.config.getPackageDirectoryPath() + / "lookups" + / "data_sources.yml" + ) # Construct a special lookup whose CSV is created at runtime and - # written directly into the lookups folder. We will delete this after a build, + # written directly into the lookups folder. We will delete this after a build, # assuming that it is successful. - data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv" - - - - DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path) - input_dto.director_output_dto.addContentToDictMappings(CSVLookup.model_construct(name="data_sources", - id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"), - version=1, - author=input_dto.config.app.author_name, - date = datetime.date.today(), - description= "A lookup file that will contain the data source objects for detections.", - lookup_type=Lookup_Type.csv, - file_path=data_sources_fake_yml_path)) + data_sources_lookup_csv_path = ( + input_dto.config.getPackageDirectoryPath() + / "lookups" + / "data_sources.csv" + ) + + DataSourceWriter.writeDataSourceCsv( + input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path + ) + input_dto.director_output_dto.addContentToDictMappings( + CSVLookup.model_construct( + name="data_sources", + id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"), + version=1, + author=input_dto.config.app.author_name, + date=datetime.date.today(), + description="A lookup file that will contain the data source objects for detections.", + lookup_type=Lookup_Type.csv, + file_path=data_sources_fake_yml_path, + ) + ) updated_conf_files.update(conf_output.writeHeaders()) - updated_conf_files.update(conf_output.writeLookups(input_dto.director_output_dto.lookups)) - updated_conf_files.update(conf_output.writeDetections(input_dto.director_output_dto.detections)) - updated_conf_files.update(conf_output.writeStories(input_dto.director_output_dto.stories)) - updated_conf_files.update(conf_output.writeBaselines(input_dto.director_output_dto.baselines)) - updated_conf_files.update(conf_output.writeInvestigations(input_dto.director_output_dto.investigations)) - updated_conf_files.update(conf_output.writeMacros(input_dto.director_output_dto.macros)) - updated_conf_files.update(conf_output.writeDashboards(input_dto.director_output_dto.dashboards)) + updated_conf_files.update( + conf_output.writeLookups(input_dto.director_output_dto.lookups) + ) + updated_conf_files.update( + conf_output.writeDetections(input_dto.director_output_dto.detections) + ) + updated_conf_files.update( + conf_output.writeStories(input_dto.director_output_dto.stories) + ) + updated_conf_files.update( + conf_output.writeBaselines(input_dto.director_output_dto.baselines) + ) + updated_conf_files.update( + conf_output.writeInvestigations( + input_dto.director_output_dto.investigations + ) + ) + updated_conf_files.update( + conf_output.writeMacros(input_dto.director_output_dto.macros) + ) + updated_conf_files.update( + conf_output.writeDashboards(input_dto.director_output_dto.dashboards) + ) updated_conf_files.update(conf_output.writeMiscellaneousAppFiles()) - - - - #Ensure that the conf file we just generated/update is syntactically valid + # Ensure that the conf file we just generated/update is syntactically valid for conf_file in updated_conf_files: - ConfWriter.validateConfFile(conf_file) - + ConfWriter.validateConfFile(conf_file) + conf_output.packageApp() - print(f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}") - + print( + f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}" + ) - if input_dto.config.build_api: + if input_dto.config.build_api: shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True) input_dto.config.getAPIPath().mkdir(parents=True) - api_json_output = ApiJsonOutput(input_dto.config.getAPIPath(), input_dto.config.app.label) + api_json_output = ApiJsonOutput( + input_dto.config.getAPIPath(), input_dto.config.app.label + ) api_json_output.writeDetections(input_dto.director_output_dto.detections) api_json_output.writeStories(input_dto.director_output_dto.stories) api_json_output.writeBaselines(input_dto.director_output_dto.baselines) - api_json_output.writeInvestigations(input_dto.director_output_dto.investigations) + api_json_output.writeInvestigations( + input_dto.director_output_dto.investigations + ) api_json_output.writeLookups(input_dto.director_output_dto.lookups) api_json_output.writeMacros(input_dto.director_output_dto.macros) api_json_output.writeDeployments(input_dto.director_output_dto.deployments) - - #create version file for sse api - version_file = input_dto.config.getAPIPath()/"version.json" - utc_time = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0,tzinfo=None).isoformat() - version_dict = {"version":{"name":f"v{input_dto.config.app.version}","published_at": f"{utc_time}Z" }} - with open(version_file,"w") as version_f: - json.dump(version_dict,version_f) - - print(f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}") - - return input_dto.director_output_dto \ No newline at end of file + # create version file for sse api + version_file = input_dto.config.getAPIPath() / "version.json" + utc_time = ( + datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0, tzinfo=None) + .isoformat() + ) + version_dict = { + "version": { + "name": f"v{input_dto.config.app.version}", + "published_at": f"{utc_time}Z", + } + } + with open(version_file, "w") as version_f: + json.dump(version_dict, version_f) + + print( + f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}" + ) + + return input_dto.director_output_dto diff --git a/contentctl/actions/deploy_acs.py b/contentctl/actions/deploy_acs.py index 2b07b2f0..b38e1273 100644 --- a/contentctl/actions/deploy_acs.py +++ b/contentctl/actions/deploy_acs.py @@ -4,39 +4,38 @@ class Deploy: - def execute(self, config: deploy_acs, appinspect_token:str) -> None: - - #The following common headers are used by both Clasic and Victoria + def execute(self, config: deploy_acs, appinspect_token: str) -> None: + # The following common headers are used by both Clasic and Victoria headers = { - 'Authorization': f'Bearer {config.splunk_cloud_jwt_token}', - 'ACS-Legal-Ack': 'Y' + "Authorization": f"Bearer {config.splunk_cloud_jwt_token}", + "ACS-Legal-Ack": "Y", } try: - - with open(config.getPackageFilePath(include_version=False),'rb') as app_data: - #request_data = app_data.read() + with open( + config.getPackageFilePath(include_version=False), "rb" + ) as app_data: + # request_data = app_data.read() if config.stack_type == StackType.classic: # Classic instead uses a form to store token and package # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps" - - form_data = { - 'token': (None, appinspect_token), - 'package': app_data - } - res = post(address, headers=headers, files = form_data) + + form_data = {"token": (None, appinspect_token), "package": app_data} + res = post(address, headers=headers, files=form_data) elif config.stack_type == StackType.victoria: # Victoria uses the X-Splunk-Authorization Header # It also uses --data-binary for the app content # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience - headers.update({'X-Splunk-Authorization': appinspect_token}) + headers.update({"X-Splunk-Authorization": appinspect_token}) address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria" res = post(address, headers=headers, data=app_data.read()) else: raise Exception(f"Unsupported stack type: '{config.stack_type}'") except Exception as e: - raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}") - + raise Exception( + f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}" + ) + try: # Request went through and completed, but may have returned a non-successful error code. # This likely includes a more verbose response describing the error @@ -48,8 +47,14 @@ def execute(self, config: deploy_acs, appinspect_token:str) -> None: except Exception: error_text = "No error text - request failed" formatted_error_text = pprint.pformat(error_text) - print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n" - "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.") - raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}") - - print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!") \ No newline at end of file + print( + "While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n" + "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase." + ) + raise Exception( + f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}" + ) + + print( + f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!" + ) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 3696dbfb..8a8dd741 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -1,12 +1,20 @@ -from typing import List,Union -from contentctl.objects.config import test, test_servers, Container,Infrastructure -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer -from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer +from typing import List, Union +from contentctl.objects.config import test, test_servers, Container, Infrastructure +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( + DetectionTestingInfrastructure, +) +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import ( + DetectionTestingInfrastructureContainer, +) +from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import ( + DetectionTestingInfrastructureServer, +) import signal import datetime + # from queue import Queue from dataclasses import dataclass + # import threading from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( DetectionTestingManagerOutputDto, @@ -23,7 +31,7 @@ @dataclass(frozen=False) class DetectionTestingManagerInputDto: - config: Union[test,test_servers] + config: Union[test, test_servers] detections: List[Detection] views: list[DetectionTestingView] @@ -60,15 +68,18 @@ def sigint_handler(signum, frame): print("*******************************") signal.signal(signal.SIGINT, sigint_handler) - - with concurrent.futures.ThreadPoolExecutor( - max_workers=len(self.input_dto.config.test_instances), - ) as instance_pool, concurrent.futures.ThreadPoolExecutor( - max_workers=len(self.input_dto.views) - ) as view_runner, concurrent.futures.ThreadPoolExecutor( - max_workers=len(self.input_dto.config.test_instances), - ) as view_shutdowner: + with ( + concurrent.futures.ThreadPoolExecutor( + max_workers=len(self.input_dto.config.test_instances), + ) as instance_pool, + concurrent.futures.ThreadPoolExecutor( + max_workers=len(self.input_dto.views) + ) as view_runner, + concurrent.futures.ThreadPoolExecutor( + max_workers=len(self.input_dto.config.test_instances), + ) as view_shutdowner, + ): # Start all the views future_views = { view_runner.submit(view.setup): view for view in self.input_dto.views @@ -124,20 +135,29 @@ def sigint_handler(signum, frame): return self.output_dto def create_DetectionTestingInfrastructureObjects(self): - #Make sure that, if we need to, we pull the appropriate container + # Make sure that, if we need to, we pull the appropriate container for infrastructure in self.input_dto.config.test_instances: - if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)): + if isinstance(self.input_dto.config, test) and isinstance( + infrastructure, Container + ): try: client = docker.from_env() except Exception: - raise Exception("Unable to connect to docker. Are you sure that docker is running on this host?") + raise Exception( + "Unable to connect to docker. Are you sure that docker is running on this host?" + ) try: - - parts = self.input_dto.config.container_settings.full_image_path.split(':') + parts = ( + self.input_dto.config.container_settings.full_image_path.split( + ":" + ) + ) if len(parts) != 2: - raise Exception(f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, " - f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'") - + raise Exception( + f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, " + f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'" + ) + print( f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...", end="", @@ -147,12 +167,15 @@ def create_DetectionTestingInfrastructureObjects(self): print("done!") break except Exception as e: - raise Exception(f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}") + raise Exception( + f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}" + ) already_staged_container_files = False for infrastructure in self.input_dto.config.test_instances: - - if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)): + if isinstance(self.input_dto.config, test) and isinstance( + infrastructure, Container + ): # Stage the files in the apps dir so that they can be passed directly to # subsequent containers. Do this here, instead of inside each container, to # avoid duplicate downloads/moves/copies @@ -162,18 +185,24 @@ def create_DetectionTestingInfrastructureObjects(self): self.detectionTestingInfrastructureObjects.append( DetectionTestingInfrastructureContainer( - global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto + global_config=self.input_dto.config, + infrastructure=infrastructure, + sync_obj=self.output_dto, ) ) - elif (isinstance(self.input_dto.config, test_servers) and isinstance(infrastructure, Infrastructure)): + elif isinstance(self.input_dto.config, test_servers) and isinstance( + infrastructure, Infrastructure + ): self.detectionTestingInfrastructureObjects.append( DetectionTestingInfrastructureServer( - global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto + global_config=self.input_dto.config, + infrastructure=infrastructure, + sync_obj=self.output_dto, ) ) else: - - raise Exception(f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}") - + raise Exception( + f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}" + ) diff --git a/contentctl/actions/detection_testing/GitService.py b/contentctl/actions/detection_testing/GitService.py index bbfb7c2d..646c384f 100644 --- a/contentctl/actions/detection_testing/GitService.py +++ b/contentctl/actions/detection_testing/GitService.py @@ -23,9 +23,6 @@ LOGGER = logging.getLogger(__name__) - - - class GitService(BaseModel): director: DirectorOutputDto config: test_common diff --git a/contentctl/actions/detection_testing/generate_detection_coverage_badge.py b/contentctl/actions/detection_testing/generate_detection_coverage_badge.py index 9962a4ef..749a6b75 100644 --- a/contentctl/actions/detection_testing/generate_detection_coverage_badge.py +++ b/contentctl/actions/detection_testing/generate_detection_coverage_badge.py @@ -2,7 +2,7 @@ import json import sys -RAW_BADGE_SVG = ''' +RAW_BADGE_SVG = """ @@ -19,47 +19,65 @@ {} {} -''' - - -parser = argparse.ArgumentParser(description='Use a summary.json file to generate a test coverage badge') -parser.add_argument('-i', "--input_summary_file", type=argparse.FileType('r'), required = True, - help='Summary file to use to generate the pass percentage badge') -parser.add_argument('-o', "--output_badge_file", type=argparse.FileType('w'), required = True, - help='Name of the badge to output') -parser.add_argument('-s', "--badge_string", type=str, required = True, - help='Name of the badge to output') - +""" + + +parser = argparse.ArgumentParser( + description="Use a summary.json file to generate a test coverage badge" +) +parser.add_argument( + "-i", + "--input_summary_file", + type=argparse.FileType("r"), + required=True, + help="Summary file to use to generate the pass percentage badge", +) +parser.add_argument( + "-o", + "--output_badge_file", + type=argparse.FileType("w"), + required=True, + help="Name of the badge to output", +) +parser.add_argument( + "-s", "--badge_string", type=str, required=True, help="Name of the badge to output" +) try: - results = parser.parse_args() + results = parser.parse_args() except Exception as e: - print(f"Error parsing arguments: {str(e)}") - exit(1) + print(f"Error parsing arguments: {str(e)}") + exit(1) try: - summary_info = json.loads(results.input_summary_file.read()) + summary_info = json.loads(results.input_summary_file.read()) except Exception as e: - print(f"Error loading {results.input_summary_file.name} JSON file: {str(e)}") - sys.exit(1) - -if 'summary' not in summary_info: - print("Missing 'summary' key in {results.input_summary_file.name}") - sys.exit(1) -elif 'PASS_RATE' not in summary_info['summary'] or 'TESTS_PASSED' not in summary_info['summary']: - print(f"Missing PASS_RATE in 'summary' section of {results.input_summary_file.name}") - sys.exit(1) -pass_percent = 100 * summary_info['summary']['PASS_RATE'] + print(f"Error loading {results.input_summary_file.name} JSON file: {str(e)}") + sys.exit(1) + +if "summary" not in summary_info: + print("Missing 'summary' key in {results.input_summary_file.name}") + sys.exit(1) +elif ( + "PASS_RATE" not in summary_info["summary"] + or "TESTS_PASSED" not in summary_info["summary"] +): + print( + f"Missing PASS_RATE in 'summary' section of {results.input_summary_file.name}" + ) + sys.exit(1) +pass_percent = 100 * summary_info["summary"]["PASS_RATE"] try: - results.output_badge_file.write(RAW_BADGE_SVG.format(results.badge_string, "{:2.1f}%".format(pass_percent))) + results.output_badge_file.write( + RAW_BADGE_SVG.format(results.badge_string, "{:2.1f}%".format(pass_percent)) + ) except Exception as e: - print(f"Error generating badge: {str(e)}") - sys.exit(1) + print(f"Error generating badge: {str(e)}") + sys.exit(1) print(f"Badge {results.output_badge_file.name} successfully generated!") sys.exit(0) - diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 42cad6c0..83e35a71 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -5,7 +5,7 @@ import configparser import json import datetime -import tqdm # type: ignore +import tqdm # type: ignore import pathlib from tempfile import TemporaryDirectory, mktemp from ssl import SSLEOFError, SSLZeroReturnError @@ -14,10 +14,10 @@ from typing import Union, Optional from pydantic import ConfigDict, BaseModel, PrivateAttr, Field, dataclasses -import requests # type: ignore -import splunklib.client as client # type: ignore -from splunklib.binding import HTTPError # type: ignore -from splunklib.results import JSONResultsReader, Message # type: ignore +import requests # type: ignore +import splunklib.client as client # type: ignore +from splunklib.binding import HTTPError # type: ignore +from splunklib.results import JSONResultsReader, Message # type: ignore import splunklib.results from urllib3 import disable_warnings import urllib.parse @@ -39,7 +39,7 @@ format_pbar_string, TestReportingType, FinalTestingStates, - TestingStates + TestingStates, ) @@ -48,9 +48,7 @@ class SetupTestGroupResults(BaseModel): success: bool = True duration: float = 0 start_time: float - model_config = ConfigDict( - arbitrary_types_allowed=True - ) + model_config = ConfigDict(arbitrary_types_allowed=True) class CleanupTestGroupResults(BaseModel): @@ -60,26 +58,31 @@ class CleanupTestGroupResults(BaseModel): class ContainerStoppedException(Exception): pass + + class CannotRunBaselineException(Exception): - # Support for testing detections with baselines + # Support for testing detections with baselines # does not currently exist in contentctl. - # As such, whenever we encounter a detection + # As such, whenever we encounter a detection # with baselines we should generate a descriptive # exception pass + class ReplayIndexDoesNotExistOnServer(Exception): - ''' + """ In order to replay data files into the Splunk Server for testing, they must be replayed into an index that exists. If that index does not exist, this error will be generated and raised before we try to do anything else with that Data File. - ''' + """ + pass + @dataclasses.dataclass(frozen=False) -class DetectionTestingManagerOutputDto(): +class DetectionTestingManagerOutputDto: inputQueue: list[Detection] = Field(default_factory=list) outputQueue: list[Detection] = Field(default_factory=list) currentTestingQueue: dict[str, Union[Detection, None]] = Field(default_factory=dict) @@ -101,9 +104,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC): _conn: client.Service = PrivateAttr() pbar: tqdm.tqdm = None start_time: Optional[float] = None - model_config = ConfigDict( - arbitrary_types_allowed=True - ) + model_config = ConfigDict(arbitrary_types_allowed=True) def __init__(self, **data): super().__init__(**data) @@ -131,7 +132,7 @@ def setup(self): bar_format=f"{self.get_name()} starting", miniters=0, mininterval=0, - file=stdout + file=stdout, ) self.start_time = time.time() @@ -140,14 +141,16 @@ def setup(self): (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), - (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + ( + self.create_replay_index, + f"Create index '{self.sync_obj.replay_index}'", + ), (self.get_all_indexes, "Getting all indexes from server"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), - (self.wait_for_ui_ready, "Finishing Setup") + (self.wait_for_ui_ready, "Finishing Setup"), ]: - self.format_pbar_string( TestReportingType.SETUP, self.get_name(), @@ -162,7 +165,9 @@ def setup(self): self.finish() return - self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") + self.format_pbar_string( + TestReportingType.SETUP, self.get_name(), "Finished Setup!" + ) def wait_for_ui_ready(self): self.get_conn() @@ -184,7 +189,9 @@ def configure_hec(self): name="DETECTION_TESTING_HEC", kind="http", index=self.sync_obj.replay_index, - indexes=",".join(self.all_indexes_on_server), # This allows the HEC to write to all indexes + indexes=",".join( + self.all_indexes_on_server + ), # This allows the HEC to write to all indexes useACK=True, ) self.hec_token = str(res.token) @@ -234,7 +241,6 @@ def connect_to_api(self, sleep_seconds: int = 5): while True: self.check_for_teardown() try: - conn = client.connect( host=self.infrastructure.instance_address, port=self.infrastructure.api_port, @@ -277,7 +283,6 @@ def connect_to_api(self, sleep_seconds: int = 5): time.sleep(1) def create_replay_index(self): - try: self.get_conn().indexes.create(name=self.sync_obj.replay_index) except HTTPError as e: @@ -292,7 +297,7 @@ def configure_imported_roles( self, imported_roles: list[str] = ["user", "power", "can_delete"], enterprise_security_roles: list[str] = ["ess_admin", "ess_analyst", "ess_user"], - ): + ): try: # Set which roles should be configured. For Enterprise Security/Integration Testing, # we must add some extra foles. @@ -334,9 +339,7 @@ def wait_for_conf_file(self, app_name: str, conf_file_name: str): self.check_for_teardown() time.sleep(1) try: - _ = self.get_conn().get( - f"configs/conf-{conf_file_name}", app=app_name - ) + _ = self.get_conn().get(f"configs/conf-{conf_file_name}", app=app_name) return except Exception: pass @@ -366,7 +369,7 @@ def configure_conf_file_datamodels(self, APP_NAME: str = "Splunk_SA_CIM"): parser.read(custom_acceleration_datamodels) if len(parser.keys()) > 1: self.pbar.write( - f"Read {len(parser)-1} custom datamodels from {str(custom_acceleration_datamodels)}!" + f"Read {len(parser) - 1} custom datamodels from {str(custom_acceleration_datamodels)}!" ) if not cim_acceleration_datamodels.is_file(): @@ -414,11 +417,15 @@ def execute(self): try: self.test_detection(detection) except ContainerStoppedException: - self.pbar.write(f"Warning - container was stopped when trying to execute detection [{self.get_name()}]") + self.pbar.write( + f"Warning - container was stopped when trying to execute detection [{self.get_name()}]" + ) self.finish() return except Exception as e: - self.pbar.write(f"Error testing detection: {type(e).__name__}: {str(e)}") + self.pbar.write( + f"Error testing detection: {type(e).__name__}: {str(e)}" + ) raise e finally: self.sync_obj.outputQueue.append(detection) @@ -460,22 +467,32 @@ def test_detection(self, detection: Detection) -> None: detection, test_group.integration_test, setup_results, - test_group.unit_test.result + test_group.unit_test.result, ) # cleanup - cleanup_results = self.cleanup_test_group(test_group, setup_results.start_time) + cleanup_results = self.cleanup_test_group( + test_group, setup_results.start_time + ) # update the results duration w/ the setup/cleanup time (for those not skipped) - if (test_group.unit_test.result is not None) and (not test_group.unit_test_skipped()): + if (test_group.unit_test.result is not None) and ( + not test_group.unit_test_skipped() + ): test_group.unit_test.result.duration = round( - test_group.unit_test.result.duration + setup_results.duration + cleanup_results.duration, - 2 + test_group.unit_test.result.duration + + setup_results.duration + + cleanup_results.duration, + 2, ) - if (test_group.integration_test.result is not None) and (not test_group.integration_test_skipped()): + if (test_group.integration_test.result is not None) and ( + not test_group.integration_test_skipped() + ): test_group.integration_test.result.duration = round( - test_group.integration_test.result.duration + setup_results.duration + cleanup_results.duration, - 2 + test_group.integration_test.result.duration + + setup_results.duration + + cleanup_results.duration, + 2, ) # Write test group status @@ -505,7 +522,7 @@ def setup_test_group(self, test_group: TestGroup) -> SetupTestGroupResults: TestReportingType.GROUP, test_group.name, TestingStates.BEGINNING_GROUP, - start_time=setup_start_time + start_time=setup_start_time, ) # https://github.com/WoLpH/python-progressbar/issues/164 # Use NullBar if there is more than 1 container or we are running @@ -554,8 +571,7 @@ def cleanup_test_group( # Return the cleanup metadata, adding start time and duration return CleanupTestGroupResults( - duration=time.time() - cleanup_start_time, - start_time=cleanup_start_time + duration=time.time() - cleanup_start_time, start_time=cleanup_start_time ) def format_pbar_string( @@ -589,17 +605,12 @@ def format_pbar_string( # invoke the helper method new_string = format_pbar_string( - self.pbar, - test_reporting_type, - test_name, - state, - start_time, - set_pbar + self.pbar, test_reporting_type, test_name, state, start_time, set_pbar ) # update sync status if needed if update_sync_status: - self.sync_obj.currentTestingQueue[self.get_name()] = { # type: ignore + self.sync_obj.currentTestingQueue[self.get_name()] = { # type: ignore "name": state, "search": "N/A", } @@ -612,7 +623,7 @@ def execute_unit_test( detection: Detection, test: UnitTest, setup_results: SetupTestGroupResults, - FORCE_ALL_TIME: bool = True + FORCE_ALL_TIME: bool = True, ): """ Execute a unit test and set its results appropriately @@ -656,7 +667,7 @@ def execute_unit_test( self.infrastructure, TestResultStatus.ERROR, exception=setup_results.exception, - duration=time.time() - test_start_time + duration=time.time() - test_start_time, ) # report the failure to the CLI @@ -686,10 +697,12 @@ def execute_unit_test( try: # Iterate over baselines (if any) for baseline in detection.baselines: - raise CannotRunBaselineException("Detection requires Execution of a Baseline, " - "however Baseline execution is not " - "currently supported in contentctl. Mark " - "this as manual_test.") + raise CannotRunBaselineException( + "Detection requires Execution of a Baseline, " + "however Baseline execution is not " + "currently supported in contentctl. Mark " + "this as manual_test." + ) self.retry_search_until_timeout(detection, test, kwargs, test_start_time) except CannotRunBaselineException as e: # Init the test result and record a failure if there was an issue during the search @@ -699,7 +712,7 @@ def execute_unit_test( self.infrastructure, TestResultStatus.ERROR, exception=e, - duration=time.time() - test_start_time + duration=time.time() - test_start_time, ) except ContainerStoppedException as e: raise e @@ -712,7 +725,7 @@ def execute_unit_test( self.infrastructure, TestResultStatus.ERROR, exception=e, - duration=time.time() - test_start_time + duration=time.time() - test_start_time, ) # Pause here if the terminate flag has NOT been set AND either of the below are true: @@ -724,7 +737,7 @@ def execute_unit_test( res = "ERROR" link = detection.search else: - res = test.result.status.upper() # type: ignore + res = test.result.status.upper() # type: ignore link = test.result.get_summary_dict()["sid_link"] self.format_pbar_string( @@ -746,7 +759,7 @@ def execute_unit_test( test.result = UnitTestResult( message=message, exception=ValueError(message), - status=TestResultStatus.ERROR + status=TestResultStatus.ERROR, ) # Report a pass @@ -811,7 +824,7 @@ def execute_integration_test( detection: Detection, test: IntegrationTest, setup_results: SetupTestGroupResults, - unit_test_result: Optional[UnitTestResult] + unit_test_result: Optional[UnitTestResult], ): """ Executes an integration test on the detection @@ -883,7 +896,7 @@ def execute_integration_test( ), exception=setup_results.exception, duration=round(time.time() - test_start_time, 2), - status=TestResultStatus.ERROR + status=TestResultStatus.ERROR, ) # report the failure to the CLI @@ -905,7 +918,7 @@ def execute_integration_test( pbar_data = PbarData( pbar=self.pbar, fq_test_name=f"{detection.name}:{test.name}", - start_time=test_start_time + start_time=test_start_time, ) # TODO (#228): consider reusing CorrelationSearch instances across test cases @@ -923,7 +936,7 @@ def execute_integration_test( test.result = IntegrationTestResult( message="TEST ERROR: unhandled exception in CorrelationSearch", exception=e, - status=TestResultStatus.ERROR + status=TestResultStatus.ERROR, ) # TODO (#229): when in interactive mode, cleanup should happen after user interaction @@ -935,7 +948,7 @@ def execute_integration_test( if test.result is None: res = "ERROR" else: - res = test.result.status.upper() # type: ignore + res = test.result.status.upper() # type: ignore # Get the link to the saved search in this specific instance link = f"https://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}" @@ -959,7 +972,7 @@ def execute_integration_test( test.result = IntegrationTestResult( message=message, exception=ValueError(message), - status=TestResultStatus.ERROR + status=TestResultStatus.ERROR, ) # Report a pass @@ -1028,7 +1041,10 @@ def pause_for_user(self, test: BaseTest) -> bool: # check if the behavior is to always pause if self.global_config.post_test_behavior == PostTestBehavior.always_pause: return True - elif self.global_config.post_test_behavior == PostTestBehavior.pause_on_failure: + elif ( + self.global_config.post_test_behavior + == PostTestBehavior.pause_on_failure + ): # If the behavior is to pause on failure, check for failure (either explicitly, or # just a lack of a result) if test.result is None or test.result.failed: @@ -1053,15 +1069,15 @@ def retry_search_until_timeout( """ # Get the start time and compute the timeout search_start_time = time.time() - search_stop_time = time.time() + self.sync_obj.timeout_seconds + search_stop_time = time.time() + self.sync_obj.timeout_seconds # Make a copy of the search string since we may # need to make some small changes to it below search = detection.search # Ensure searches that do not begin with '|' must begin with 'search ' - if not search.strip().startswith("|"): - if not search.strip().startswith("search "): + if not search.strip().startswith("|"): + if not search.strip().startswith("search "): search = f"search {search}" # exponential backoff for wait time @@ -1069,7 +1085,6 @@ def retry_search_until_timeout( # Retry until timeout while time.time() < search_stop_time: - # This loop allows us to capture shutdown events without being # stuck in an extended sleep. Remember that this raises an exception for _ in range(pow(2, tick - 1)): @@ -1078,7 +1093,7 @@ def retry_search_until_timeout( TestReportingType.UNIT, f"{detection.name}:{test.name}", TestingStates.PROCESSING, - start_time=start_time + start_time=start_time, ) time.sleep(1) @@ -1096,9 +1111,15 @@ def retry_search_until_timeout( # TODO (cmcginley): @ljstella you're removing this ultimately, right? # Consolidate a set of the distinct observable field names - observable_fields_set = set([o.name for o in detection.tags.observable]) # keeping this around for later - risk_object_fields_set = set([o.name for o in detection.tags.observable if "Victim" in o.role ]) # just the "Risk Objects" - threat_object_fields_set = set([o.name for o in detection.tags.observable if "Attacker" in o.role]) # just the "threat objects" + observable_fields_set = set( + [o.name for o in detection.tags.observable] + ) # keeping this around for later + risk_object_fields_set = set( + [o.name for o in detection.tags.observable if "Victim" in o.role] + ) # just the "Risk Objects" + threat_object_fields_set = set( + [o.name for o in detection.tags.observable if "Attacker" in o.role] + ) # just the "threat objects" # Ensure the search had at least one result if int(job.content.get("resultCount", "0")) > 0: @@ -1134,7 +1155,7 @@ def retry_search_until_timeout( duration=time.time() - search_start_time, ) - return + return # If we find one or more risk object fields that contain the string "null" then they were # not populated and we should throw an error. This can happen if there is a typo @@ -1144,9 +1165,11 @@ def retry_search_until_timeout( # TODO (cmcginley): @ljstella is this something we're keeping for testing as # well? for field in observable_fields_set: - if result.get(field, 'null') == 'null': + if result.get(field, "null") == "null": if field in risk_object_fields_set: - e = Exception(f"The risk object field {field} is missing in at least one result.") + e = Exception( + f"The risk object field {field} is missing in at least one result." + ) test.result.set_job_content( job.content, self.infrastructure, @@ -1177,7 +1200,9 @@ def retry_search_until_timeout( else: empty_fields = empty_fields.union(current_empty_fields) - missing_threat_objects = threat_object_fields_set - present_threat_objects + missing_threat_objects = ( + threat_object_fields_set - present_threat_objects + ) # Report a failure if there were empty fields in a threat object in all results if len(missing_threat_objects) > 0: e = Exception( @@ -1194,12 +1219,12 @@ def retry_search_until_timeout( return test.result.set_job_content( - job.content, - self.infrastructure, - TestResultStatus.PASS, - duration=time.time() - search_start_time, - ) - return + job.content, + self.infrastructure, + TestResultStatus.PASS, + duration=time.time() - search_start_time, + ) + return else: # Report a failure if there were no results at all @@ -1221,7 +1246,6 @@ def delete_attack_data(self, attack_data_files: list[TestAttackData]): splunk_search = f'search index="{index}" host="{host}" | delete' kwargs = {"exec_mode": "blocking"} try: - job = self.get_conn().jobs.create(splunk_search, **kwargs) results_stream = job.results(output_mode="json") # TODO: should we be doing something w/ this reader? @@ -1255,8 +1279,10 @@ def replay_attack_data_file( # Before attempting to replay the file, ensure that the index we want # to replay into actuall exists. If not, we should throw a detailed # exception that can easily be interpreted by the user. - if attack_data_file.custom_index is not None and \ - attack_data_file.custom_index not in self.all_indexes_on_server: + if ( + attack_data_file.custom_index is not None + and attack_data_file.custom_index not in self.all_indexes_on_server + ): raise ReplayIndexDoesNotExistOnServer( f"Unable to replay data file {attack_data_file.data} " f"into index '{attack_data_file.custom_index}'. " @@ -1265,13 +1291,17 @@ def replay_attack_data_file( ) tempfile = mktemp(dir=tmp_dir) - if not (str(attack_data_file.data).startswith("http://") or - str(attack_data_file.data).startswith("https://")) : + if not ( + str(attack_data_file.data).startswith("http://") + or str(attack_data_file.data).startswith("https://") + ): if pathlib.Path(str(attack_data_file.data)).is_file(): - self.format_pbar_string(TestReportingType.GROUP, - test_group.name, - "Copying Data", - test_group_start_time) + self.format_pbar_string( + TestReportingType.GROUP, + test_group.name, + "Copying Data", + test_group_start_time, + ) try: copyfile(str(attack_data_file.data), tempfile) @@ -1296,7 +1326,7 @@ def replay_attack_data_file( TestReportingType.GROUP, test_group.name, TestingStates.DOWNLOADING, - start_time=test_group_start_time + start_time=test_group_start_time, ) Utils.download_file_from_http( @@ -1314,7 +1344,7 @@ def replay_attack_data_file( TestReportingType.GROUP, test_group.name, TestingStates.REPLAYING, - start_time=test_group_start_time + start_time=test_group_start_time, ) self.hec_raw_replay(tempfile, attack_data_file) @@ -1398,7 +1428,6 @@ def hec_raw_replay( requested_acks = {"acks": [jsonResponse["ackId"]]} while True: try: - res = requests.post( url_with_hec_ack_path, json=requested_acks, @@ -1430,7 +1459,9 @@ def status(self): pass def finish(self): - self.pbar.bar_format = f"Finished running tests on instance: [{self.get_name()}]" + self.pbar.bar_format = ( + f"Finished running tests on instance: [{self.get_name()}]" + ) self.pbar.update() self.pbar.close() diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py index 0a19003a..ceced0eb 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py @@ -17,10 +17,12 @@ def start(self): # If we are configured to use the persistent container, then check and see if it's already # running. If so, just use it without additional configuration. try: - self.container = self.get_docker_client().containers.get(self.get_name()) + self.container = self.get_docker_client().containers.get( + self.get_name() + ) return except Exception: - #We did not find the container running, we will set it up + # We did not find the container running, we will set it up pass self.container = self.make_container() @@ -47,9 +49,10 @@ def get_docker_client(self): raise (Exception(f"Failed to get docker client: {str(e)}")) def check_for_teardown(self): - try: - container: docker.models.containers.Container = self.get_docker_client().containers.get(self.get_name()) + container: docker.models.containers.Container = ( + self.get_docker_client().containers.get(self.get_name()) + ) except Exception as e: if self.sync_obj.terminate is not True: self.pbar.write( @@ -57,7 +60,7 @@ def check_for_teardown(self): ) self.sync_obj.terminate = True else: - if container.status != 'running': + if container.status != "running": self.sync_obj.terminate = True self.container = None @@ -90,28 +93,33 @@ def make_container(self) -> docker.models.resource.Model: environment["SPLUNK_PASSWORD"] = self.infrastructure.splunk_app_password # Files have already been staged by the time that we call this. Files must only be staged # once, not staged by every container - environment["SPLUNK_APPS_URL"] = self.global_config.getContainerEnvironmentString(stage_file=False) + environment["SPLUNK_APPS_URL"] = ( + self.global_config.getContainerEnvironmentString(stage_file=False) + ) if ( self.global_config.splunk_api_username is not None and self.global_config.splunk_api_password is not None ): environment["SPLUNKBASE_USERNAME"] = self.global_config.splunk_api_username environment["SPLUNKBASE_PASSWORD"] = self.global_config.splunk_api_password - - def emit_docker_run_equivalent(): - environment_string = " ".join([f'-e "{k}={environment.get(k)}"' for k in environment.keys()]) - print(f"\n\ndocker run -d "\ - f"-p {self.infrastructure.web_ui_port}:8000 " - f"-p {self.infrastructure.hec_port}:8088 " - f"-p {self.infrastructure.api_port}:8089 " - f"{environment_string} " - f" --name {self.get_name()} " - f"--platform linux/amd64 " - f"{self.global_config.container_settings.full_image_path}\n\n") - #emit_docker_run_equivalent() - + environment_string = " ".join( + [f'-e "{k}={environment.get(k)}"' for k in environment.keys()] + ) + print( + f"\n\ndocker run -d " + f"-p {self.infrastructure.web_ui_port}:8000 " + f"-p {self.infrastructure.hec_port}:8088 " + f"-p {self.infrastructure.api_port}:8089 " + f"{environment_string} " + f" --name {self.get_name()} " + f"--platform linux/amd64 " + f"{self.global_config.container_settings.full_image_path}\n\n" + ) + + # emit_docker_run_equivalent() + container = self.get_docker_client().containers.create( self.global_config.container_settings.full_image_path, ports=ports_dict, @@ -119,20 +127,21 @@ def emit_docker_run_equivalent(): name=self.get_name(), mounts=mounts, detach=True, - platform="linux/amd64" + platform="linux/amd64", ) - + if self.global_config.enterpriseSecurityInApps(): - #ES sets up https, so make sure it is included in the link + # ES sets up https, so make sure it is included in the link address = f"https://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}" else: address = f"http://{self.infrastructure.instance_address}:{self.infrastructure.web_ui_port}" - print(f"\nStarted container with the following information:\n" - f"\tname : [{self.get_name()}]\n" - f"\taddress : [{address}]\n" - f"\tusername: [{self.infrastructure.splunk_app_username}]\n" - f"\tpassword: [{self.infrastructure.splunk_app_password}]\n" - ) + print( + f"\nStarted container with the following information:\n" + f"\tname : [{self.get_name()}]\n" + f"\taddress : [{address}]\n" + f"\tusername: [{self.infrastructure.splunk_app_username}]\n" + f"\tpassword: [{self.infrastructure.splunk_app_password}]\n" + ) return container @@ -146,10 +155,14 @@ def removeContainer(self, removeVolumes: bool = True, forceRemove: bool = True): return try: # If the user wants to persist the container (or use a previously configured container), then DO NOT remove it. - # Emit the following message, which they will see on initial setup and teardown at the end of the test. + # Emit the following message, which they will see on initial setup and teardown at the end of the test. if self.global_config.container_settings.leave_running: - print(f"\nContainer [{self.get_name()}] has NOT been terminated because 'contentctl_test.yml ---> infrastructure_config ---> persist_and_reuse_container = True'") - print(f"To remove it, please manually run the following at the command line: `docker container rm -fv {self.get_name()}`\n") + print( + f"\nContainer [{self.get_name()}] has NOT been terminated because 'contentctl_test.yml ---> infrastructure_config ---> persist_and_reuse_container = True'" + ) + print( + f"To remove it, please manually run the following at the command line: `docker container rm -fv {self.get_name()}`\n" + ) return # container was found, so now we try to remove it # v also removes volumes linked to the container diff --git a/contentctl/actions/detection_testing/progress_bar.py b/contentctl/actions/detection_testing/progress_bar.py index 5b5abd1a..183497d4 100644 --- a/contentctl/actions/detection_testing/progress_bar.py +++ b/contentctl/actions/detection_testing/progress_bar.py @@ -8,6 +8,7 @@ class TestReportingType(StrEnum): """ 5-char identifiers for the type of testing being reported on """ + # Reporting around general testing setup (e.g. infra, role configuration) SETUP = "SETUP" @@ -25,6 +26,7 @@ class TestingStates(StrEnum): """ Defined testing states """ + BEGINNING_GROUP = "Beginning Test Group" BEGINNING_TEST = "Beginning Test" DOWNLOADING = "Downloading Data" @@ -47,6 +49,7 @@ class FinalTestingStates(StrEnum): """ The possible final states for a test (for pbar reporting) """ + FAIL = "\x1b[0;30;41m" + "FAIL ".ljust(LONGEST_STATE) + "\x1b[0m" ERROR = "\x1b[0;30;41m" + "ERROR".ljust(LONGEST_STATE) + "\x1b[0m" PASS = "\x1b[0;30;42m" + "PASS ".ljust(LONGEST_STATE) + "\x1b[0m" diff --git a/contentctl/actions/detection_testing/views/DetectionTestingView.py b/contentctl/actions/detection_testing/views/DetectionTestingView.py index 98cc7122..b32e4223 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingView.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingView.py @@ -64,7 +64,7 @@ def getETA(self) -> datetime.timedelta: try: runtime = self.getRuntime() time_per_detection = runtime / num_tested - remaining_time = (num_untested+.5) * time_per_detection + remaining_time = (num_untested + 0.5) * time_per_detection remaining_time -= datetime.timedelta( microseconds=remaining_time.microseconds ) @@ -74,7 +74,14 @@ def getETA(self) -> datetime.timedelta: def getSummaryObject( self, - test_result_fields: list[str] = ["success", "message", "exception", "status", "duration", "wait_duration"], + test_result_fields: list[str] = [ + "success", + "message", + "exception", + "status", + "duration", + "wait_duration", + ], test_job_fields: list[str] = ["resultCount", "runDuration"], ) -> dict[str, dict[str, Any] | list[dict[str, Any]] | str]: """ @@ -110,11 +117,11 @@ def getSummaryObject( total_skipped += 1 # Aggregate production status metrics - if detection.status == DetectionStatus.production: + if detection.status == DetectionStatus.production: total_production += 1 - elif detection.status == DetectionStatus.experimental: + elif detection.status == DetectionStatus.experimental: total_experimental += 1 - elif detection.status == DetectionStatus.deprecated: + elif detection.status == DetectionStatus.deprecated: total_deprecated += 1 # Check if the detection is manual_test @@ -128,19 +135,11 @@ def getSummaryObject( tested_detections.append(summary) # Sort tested detections s.t. all failures appear first, then by name - tested_detections.sort( - key=lambda x: ( - x["success"], - x["name"] - ) - ) + tested_detections.sort(key=lambda x: (x["success"], x["name"])) # Sort skipped detections s.t. detections w/ tests appear before those w/o, then by name skipped_detections.sort( - key=lambda x: ( - 0 if len(x["tests"]) > 0 else 1, - x["name"] - ) + key=lambda x: (0 if len(x["tests"]) > 0 else 1, x["name"]) ) # TODO (#267): Align test reporting more closely w/ status enums (as it relates to @@ -170,9 +169,7 @@ def getSummaryObject( percent_complete = Utils.getPercent( len(tested_detections), len(untested_detections), 1 ) - success_rate = Utils.getPercent( - total_pass, total_tested_detections, 1 - ) + success_rate = Utils.getPercent(total_pass, total_tested_detections, 1) # TODO (#230): expand testing metrics reported (and make nested) # Construct and return the larger results dict diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index 7ae1c8c3..2050fc10 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -38,15 +38,12 @@ def setup(self): miniters=0, mininterval=0, ) - self.format_pbar( - len(self.sync_obj.outputQueue), len(self.sync_obj.inputQueue) - ) + self.format_pbar(len(self.sync_obj.outputQueue), len(self.sync_obj.inputQueue)) self.showStatus() # TODO (#267): Align test reporting more closely w/ status enums (as it relates to "untested") def showStatus(self, interval: int = 1): - while True: summary = self.getSummaryObject() diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewFile.py b/contentctl/actions/detection_testing/views/DetectionTestingViewFile.py index 0e18b1c7..40f5be9e 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewFile.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewFile.py @@ -13,7 +13,7 @@ class DetectionTestingViewFile(DetectionTestingView): output_filename: str = OUTPUT_FILENAME def getOutputFilePath(self) -> pathlib.Path: - folder_path = pathlib.Path('.') / self.output_folder + folder_path = pathlib.Path(".") / self.output_folder output_file = folder_path / self.output_filename return output_file @@ -22,7 +22,7 @@ def setup(self): pass def stop(self): - folder_path = pathlib.Path('.') / self.output_folder + folder_path = pathlib.Path(".") / self.output_folder output_file = self.getOutputFilePath() folder_path.mkdir(parents=True, exist_ok=True) diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py b/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py index cd50d978..d7cf73fa 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py @@ -102,9 +102,7 @@ def log_exception(*args, **kwargs): class DetectionTestingViewWeb(DetectionTestingView): bottleApp: Bottle = Bottle() server: SimpleWebServer = SimpleWebServer(host="0.0.0.0", port=DEFAULT_WEB_UI_PORT) - model_config = ConfigDict( - arbitrary_types_allowed=True - ) + model_config = ConfigDict(arbitrary_types_allowed=True) def setup(self): self.bottleApp.route("/", callback=self.showStatus) @@ -123,7 +121,6 @@ def setup(self): print(f"Could not open webbrowser for status page: {str(e)}") def stop(self): - if self.server.server is None: print("Web Server is not running anyway - nothing to shut down") return diff --git a/contentctl/actions/doc_gen.py b/contentctl/actions/doc_gen.py index ed5b607a..c9d89896 100644 --- a/contentctl/actions/doc_gen.py +++ b/contentctl/actions/doc_gen.py @@ -10,17 +10,21 @@ class DocGenInputDto: director_input_dto: DirectorInputDto -class DocGen: +class DocGen: def execute(self, input_dto: DocGenInputDto) -> None: - director_output_dto = DirectorOutputDto([],[],[],[],[],[],[],[],[],[]) + director_output_dto = DirectorOutputDto([], [], [], [], [], [], [], [], [], []) director = Director(director_output_dto) director.execute(input_dto.director_input_dto) doc_md_output = DocMdOutput() doc_md_output.writeObjects( - [director_output_dto.stories, director_output_dto.detections, director_output_dto.playbooks], - os.path.join(input_dto.director_input_dto.input_path, "docs") + [ + director_output_dto.stories, + director_output_dto.detections, + director_output_dto.playbooks, + ], + os.path.join(input_dto.director_input_dto.input_path, "docs"), ) - print('Generating Docs of security content successful.') \ No newline at end of file + print("Generating Docs of security content successful.") diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py index 9a57cd49..d99644a2 100644 --- a/contentctl/actions/initialize.py +++ b/contentctl/actions/initialize.py @@ -1,4 +1,3 @@ - import shutil import os import pathlib @@ -7,57 +6,70 @@ class Initialize: - def execute(self, config: test) -> None: # construct a test object from the init object # This way we can easily populate a yml with ALL the important - # fields for validating, building, and testing your app. - - YmlWriter.writeYmlFile(str(config.path/'contentctl.yml'), config.model_dump()) - - - #Create the following empty directories: - for emptyDir in ['lookups', 'baselines', 'data_sources', 'docs', 'reporting', 'investigations', - 'detections/application', 'detections/cloud', 'detections/endpoint', - 'detections/network', 'detections/web', 'macros', 'stories']: - #Throw an error if this directory already exists - (config.path/emptyDir).mkdir(exist_ok=False, parents=True) + # fields for validating, building, and testing your app. + + YmlWriter.writeYmlFile(str(config.path / "contentctl.yml"), config.model_dump()) + + # Create the following empty directories: + for emptyDir in [ + "lookups", + "baselines", + "data_sources", + "docs", + "reporting", + "investigations", + "detections/application", + "detections/cloud", + "detections/endpoint", + "detections/network", + "detections/web", + "macros", + "stories", + ]: + # Throw an error if this directory already exists + (config.path / emptyDir).mkdir(exist_ok=False, parents=True) # If this is not a bare config, then populate # a small amount of content into the directories if not config.bare: - #copy the contents of all template directories + # copy the contents of all template directories for templateDir, targetDir in [ - ('../templates/detections/', 'detections'), - ('../templates/data_sources/', 'data_sources'), - ('../templates/macros/', 'macros'), - ('../templates/stories/', 'stories'), + ("../templates/detections/", "detections"), + ("../templates/data_sources/", "data_sources"), + ("../templates/macros/", "macros"), + ("../templates/stories/", "stories"), ]: - source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir - target_directory = config.path/targetDir - - # Do not throw an exception if the directory exists. In fact, it was + source_directory = pathlib.Path(os.path.dirname(__file__)) / templateDir + target_directory = config.path / targetDir + + # Do not throw an exception if the directory exists. In fact, it was # created above when the structure of the app was created. shutil.copytree(source_directory, target_directory, dirs_exist_ok=True) - + # The contents of app_template must ALWAYS be copied because it contains # several special files. # For now, we also copy the deployments because the ability to create custom # deployment files is limited with built-in functionality. for templateDir, targetDir in [ - ('../templates/app_template/', 'app_template'), - ('../templates/deployments/', 'deployments') + ("../templates/app_template/", "app_template"), + ("../templates/deployments/", "deployments"), ]: - source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir - target_directory = config.path/targetDir - #Throw an exception if the target exists + source_directory = pathlib.Path(os.path.dirname(__file__)) / templateDir + target_directory = config.path / targetDir + # Throw an exception if the target exists shutil.copytree(source_directory, target_directory, dirs_exist_ok=False) # Create a README.md file. Note that this is the README.md for the repository, not the # one which will actually be packaged into the app. That is located in the app_template folder. - shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md') - - - print(f"The app '{config.app.title}' has been initialized. " - "Please run 'contentctl new --type {detection,story}' to create new content") + shutil.copyfile( + pathlib.Path(os.path.dirname(__file__)) / "../templates/README.md", + "README.md", + ) + print( + f"The app '{config.app.title}' has been initialized. " + "Please run 'contentctl new --type {detection,story}' to create new content" + ) diff --git a/contentctl/actions/inspect.py b/contentctl/actions/inspect.py index 9938b300..e5ebcb06 100644 --- a/contentctl/actions/inspect.py +++ b/contentctl/actions/inspect.py @@ -16,7 +16,7 @@ DetectionIDError, DetectionMissingError, VersionDecrementedError, - VersionBumpingError + VersionBumpingError, ) @@ -26,10 +26,8 @@ class InspectInputDto: class Inspect: - def execute(self, config: inspect) -> str: if config.build_app or config.build_api: - self.inspectAppCLI(config) appinspect_token = self.inspectAppAPI(config) @@ -48,9 +46,13 @@ def getElapsedTime(self, startTime: float) -> datetime.timedelta: def inspectAppAPI(self, config: inspect) -> str: session = Session() - session.auth = HTTPBasicAuth(config.splunk_api_username, config.splunk_api_password) - if config.stack_type not in ['victoria', 'classic']: - raise Exception(f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'") + session.auth = HTTPBasicAuth( + config.splunk_api_username, config.splunk_api_password + ) + if config.stack_type not in ["victoria", "classic"]: + raise Exception( + f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'" + ) APPINSPECT_API_LOGIN = "https://api.splunk.com/2.0/rest/login/splunk" @@ -59,21 +61,25 @@ def inspectAppAPI(self, config: inspect) -> str: res.raise_for_status() authorization_bearer = res.json().get("data", {}).get("token", None) - APPINSPECT_API_VALIDATION_REQUEST = "https://appinspect.splunk.com/v1/app/validate" + APPINSPECT_API_VALIDATION_REQUEST = ( + "https://appinspect.splunk.com/v1/app/validate" + ) headers = { "Authorization": f"bearer {authorization_bearer}", - "Cache-Control": "no-cache" + "Cache-Control": "no-cache", } package_path = config.getPackageFilePath(include_version=False) if not package_path.is_file(): - raise Exception(f"Cannot run Appinspect API on App '{config.app.title}' - " - f"no package exists as expected path '{package_path}'.\nAre you " - "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?") + raise Exception( + f"Cannot run Appinspect API on App '{config.app.title}' - " + f"no package exists as expected path '{package_path}'.\nAre you " + "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?" + ) files = { "app_package": open(package_path, "rb"), - "included_tags": (None, "cloud") + "included_tags": (None, "cloud"), } res = post(APPINSPECT_API_VALIDATION_REQUEST, headers=headers, files=files) @@ -82,26 +88,27 @@ def inspectAppAPI(self, config: inspect) -> str: request_id = res.json().get("request_id", None) APPINSPECT_API_VALIDATION_STATUS = f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}?included_tags=private_{config.stack_type}" - headers = headers = { - "Authorization": f"bearer {authorization_bearer}" - } + headers = headers = {"Authorization": f"bearer {authorization_bearer}"} startTime = timeit.default_timer() # the first time, wait for 40 seconds. subsequent times, wait for less. # this is because appinspect takes some time to return, so there is no sense # checking many times when we know it will take at least 40 seconds to run. iteration_wait_time = 40 while True: - res = get(APPINSPECT_API_VALIDATION_STATUS, headers=headers) res.raise_for_status() status = res.json().get("status", None) if status in ["PROCESSING", "PREPARING"]: - print(f"[{self.getElapsedTime(startTime)}] Appinspect API is {status}...") + print( + f"[{self.getElapsedTime(startTime)}] Appinspect API is {status}..." + ) time.sleep(iteration_wait_time) iteration_wait_time = 1 continue elif status == "SUCCESS": - print(f"[{self.getElapsedTime(startTime)}] Appinspect API has finished!") + print( + f"[{self.getElapsedTime(startTime)}] Appinspect API has finished!" + ) break else: raise Exception(f"Error - Unknown Appinspect API status '{status}'") @@ -111,7 +118,7 @@ def inspectAppAPI(self, config: inspect) -> str: # Get human-readable HTML report headers = headers = { "Authorization": f"bearer {authorization_bearer}", - "Content-Type": "text/html" + "Content-Type": "text/html", } res = get(APPINSPECT_API_REPORT, headers=headers) res.raise_for_status() @@ -120,7 +127,7 @@ def inspectAppAPI(self, config: inspect) -> str: # Get JSON report for processing headers = headers = { "Authorization": f"bearer {authorization_bearer}", - "Content-Type": "application/json" + "Content-Type": "application/json", } res = get(APPINSPECT_API_REPORT, headers=headers) res.raise_for_status() @@ -128,8 +135,12 @@ def inspectAppAPI(self, config: inspect) -> str: # Just get app path here to avoid long function calls in the open() calls below appPath = config.getPackageFilePath(include_version=True) - appinpect_html_path = appPath.with_suffix(appPath.suffix+".appinspect_api_results.html") - appinspect_json_path = appPath.with_suffix(appPath.suffix+".appinspect_api_results.json") + appinpect_html_path = appPath.with_suffix( + appPath.suffix + ".appinspect_api_results.html" + ) + appinspect_json_path = appPath.with_suffix( + appPath.suffix + ".appinspect_api_results.json" + ) # Use the full path of the app, but update the suffix to include info about appinspect with open(appinpect_html_path, "wb") as report: report.write(report_html) @@ -148,9 +159,15 @@ def inspectAppCLI(self, config: inspect) -> None: "\t - https://dev.splunk.com/enterprise/docs/developapps/testvalidate/appinspect/useappinspectclitool/" ) from splunk_appinspect.main import ( - validate, MODE_OPTION, APP_PACKAGE_ARGUMENT, OUTPUT_FILE_OPTION, - LOG_FILE_OPTION, INCLUDED_TAGS_OPTION, EXCLUDED_TAGS_OPTION, - TEST_MODE) + validate, + MODE_OPTION, + APP_PACKAGE_ARGUMENT, + OUTPUT_FILE_OPTION, + LOG_FILE_OPTION, + INCLUDED_TAGS_OPTION, + EXCLUDED_TAGS_OPTION, + TEST_MODE, + ) except Exception as e: print(e) # print("******WARNING******") @@ -175,10 +192,18 @@ def inspectAppCLI(self, config: inspect) -> None: included_tags = [] excluded_tags = [] - appinspect_output = self.dist/f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_results.json" - appinspect_logging = self.dist/f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_logging.log" + appinspect_output = ( + self.dist + / f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_results.json" + ) + appinspect_logging = ( + self.dist + / f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_logging.log" + ) try: - arguments_list = [(APP_PACKAGE_ARGUMENT, str(self.getPackagePath(include_version=False)))] + arguments_list = [ + (APP_PACKAGE_ARGUMENT, str(self.getPackagePath(include_version=False))) + ] options_list = [] options_list += [MODE_OPTION, TEST_MODE] options_list += [OUTPUT_FILE_OPTION, str(appinspect_output)] @@ -198,16 +223,22 @@ def inspectAppCLI(self, config: inspect) -> None: # The sys.exit called inside of appinspect validate closes stdin. We need to # reopen it. sys.stdin = open("/dev/stdin", "r") - print(f"AppInspect passed! Please check [ {appinspect_output} , {appinspect_logging} ] for verbose information.") + print( + f"AppInspect passed! Please check [ {appinspect_output} , {appinspect_logging} ] for verbose information." + ) else: - if sys.version.startswith('3.11') or sys.version.startswith('3.12'): - raise Exception("At this time, AppInspect may fail on valid apps under Python>=3.11 with " - "the error 'global flags not at the start of the expression at position 1'. " - "If you encounter this error, please run AppInspect on a version of Python " - "<3.11. This issue is currently tracked. Please review the appinspect " - "report output above for errors.") + if sys.version.startswith("3.11") or sys.version.startswith("3.12"): + raise Exception( + "At this time, AppInspect may fail on valid apps under Python>=3.11 with " + "the error 'global flags not at the start of the expression at position 1'. " + "If you encounter this error, please run AppInspect on a version of Python " + "<3.11. This issue is currently tracked. Please review the appinspect " + "report output above for errors." + ) else: - raise Exception("AppInspect Failure - Please review the appinspect report output above for errors.") + raise Exception( + "AppInspect Failure - Please review the appinspect report output above for errors." + ) finally: # appinspect outputs the log in json format, but does not format it to be easier # to read (it is all in one line). Read back that file and write it so it @@ -217,20 +248,26 @@ def inspectAppCLI(self, config: inspect) -> None: self.parseAppinspectJsonLogFile(appinspect_output) def parseAppinspectJsonLogFile( - self, - logfile_path: pathlib.Path, - status_types: list[str] = ["error", "failure", "manual_check", "warning"], - exception_types: list[str] = ["error", "failure", "manual_check"] + self, + logfile_path: pathlib.Path, + status_types: list[str] = ["error", "failure", "manual_check", "warning"], + exception_types: list[str] = ["error", "failure", "manual_check"], ) -> None: if not set(exception_types).issubset(set(status_types)): - raise Exception(f"Error - exception_types {exception_types} MUST be a subset of status_types {status_types}, but it is not") + raise Exception( + f"Error - exception_types {exception_types} MUST be a subset of status_types {status_types}, but it is not" + ) with open(logfile_path, "r+") as logfile: j = json.load(logfile) # Seek back to the beginning of the file. We don't need to clear # it sice we will always write AT LEAST the same number of characters # back as we read (due to the addition of whitespace) logfile.seek(0) - json.dump(j, logfile, indent=3, ) + json.dump( + j, + logfile, + indent=3, + ) reports = j.get("reports", []) if len(reports) != 1: @@ -240,7 +277,9 @@ def parseAppinspectJsonLogFile( for group in reports[0].get("groups", []): for check in group.get("checks", []): if check.get("result", "") in status_types: - verbose_errors.append(f" - {check.get('result','')} [{group.get('name','NONAME')}: {check.get('name', 'NONAME')}]") + verbose_errors.append( + f" - {check.get('result', '')} [{group.get('name', 'NONAME')}: {check.get('name', 'NONAME')}]" + ) verbose_errors.sort() summary = j.get("summary", None) @@ -250,17 +289,21 @@ def parseAppinspectJsonLogFile( generated_exception = False for key in status_types: if summary.get(key, 0) > 0: - msgs.append(f" - {summary.get(key,0)} {key}s") + msgs.append(f" - {summary.get(key, 0)} {key}s") if key in exception_types: generated_exception = True if len(msgs) > 0 or len(verbose_errors): - summary = '\n'.join(msgs) - details = '\n'.join(verbose_errors) + summary = "\n".join(msgs) + details = "\n".join(verbose_errors) summary = f"{summary}\nDetails:\n{details}" if generated_exception: - raise Exception(f"AppInspect found [{','.join(exception_types)}] that MUST be addressed to pass AppInspect API:\n{summary}") + raise Exception( + f"AppInspect found [{','.join(exception_types)}] that MUST be addressed to pass AppInspect API:\n{summary}" + ) else: - print(f"AppInspect found [{','.join(status_types)}] that MAY cause a failure during AppInspect API:\n{summary}") + print( + f"AppInspect found [{','.join(status_types)}] that MAY cause a failure during AppInspect API:\n{summary}" + ) else: print("AppInspect was successful!") @@ -283,12 +326,12 @@ def check_detection_metadata(self, config: inspect) -> None: current_build_conf = SavedsearchesConf.init_from_package( package_path=config.getPackageFilePath(include_version=False), app_name=config.app.label, - appid=config.app.appid + appid=config.app.appid, ) previous_build_conf = SavedsearchesConf.init_from_package( package_path=config.get_previous_package_file_path(), app_name=config.app.label, - appid=config.app.appid + appid=config.app.appid, ) # Compare the conf files @@ -298,31 +341,41 @@ def check_detection_metadata(self, config: inspect) -> None: # No detections should be removed from build to build if rule_name not in current_build_conf.detection_stanzas: if config.suppress_missing_content_exceptions: - print(f"[SUPPRESSED] {DetectionMissingError(rule_name=rule_name).long_message}") + print( + f"[SUPPRESSED] {DetectionMissingError(rule_name=rule_name).long_message}" + ) else: - validation_errors[rule_name].append(DetectionMissingError(rule_name=rule_name)) + validation_errors[rule_name].append( + DetectionMissingError(rule_name=rule_name) + ) continue # Pull out the individual stanza for readability previous_stanza = previous_build_conf.detection_stanzas[rule_name] current_stanza = current_build_conf.detection_stanzas[rule_name] # Detection IDs should not change - if current_stanza.metadata.detection_id != previous_stanza.metadata.detection_id: + if ( + current_stanza.metadata.detection_id + != previous_stanza.metadata.detection_id + ): validation_errors[rule_name].append( DetectionIDError( rule_name=rule_name, current_id=current_stanza.metadata.detection_id, - previous_id=previous_stanza.metadata.detection_id + previous_id=previous_stanza.metadata.detection_id, ) ) # Versions should never decrement in successive builds - if current_stanza.metadata.detection_version < previous_stanza.metadata.detection_version: + if ( + current_stanza.metadata.detection_version + < previous_stanza.metadata.detection_version + ): validation_errors[rule_name].append( VersionDecrementedError( rule_name=rule_name, current_version=current_stanza.metadata.detection_version, - previous_version=previous_stanza.metadata.detection_version + previous_version=previous_stanza.metadata.detection_version, ) ) @@ -332,12 +385,14 @@ def check_detection_metadata(self, config: inspect) -> None: VersionBumpingError( rule_name=rule_name, current_version=current_stanza.metadata.detection_version, - previous_version=previous_stanza.metadata.detection_version + previous_version=previous_stanza.metadata.detection_version, ) ) # Convert our dict mapping to a flat list of errors for use in reporting - validation_error_list = [x for inner_list in validation_errors.values() for x in inner_list] + validation_error_list = [ + x for inner_list in validation_errors.values() for x in inner_list + ] # Report failure/success print("\nDetection Metadata Validation:") @@ -350,11 +405,13 @@ def check_detection_metadata(self, config: inspect) -> None: print(f"\t\t🔸 {error.short_message}") else: # If no errors in the list, report success - print("\t✅ Detection metadata looks good and all versions were bumped appropriately :)") + print( + "\t✅ Detection metadata looks good and all versions were bumped appropriately :)" + ) # Raise an ExceptionGroup for all validation issues if len(validation_error_list) > 0: raise ExceptionGroup( "Validation errors when comparing detection stanzas in current and previous build:", - validation_error_list - ) \ No newline at end of file + validation_error_list, + ) diff --git a/contentctl/actions/new_content.py b/contentctl/actions/new_content.py index e57fc982..51ca916d 100644 --- a/contentctl/actions/new_content.py +++ b/contentctl/actions/new_content.py @@ -5,28 +5,34 @@ import uuid from datetime import datetime import pathlib -from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract +from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import ( + SecurityContentObject_Abstract, +) from contentctl.output.yml_writer import YmlWriter from contentctl.objects.enums import AssetType -from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, SES_OBSERVABLE_ROLE_MAPPING +from contentctl.objects.constants import ( + SES_OBSERVABLE_TYPE_MAPPING, + SES_OBSERVABLE_ROLE_MAPPING, +) + + class NewContent: UPDATE_PREFIX = "__UPDATE__" - + DEFAULT_DRILLDOWN_DEF = [ { "name": f'View the detection results for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', "search": f'%original_detection_search% | search "${UPDATE_PREFIX}FIRST_RISK_OBJECT = "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" second_observable_type_here = "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', - "earliest_offset": '$info_min_time$', - "latest_offset": '$info_max_time$' + "earliest_offset": "$info_min_time$", + "latest_offset": "$info_max_time$", }, { "name": f'View risk events for the last 7 days for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"', "search": f'| from datamodel Risk.All_Risk | search normalized_risk_object IN ("${UPDATE_PREFIX}FIRST_RISK_OBJECT$", "${UPDATE_PREFIX}SECOND_RISK_OBJECT$") starthoursago=168 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`', - "earliest_offset": '$info_min_time$', - "latest_offset": '$info_max_time$' - } + "earliest_offset": "$info_min_time$", + "latest_offset": "$info_max_time$", + }, ] - def buildDetection(self) -> tuple[dict[str, Any], str]: questions = NewContentQuestions.get_questions_detection() @@ -38,7 +44,9 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: raise ValueError("User didn't answer one or more questions!") data_source_field = ( - answers["data_source"] if len(answers["data_source"]) > 0 else [f"{NewContent.UPDATE_PREFIX} zero or more data_sources"] + answers["data_source"] + if len(answers["data_source"]) > 0 + else [f"{NewContent.UPDATE_PREFIX} zero or more data_sources"] ) file_name = ( answers["detection_name"] @@ -49,12 +57,16 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: .lower() ) - #Minimum lenght for a mitre tactic is 5 characters: T1000 + # Minimum lenght for a mitre tactic is 5 characters: T1000 if len(answers["mitre_attack_ids"]) >= 5: - mitre_attack_ids = [x.strip() for x in answers["mitre_attack_ids"].split(",")] + mitre_attack_ids = [ + x.strip() for x in answers["mitre_attack_ids"].split(",") + ] else: - #string was too short, so just put a placeholder - mitre_attack_ids = [f"{NewContent.UPDATE_PREFIX} zero or more mitre_attack_ids"] + # string was too short, so just put a placeholder + mitre_attack_ids = [ + f"{NewContent.UPDATE_PREFIX} zero or more mitre_attack_ids" + ] output_file_answers: dict[str, Any] = { "name": answers["detection_name"], @@ -69,17 +81,27 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: "search": f"{answers['detection_search']} | `{file_name}_filter`", "how_to_implement": f"{NewContent.UPDATE_PREFIX} how to implement your search", "known_false_positives": f"{NewContent.UPDATE_PREFIX} known false positives for your search", - "references": [f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search"], + "references": [ + f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search" + ], "drilldown_searches": NewContent.DEFAULT_DRILLDOWN_DEF, "tags": { - "analytic_story": [f"{NewContent.UPDATE_PREFIX} by providing zero or more analytic stories"], + "analytic_story": [ + f"{NewContent.UPDATE_PREFIX} by providing zero or more analytic stories" + ], "asset_type": f"{NewContent.UPDATE_PREFIX} by providing and asset type from {list(AssetType._value2member_map_)}", "confidence": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100", "impact": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100", "message": f"{NewContent.UPDATE_PREFIX} by providing a risk message. Fields in your search results can be referenced using $fieldName$", "mitre_attack_id": mitre_attack_ids, "observable": [ - {"name": f"{NewContent.UPDATE_PREFIX} the field name of the observable. This is a field that exists in your search results.", "type": f"{NewContent.UPDATE_PREFIX} the type of your observable from the list {list(SES_OBSERVABLE_TYPE_MAPPING.keys())}.", "role": [f"{NewContent.UPDATE_PREFIX} the role from the list {list(SES_OBSERVABLE_ROLE_MAPPING.keys())}"]} + { + "name": f"{NewContent.UPDATE_PREFIX} the field name of the observable. This is a field that exists in your search results.", + "type": f"{NewContent.UPDATE_PREFIX} the type of your observable from the list {list(SES_OBSERVABLE_TYPE_MAPPING.keys())}.", + "role": [ + f"{NewContent.UPDATE_PREFIX} the role from the list {list(SES_OBSERVABLE_ROLE_MAPPING.keys())}" + ], + } ], "product": [ "Splunk Enterprise", @@ -106,44 +128,54 @@ def buildDetection(self) -> tuple[dict[str, Any], str]: if answers["detection_type"] not in ["TTP", "Anomaly", "Correlation"]: del output_file_answers["drilldown_searches"] - return output_file_answers, answers['detection_kind'] + return output_file_answers, answers["detection_kind"] def buildStory(self) -> dict[str, Any]: questions = NewContentQuestions.get_questions_story() answers = questionary.prompt( - questions, - kbi_msg="User did not answer all of the prompt questions. Exiting...") + questions, + kbi_msg="User did not answer all of the prompt questions. Exiting...", + ) if not answers: raise ValueError("User didn't answer one or more questions!") - answers['name'] = answers['story_name'] - del answers['story_name'] - answers['id'] = str(uuid.uuid4()) - answers['version'] = 1 - answers['date'] = datetime.today().strftime('%Y-%m-%d') - answers['author'] = answers['story_author'] - del answers['story_author'] - answers['description'] = 'UPDATE_DESCRIPTION' - answers['narrative'] = 'UPDATE_NARRATIVE' - answers['references'] = [] - answers['tags'] = dict() - answers['tags']['category'] = answers['category'] - del answers['category'] - answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - answers['tags']['usecase'] = answers['usecase'] - del answers['usecase'] - answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE'] + answers["name"] = answers["story_name"] + del answers["story_name"] + answers["id"] = str(uuid.uuid4()) + answers["version"] = 1 + answers["date"] = datetime.today().strftime("%Y-%m-%d") + answers["author"] = answers["story_author"] + del answers["story_author"] + answers["description"] = "UPDATE_DESCRIPTION" + answers["narrative"] = "UPDATE_NARRATIVE" + answers["references"] = [] + answers["tags"] = dict() + answers["tags"]["category"] = answers["category"] + del answers["category"] + answers["tags"]["product"] = [ + "Splunk Enterprise", + "Splunk Enterprise Security", + "Splunk Cloud", + ] + answers["tags"]["usecase"] = answers["usecase"] + del answers["usecase"] + answers["tags"]["cve"] = ["UPDATE WITH CVE(S) IF APPLICABLE"] return answers def execute(self, input_dto: new) -> None: if input_dto.type == NewContentType.detection: content_dict, detection_kind = self.buildDetection() - subdirectory = pathlib.Path('detections') / detection_kind + subdirectory = pathlib.Path("detections") / detection_kind elif input_dto.type == NewContentType.story: content_dict = self.buildStory() - subdirectory = pathlib.Path('stories') + subdirectory = pathlib.Path("stories") else: raise Exception(f"Unsupported new content type: [{input_dto.type}]") - full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name')) + full_output_path = ( + input_dto.path + / subdirectory + / SecurityContentObject_Abstract.contentNameToFileName( + content_dict.get("name") + ) + ) YmlWriter.writeYmlFile(str(full_output_path), content_dict) - diff --git a/contentctl/actions/release_notes.py b/contentctl/actions/release_notes.py index fb42effc..d9272129 100644 --- a/contentctl/actions/release_notes.py +++ b/contentctl/actions/release_notes.py @@ -6,234 +6,365 @@ from typing import List, Union - class ReleaseNotes: - def create_notes(self,repo_path:pathlib.Path, file_paths:List[pathlib.Path], header:str)->dict[str,Union[List[str], str]]: - updates:List[str] = [] - warnings:List[str] = [] + def create_notes( + self, repo_path: pathlib.Path, file_paths: List[pathlib.Path], header: str + ) -> dict[str, Union[List[str], str]]: + updates: List[str] = [] + warnings: List[str] = [] for file_path in file_paths: # Check if the file exists if file_path.exists() and file_path.is_file(): # Check if the file is a YAML file - if file_path.suffix in ['.yaml', '.yml']: + if file_path.suffix in [".yaml", ".yml"]: # Read and parse the YAML file - with open(file_path, 'r') as file: + with open(file_path, "r") as file: try: data = yaml.safe_load(file) # Check and create story link - if 'name' in data and 'stories' in file_path.parts: - story_link = "https://research.splunk.com/stories/" + data['name'] - story_link=story_link.replace(" ","_") + if "name" in data and "stories" in file_path.parts: + story_link = ( + "https://research.splunk.com/stories/" + + data["name"] + ) + story_link = story_link.replace(" ", "_") story_link = story_link.lower() - updates.append("- "+"["+f"{data['name']}"+"]"+"("+story_link+")") - - if 'name' in data and'playbooks' in file_path.parts: - playbook_link = "https://research.splunk.com/" + str(file_path).replace(str(repo_path),"") - playbook_link=playbook_link.replace(".yml","/").lower() - updates.append("- "+"["+f"{data['name']}"+"]"+"("+playbook_link+")") - - if 'name' in data and'macros' in file_path.parts: + updates.append( + "- " + + "[" + + f"{data['name']}" + + "]" + + "(" + + story_link + + ")" + ) + + if "name" in data and "playbooks" in file_path.parts: + playbook_link = "https://research.splunk.com/" + str( + file_path + ).replace(str(repo_path), "") + playbook_link = playbook_link.replace( + ".yml", "/" + ).lower() + updates.append( + "- " + + "[" + + f"{data['name']}" + + "]" + + "(" + + playbook_link + + ")" + ) + + if "name" in data and "macros" in file_path.parts: updates.append("- " + f"{data['name']}") - if 'name' in data and'lookups' in file_path.parts: + if "name" in data and "lookups" in file_path.parts: updates.append("- " + f"{data['name']}") # Create only SSA link when its production - if 'name' in data and 'id' in data and 'ssa_detections' in file_path.parts: - if data['status'] == "production": - temp_link = "https://research.splunk.com/" + str(file_path).replace(str(repo_path),"") - pattern = r'(?<=/)[^/]*$' - detection_link = re.sub(pattern, data['id'], temp_link) - detection_link = detection_link.replace("detections","" ) - detection_link = detection_link.replace("ssa_/","" ) - updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")") - - if data['status'] == "validation": - updates.append("- "+f"{data['name']}"+" (Validation Mode)") - + if ( + "name" in data + and "id" in data + and "ssa_detections" in file_path.parts + ): + if data["status"] == "production": + temp_link = "https://research.splunk.com/" + str( + file_path + ).replace(str(repo_path), "") + pattern = r"(?<=/)[^/]*$" + detection_link = re.sub( + pattern, data["id"], temp_link + ) + detection_link = detection_link.replace( + "detections", "" + ) + detection_link = detection_link.replace("ssa_/", "") + updates.append( + "- " + + "[" + + f"{data['name']}" + + "]" + + "(" + + detection_link + + ")" + ) + + if data["status"] == "validation": + updates.append( + "- " + f"{data['name']}" + " (Validation Mode)" + ) # Check and create detection link - if 'name' in data and 'id' in data and 'detections' in file_path.parts and 'ssa_detections' not in file_path.parts and 'detections/deprecated' not in file_path.parts: - - if data['status'] == "production": - temp_link = "https://research.splunk.com" + str(file_path).replace(str(repo_path),"") - pattern = r'(?<=/)[^/]*$' - detection_link = re.sub(pattern, data['id'], temp_link) - detection_link = detection_link.replace("detections","" ) - detection_link = detection_link.replace(".com//",".com/" ) - updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")") - - if data['status'] == "deprecated": - temp_link = "https://research.splunk.com" + str(file_path).replace(str(repo_path),"") - pattern = r'(?<=/)[^/]*$' - detection_link = re.sub(pattern, data['id'], temp_link) - detection_link = detection_link.replace("detections","" ) - detection_link = detection_link.replace(".com//",".com/" ) - updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")") - + if ( + "name" in data + and "id" in data + and "detections" in file_path.parts + and "ssa_detections" not in file_path.parts + and "detections/deprecated" not in file_path.parts + ): + if data["status"] == "production": + temp_link = "https://research.splunk.com" + str( + file_path + ).replace(str(repo_path), "") + pattern = r"(?<=/)[^/]*$" + detection_link = re.sub( + pattern, data["id"], temp_link + ) + detection_link = detection_link.replace( + "detections", "" + ) + detection_link = detection_link.replace( + ".com//", ".com/" + ) + updates.append( + "- " + + "[" + + f"{data['name']}" + + "]" + + "(" + + detection_link + + ")" + ) + + if data["status"] == "deprecated": + temp_link = "https://research.splunk.com" + str( + file_path + ).replace(str(repo_path), "") + pattern = r"(?<=/)[^/]*$" + detection_link = re.sub( + pattern, data["id"], temp_link + ) + detection_link = detection_link.replace( + "detections", "" + ) + detection_link = detection_link.replace( + ".com//", ".com/" + ) + updates.append( + "- " + + "[" + + f"{data['name']}" + + "]" + + "(" + + detection_link + + ")" + ) + except yaml.YAMLError as exc: - raise Exception(f"Error parsing YAML file for release_notes {file_path}: {str(exc)}") + raise Exception( + f"Error parsing YAML file for release_notes {file_path}: {str(exc)}" + ) else: - warnings.append(f"Error parsing YAML file for release_notes. File not found or is not a file: {file_path}") - #print out all updates at once - success_header = f'### {header} - [{len(updates)}]' - warning_header = f'### {header} - [{len(warnings)}]' - return {'header': success_header, 'changes': sorted(updates), - 'warning_header': warning_header, 'warnings': warnings} - - - def release_notes(self, config:release_notes) -> None: + warnings.append( + f"Error parsing YAML file for release_notes. File not found or is not a file: {file_path}" + ) + # print out all updates at once + success_header = f"### {header} - [{len(updates)}]" + warning_header = f"### {header} - [{len(warnings)}]" + return { + "header": success_header, + "changes": sorted(updates), + "warning_header": warning_header, + "warnings": warnings, + } + def release_notes(self, config: release_notes) -> None: ### Remove hard coded path - directories = ['detections/','stories/','macros/','lookups/','playbooks/','ssa_detections/'] - + directories = [ + "detections/", + "stories/", + "macros/", + "lookups/", + "playbooks/", + "ssa_detections/", + ] + repo = Repo(config.path) # Ensure the new tag is in the tags if tags are supplied - - if config.new_tag: + + if config.new_tag: if config.new_tag not in repo.tags: - raise Exception(f"new_tag {config.new_tag} does not exist in the repository. Make sure your branch nameis ") + raise Exception( + f"new_tag {config.new_tag} does not exist in the repository. Make sure your branch nameis " + ) if config.old_tag is None: - #Old tag was not supplied, so find the index of the new tag, then get the tag before it - tags_sorted = sorted(repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True) - tags_names_sorted = [tag.name for tag in tags_sorted] + # Old tag was not supplied, so find the index of the new tag, then get the tag before it + tags_sorted = sorted( + repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True + ) + tags_names_sorted = [tag.name for tag in tags_sorted] new_tag_index = tags_names_sorted.index(config.new_tag) try: - config.old_tag = tags_names_sorted[new_tag_index+1] + config.old_tag = tags_names_sorted[new_tag_index + 1] except Exception: - raise Exception(f"old_tag cannot be inferred. {config.new_tag} is the oldest tag in the repo!") + raise Exception( + f"old_tag cannot be inferred. {config.new_tag} is the oldest tag in the repo!" + ) latest_tag = config.new_tag - previous_tag = config.old_tag + previous_tag = config.old_tag commit1 = repo.commit(latest_tag) - commit2 = repo.commit(previous_tag) + commit2 = repo.commit(previous_tag) diff_index = commit2.diff(commit1) - # Ensure the branch is in the repo + # Ensure the branch is in the repo if config.latest_branch: - #If a branch name is supplied, compare against develop + # If a branch name is supplied, compare against develop if config.latest_branch not in repo.branches: - raise ValueError(f"latest branch {config.latest_branch} does not exist in the repository. Make sure your branch name is correct") + raise ValueError( + f"latest branch {config.latest_branch} does not exist in the repository. Make sure your branch name is correct" + ) if config.compare_against not in repo.branches: - raise ValueError(f"compare_against branch {config.compare_against} does not exist in the repository. Make sure your branch name is correct") - + raise ValueError( + f"compare_against branch {config.compare_against} does not exist in the repository. Make sure your branch name is correct" + ) + commit1 = repo.commit(config.latest_branch) - commit2 = repo.commit(config.compare_against) + commit2 = repo.commit(config.compare_against) diff_index = commit2.diff(commit1) - - modified_files:List[pathlib.Path] = [] - added_files:List[pathlib.Path] = [] + + modified_files: List[pathlib.Path] = [] + added_files: List[pathlib.Path] = [] for diff in diff_index: file_path = pathlib.Path(diff.a_path) # Check if the file is in the specified directories if any(str(file_path).startswith(directory) for directory in directories): # Check if a file is Modified - if diff.change_type == 'M': + if diff.change_type == "M": modified_files.append(file_path) - # Check if a file is Added - elif diff.change_type == 'A': + elif diff.change_type == "A": added_files.append(file_path) # print(added_files) - detections_added:List[pathlib.Path] = [] - ba_detections_added:List[pathlib.Path] = [] - stories_added:List[pathlib.Path] = [] - macros_added:List[pathlib.Path] = [] - lookups_added:List[pathlib.Path] = [] - playbooks_added:List[pathlib.Path] = [] - detections_modified:List[pathlib.Path] = [] - ba_detections_modified:List[pathlib.Path] = [] - stories_modified:List[pathlib.Path] = [] - macros_modified:List[pathlib.Path] = [] - lookups_modified:List[pathlib.Path] = [] - playbooks_modified:List[pathlib.Path] = [] - detections_deprecated:List[pathlib.Path] = [] + detections_added: List[pathlib.Path] = [] + ba_detections_added: List[pathlib.Path] = [] + stories_added: List[pathlib.Path] = [] + macros_added: List[pathlib.Path] = [] + lookups_added: List[pathlib.Path] = [] + playbooks_added: List[pathlib.Path] = [] + detections_modified: List[pathlib.Path] = [] + ba_detections_modified: List[pathlib.Path] = [] + stories_modified: List[pathlib.Path] = [] + macros_modified: List[pathlib.Path] = [] + lookups_modified: List[pathlib.Path] = [] + playbooks_modified: List[pathlib.Path] = [] + detections_deprecated: List[pathlib.Path] = [] for file in modified_files: - file= config.path / file - if 'detections' in file.parts and 'ssa_detections' not in file.parts and 'deprecated' not in file.parts: + file = config.path / file + if ( + "detections" in file.parts + and "ssa_detections" not in file.parts + and "deprecated" not in file.parts + ): detections_modified.append(file) - if 'detections' in file.parts and 'ssa_detections' not in file.parts and 'deprecated' in file.parts: + if ( + "detections" in file.parts + and "ssa_detections" not in file.parts + and "deprecated" in file.parts + ): detections_deprecated.append(file) - if 'stories' in file.parts: + if "stories" in file.parts: stories_modified.append(file) - if 'macros' in file.parts: + if "macros" in file.parts: macros_modified.append(file) - if 'lookups' in file.parts: + if "lookups" in file.parts: lookups_modified.append(file) - if 'playbooks' in file.parts: + if "playbooks" in file.parts: playbooks_modified.append(file) - if 'ssa_detections' in file.parts: + if "ssa_detections" in file.parts: ba_detections_modified.append(file) for file in added_files: - file=config.path / file - if 'detections' in file.parts and 'ssa_detections' not in file.parts: + file = config.path / file + if "detections" in file.parts and "ssa_detections" not in file.parts: detections_added.append(file) - if 'stories' in file.parts: + if "stories" in file.parts: stories_added.append(file) - if 'macros' in file.parts: + if "macros" in file.parts: macros_added.append(file) - if 'lookups' in file.parts: + if "lookups" in file.parts: lookups_added.append(file) - if 'playbooks' in file.parts: + if "playbooks" in file.parts: playbooks_added.append(file) - if 'ssa_detections' in file.parts: + if "ssa_detections" in file.parts: ba_detections_added.append(file) if config.new_tag: - print(f"Generating release notes - \033[92m{latest_tag}\033[0m") print(f"Compared against - \033[92m{previous_tag}\033[0m") print("\n## Release notes for ESCU " + latest_tag) if config.latest_branch: - print(f"Generating release notes - \033[92m{config.latest_branch}\033[0m") - print(f"Compared against - \033[92m{config.compare_against}\033[0m") + print( + f"Generating release notes - \033[92m{config.latest_branch}\033[0m" + ) + print( + f"Compared against - \033[92m{config.compare_against}\033[0m" + ) print("\n## Release notes for ESCU " + config.latest_branch) - notes = [self.create_notes(config.path, stories_added, header="New Analytic Story"), - self.create_notes(config.path,stories_modified, header="Updated Analytic Story"), - self.create_notes(config.path,detections_added, header="New Analytics"), - self.create_notes(config.path,detections_modified, header="Updated Analytics"), - self.create_notes(config.path,macros_added, header="Macros Added"), - self.create_notes(config.path,macros_modified, header="Macros Updated"), - self.create_notes(config.path,lookups_added, header="Lookups Added"), - self.create_notes(config.path,lookups_modified, header="Lookups Updated"), - self.create_notes(config.path,playbooks_added, header="Playbooks Added"), - self.create_notes(config.path,playbooks_modified, header="Playbooks Updated"), - self.create_notes(config.path,detections_deprecated, header="Deprecated Analytics")] - - #generate and show ba_notes in a different section - ba_notes = [self.create_notes(config.path,ba_detections_added, header="New BA Analytics"), - self.create_notes(config.path,ba_detections_modified, header="Updated BA Analytics") ] - - - def printNotes(notes:List[dict[str,Union[List[str], str]]], outfile:Union[pathlib.Path,None]=None): - num_changes = sum([len(note['changes']) for note in notes]) - num_warnings = sum([len(note['warnings']) for note in notes]) - lines:List[str] = [] + notes = [ + self.create_notes(config.path, stories_added, header="New Analytic Story"), + self.create_notes( + config.path, stories_modified, header="Updated Analytic Story" + ), + self.create_notes(config.path, detections_added, header="New Analytics"), + self.create_notes( + config.path, detections_modified, header="Updated Analytics" + ), + self.create_notes(config.path, macros_added, header="Macros Added"), + self.create_notes(config.path, macros_modified, header="Macros Updated"), + self.create_notes(config.path, lookups_added, header="Lookups Added"), + self.create_notes(config.path, lookups_modified, header="Lookups Updated"), + self.create_notes(config.path, playbooks_added, header="Playbooks Added"), + self.create_notes( + config.path, playbooks_modified, header="Playbooks Updated" + ), + self.create_notes( + config.path, detections_deprecated, header="Deprecated Analytics" + ), + ] + + # generate and show ba_notes in a different section + ba_notes = [ + self.create_notes( + config.path, ba_detections_added, header="New BA Analytics" + ), + self.create_notes( + config.path, ba_detections_modified, header="Updated BA Analytics" + ), + ] + + def printNotes( + notes: List[dict[str, Union[List[str], str]]], + outfile: Union[pathlib.Path, None] = None, + ): + num_changes = sum([len(note["changes"]) for note in notes]) + num_warnings = sum([len(note["warnings"]) for note in notes]) + lines: List[str] = [] lines.append(f"Total New and Updated Content: [{num_changes}]") for note in notes: lines.append("") - lines.append(note['header']) - lines+=(note['changes']) - + lines.append(note["header"]) + lines += note["changes"] + lines.append(f"\n\nTotal Warnings: [{num_warnings}]") for note in notes: - if len(note['warnings']) > 0: - lines.append(note['warning_header']) - lines+=note['warnings'] - text_blob = '\n'.join(lines) + if len(note["warnings"]) > 0: + lines.append(note["warning_header"]) + lines += note["warnings"] + text_blob = "\n".join(lines) print(text_blob) if outfile is not None: - with open(outfile,'w') as writer: + with open(outfile, "w") as writer: writer.write(text_blob) - + printNotes(notes, config.releaseNotesFilename("release_notes.txt")) print("\n\n### Other Updates\n-\n") print("\n## BA Release Notes") printNotes(ba_notes, config.releaseNotesFilename("ba_release_notes.txt")) - print("Release notes completed succesfully") \ No newline at end of file + print("Release notes completed succesfully") diff --git a/contentctl/actions/reporting.py b/contentctl/actions/reporting.py index db3a278d..70e1cd33 100644 --- a/contentctl/actions/reporting.py +++ b/contentctl/actions/reporting.py @@ -1,4 +1,3 @@ - from dataclasses import dataclass from contentctl.input.director import DirectorOutputDto @@ -6,38 +5,44 @@ from contentctl.output.attack_nav_output import AttackNavOutput from contentctl.objects.config import report + @dataclass(frozen=True) class ReportingInputDto: director_output_dto: DirectorOutputDto config: report -class Reporting: +class Reporting: def execute(self, input_dto: ReportingInputDto) -> None: - - - #Ensure the reporting path exists + # Ensure the reporting path exists try: - input_dto.config.getReportingPath().mkdir(exist_ok=True,parents=True) + input_dto.config.getReportingPath().mkdir(exist_ok=True, parents=True) except Exception as e: if input_dto.config.getReportingPath().is_file(): - raise Exception(f"Error writing reporting: '{input_dto.config.getReportingPath()}' is a file, not a directory.") + raise Exception( + f"Error writing reporting: '{input_dto.config.getReportingPath()}' is a file, not a directory." + ) else: - raise Exception(f"Error writing reporting : '{input_dto.config.getReportingPath()}': {str(e)}") + raise Exception( + f"Error writing reporting : '{input_dto.config.getReportingPath()}': {str(e)}" + ) print("Creating GitHub Badges...") - #Generate GitHub Badges + # Generate GitHub Badges svg_output = SvgOutput() svg_output.writeObjects( - input_dto.director_output_dto.detections, - input_dto.config.getReportingPath()) - - #Generate coverage json + input_dto.director_output_dto.detections, + input_dto.config.getReportingPath(), + ) + + # Generate coverage json print("Generating coverage.json...") - attack_nav_output = AttackNavOutput() + attack_nav_output = AttackNavOutput() attack_nav_output.writeObjects( - input_dto.director_output_dto.detections, - input_dto.config.getReportingPath() + input_dto.director_output_dto.detections, + input_dto.config.getReportingPath(), + ) + + print( + f"Reporting successfully written to '{input_dto.config.getReportingPath()}'" ) - - print(f"Reporting successfully written to '{input_dto.config.getReportingPath()}'") \ No newline at end of file diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index e45688f7..96b070fe 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -39,7 +39,7 @@ class TestInputDto: detections: List[Detection] config: test_common - + class Test: def filter_tests(self, input_dto: TestInputDto) -> None: @@ -50,7 +50,7 @@ def filter_tests(self, input_dto: TestInputDto) -> None: Args: input_dto (TestInputDto): A configuration of the test and all of the tests to be run. - """ + """ if not input_dto.config.enable_integration_testing: # Skip all integraiton tests if integration testing is not enabled: @@ -59,7 +59,6 @@ def filter_tests(self, input_dto: TestInputDto) -> None: if isinstance(test, IntegrationTest): test.skip("TEST SKIPPED: Skipping all integration tests") - def execute(self, input_dto: TestInputDto) -> bool: output_dto = DetectionTestingManagerOutputDto() @@ -86,10 +85,21 @@ def execute(self, input_dto: TestInputDto) -> bool: # detections were tested. file.stop() else: - print(f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections") - if isinstance(input_dto.config.mode, Selected) or isinstance(input_dto.config.mode, Changes): - files_string = '\n- '.join( - [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections] + print( + f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections" + ) + if isinstance(input_dto.config.mode, Selected) or isinstance( + input_dto.config.mode, Changes + ): + files_string = "\n- ".join( + [ + str( + pathlib.Path(detection.file_path).relative_to( + input_dto.config.path + ) + ) + for detection in input_dto.detections + ] ) print(f"Detections:\n- {files_string}") @@ -100,43 +110,41 @@ def execute(self, input_dto: TestInputDto) -> bool: summary_results = file.getSummaryObject() summary = summary_results.get("summary", {}) - print(f"Test Summary (mode: {summary.get('mode','Error')})") - print(f"\tSuccess : {summary.get('success',False)}") - print( - f"\tSuccess Rate : {summary.get('success_rate','ERROR')}" - ) + print(f"Test Summary (mode: {summary.get('mode', 'Error')})") + print(f"\tSuccess : {summary.get('success', False)}") print( - f"\tTotal Detections : {summary.get('total_detections','ERROR')}" + f"\tSuccess Rate : {summary.get('success_rate', 'ERROR')}" ) print( - f"\tTotal Tested Detections : {summary.get('total_tested_detections','ERROR')}" + f"\tTotal Detections : {summary.get('total_detections', 'ERROR')}" ) print( - f"\t Passed Detections : {summary.get('total_pass','ERROR')}" + f"\tTotal Tested Detections : {summary.get('total_tested_detections', 'ERROR')}" ) print( - f"\t Failed Detections : {summary.get('total_fail','ERROR')}" + f"\t Passed Detections : {summary.get('total_pass', 'ERROR')}" ) print( - f"\tSkipped Detections : {summary.get('total_skipped','ERROR')}" + f"\t Failed Detections : {summary.get('total_fail', 'ERROR')}" ) print( - "\tProduction Status :" + f"\tSkipped Detections : {summary.get('total_skipped', 'ERROR')}" ) + print("\tProduction Status :") print( - f"\t Production Detections : {summary.get('total_production','ERROR')}" + f"\t Production Detections : {summary.get('total_production', 'ERROR')}" ) print( - f"\t Experimental Detections : {summary.get('total_experimental','ERROR')}" + f"\t Experimental Detections : {summary.get('total_experimental', 'ERROR')}" ) print( - f"\t Deprecated Detections : {summary.get('total_deprecated','ERROR')}" + f"\t Deprecated Detections : {summary.get('total_deprecated', 'ERROR')}" ) print( - f"\tManually Tested Detections : {summary.get('total_manual','ERROR')}" + f"\tManually Tested Detections : {summary.get('total_manual', 'ERROR')}" ) print( - f"\tUntested Detections : {summary.get('total_untested','ERROR')}" + f"\tUntested Detections : {summary.get('total_untested', 'ERROR')}" ) print(f"\tTest Results File : {file.getOutputFilePath()}") print( diff --git a/contentctl/actions/validate.py b/contentctl/actions/validate.py index c2756f75..2a16aa37 100644 --- a/contentctl/actions/validate.py +++ b/contentctl/actions/validate.py @@ -1,4 +1,3 @@ - import pathlib from contentctl.input.director import Director, DirectorOutputDto @@ -27,7 +26,7 @@ def execute(self, input_dto: validate) -> DirectorOutputDto: [], [], [], - [] + [], ) director = Director(director_output_dto) @@ -35,51 +34,69 @@ def execute(self, input_dto: validate) -> DirectorOutputDto: self.ensure_no_orphaned_files_in_lookups(input_dto.path, director_output_dto) if input_dto.data_source_TA_validation: self.validate_latest_TA_information(director_output_dto.data_sources) - + return director_output_dto - - def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_output_dto:DirectorOutputDto): + def ensure_no_orphaned_files_in_lookups( + self, repo_path: pathlib.Path, director_output_dto: DirectorOutputDto + ): """ This function ensures that only files which are relevant to lookups are included in the lookups folder. This means that a file must be either: 1. A lookup YML (.yml) 2. A lookup CSV (.csv) which is referenced by a YML 3. A lookup MLMODEL (.mlmodel) which is referenced by a YML. - + All other files, includes CSV and MLMODEL files which are NOT referenced by a YML, will generate an exception from this function. - + Args: repo_path (pathlib.Path): path to the root of the app director_output_dto (DirectorOutputDto): director object with all constructed content Raises: - Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel - files in this directory. Additionally, an exception will be raised if there - exists one or more .csv or .mlmodel files that are not referenced by at least 1 - detection .yml file in this directory. + Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel + files in this directory. Additionally, an exception will be raised if there + exists one or more .csv or .mlmodel files that are not referenced by at least 1 + detection .yml file in this directory. This avoids having additional, unused files in this directory that may be copied into the app when it is built (which can cause appinspect errors or larger app size.) - """ - lookupsDirectory = repo_path/"lookups" - + """ + lookupsDirectory = repo_path / "lookups" + # Get all of the files referneced by Lookups - usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if isinstance(lookup, FileBackedLookup)] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None] + usedLookupFiles: list[pathlib.Path] = [ + lookup.filename + for lookup in director_output_dto.lookups + if isinstance(lookup, FileBackedLookup) + ] + [ + lookup.file_path + for lookup in director_output_dto.lookups + if lookup.file_path is not None + ] # Get all of the mlmodel and csv files in the lookups directory - csvAndMlmodelFiles = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"]) - + csvAndMlmodelFiles = Utils.get_security_content_files_from_directory( + lookupsDirectory, + allowedFileExtensions=[".yml", ".csv", ".mlmodel"], + fileExtensionsToReturn=[".csv", ".mlmodel"], + ) + # Generate an exception of any csv or mlmodel files exist but are not used - unusedLookupFiles:list[pathlib.Path] = [testFile for testFile in csvAndMlmodelFiles if testFile not in usedLookupFiles] + unusedLookupFiles: list[pathlib.Path] = [ + testFile + for testFile in csvAndMlmodelFiles + if testFile not in usedLookupFiles + ] if len(unusedLookupFiles) > 0: - raise Exception(f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}") + raise Exception( + f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}" + ) return - def validate_latest_TA_information(self, data_sources: list[DataSource]) -> None: validated_TAs: list[tuple[str, str]] = [] - errors:list[str] = [] + errors: list[str] = [] print("----------------------") print("Validating latest TA:") print("----------------------") @@ -90,22 +107,25 @@ def validate_latest_TA_information(self, data_sources: list[DataSource]) -> None continue if supported_TA.url is not None: validated_TAs.append(ta_identifier) - uid = int(str(supported_TA.url).rstrip('/').split("/")[-1]) + uid = int(str(supported_TA.url).rstrip("/").split("/")[-1]) try: splunk_app = SplunkApp(app_uid=uid) if splunk_app.latest_version != supported_TA.version: - errors.append(f"Version mismatch in '{data_source.file_path}' supported TA '{supported_TA.name}'" - f"\n Latest version on Splunkbase : {splunk_app.latest_version}" - f"\n Version specified in data source: {supported_TA.version}") + errors.append( + f"Version mismatch in '{data_source.file_path}' supported TA '{supported_TA.name}'" + f"\n Latest version on Splunkbase : {splunk_app.latest_version}" + f"\n Version specified in data source: {supported_TA.version}" + ) except Exception as e: - errors.append(f"Error processing checking version of TA {supported_TA.name}: {str(e)}") - + errors.append( + f"Error processing checking version of TA {supported_TA.name}: {str(e)}" + ) + if len(errors) > 0: - errorString = '\n\n'.join(errors) - raise Exception(f"[{len(errors)}] or more TA versions are out of date or have other errors." - f"Please update the following data sources with the latest versions of " - f"their supported tas:\n\n{errorString}") + errorString = "\n\n".join(errors) + raise Exception( + f"[{len(errors)}] or more TA versions are out of date or have other errors." + f"Please update the following data sources with the latest versions of " + f"their supported tas:\n\n{errorString}" + ) print("All TA versions are up to date.") - - - diff --git a/contentctl/api.py b/contentctl/api.py index 8c996549..037ac5ce 100644 --- a/contentctl/api.py +++ b/contentctl/api.py @@ -5,42 +5,48 @@ from contentctl.objects.security_content_object import SecurityContentObject from contentctl.input.director import DirectorOutputDto -def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={}, - configType:Type[Union[test,test_servers]]=test)->test_common: - + +def config_from_file( + path: Path = Path("contentctl.yml"), + config: dict[str, Any] = {}, + configType: Type[Union[test, test_servers]] = test, +) -> test_common: """ Fetch a configuration object that can be used for a number of different contentctl - operations including validate, build, inspect, test, and test_servers. A file will + operations including validate, build, inspect, test, and test_servers. A file will be used as the basis for constructing the configuration. Args: - path (Path, optional): Relative or absolute path to a contentctl config file. + path (Path, optional): Relative or absolute path to a contentctl config file. Defaults to Path("contentctl.yml"), which is the default name and location (in the current directory) of the configuration files which are automatically generated for contentctl. config (dict[], optional): Dictionary of values to override values read from the YML path passed as the first argument. Defaults to {}, an empty dict meaning that nothing - will be overwritten - configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. + will be overwritten + configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test. Returns: test_common: Returns a complete contentctl test_common configuration. Note that this configuration will have all applicable field for validate and build as well, but can also be used for easily - construction a test or test_servers object. - """ + construction a test or test_servers object. + """ try: yml_dict = YmlReader.load_file(path, add_fields=False) - - + except Exception as e: - raise Exception(f"Failed to load contentctl configuration from file '{path}': {str(e)}") - + raise Exception( + f"Failed to load contentctl configuration from file '{path}': {str(e)}" + ) + # Apply settings that have been overridden from the ones in the file try: yml_dict.update(config) except Exception as e: - raise Exception(f"Failed updating dictionary of values read from file '{path}'" - f" with the dictionary of arguments passed: {str(e)}") + raise Exception( + f"Failed updating dictionary of values read from file '{path}'" + f" with the dictionary of arguments passed: {str(e)}" + ) # The function below will throw its own descriptive exception if it fails configObject = config_from_dict(yml_dict, configType=configType) @@ -48,13 +54,12 @@ def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={}, return configObject - - -def config_from_dict(config: dict[str,Any]={}, - configType:Type[Union[test,test_servers]]=test)->test_common: +def config_from_dict( + config: dict[str, Any] = {}, configType: Type[Union[test, test_servers]] = test +) -> test_common: """ Fetch a configuration object that can be used for a number of different contentctl - operations including validate, build, inspect, test, and test_servers. A dict will + operations including validate, build, inspect, test, and test_servers. A dict will be used as the basis for constructing the configuration. Args: @@ -63,29 +68,30 @@ def config_from_dict(config: dict[str,Any]={}, values. Note that based on default values in the contentctl/objects/config.py file, this may raise an exception. If so, please set appropriate default values in the file above or supply those values via this argument. - configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. + configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test. Returns: test_common: Returns a complete contentctl test_common configuration. Note that this configuration will have all applicable field for validate and build as well, but can also be used for easily - construction a test or test_servers object. - """ + construction a test or test_servers object. + """ try: test_object = configType.model_validate(config) except Exception as e: raise Exception(f"Failed to load contentctl configuration from dict:\n{str(e)}") - + return test_object -def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,Any])->test_common: - +def update_config( + config: Union[test, test_servers], **key_value_updates: dict[str, Any] +) -> test_common: """Update any relevant keys in a config file with the specified values. Full validation will be performed after this update and descriptive errors will be produced Args: - config (test_common): A previously-constructed test_common object. This can be + config (test_common): A previously-constructed test_common object. This can be build using the configFromDict or configFromFile functions. key_value_updates (kwargs, optional): Additional keyword/argument pairs to update arbitrary fields in the configuration. @@ -101,37 +107,40 @@ def update_config(config:Union[test,test_servers], **key_value_updates:dict[str, # Force validation of assignment since doing so via arbitrary dict can be error prone # Also, ensure that we do not try to add fields that are not part of the model - config_copy.model_config.update({'validate_assignment': True, 'extra': 'forbid'}) + config_copy.model_config.update({"validate_assignment": True, "extra": "forbid"}) - - # Collect any errors that may occur - errors:list[Exception] = [] - - # We need to do this one by one because the extra:forbid argument does not appear to + errors: list[Exception] = [] + + # We need to do this one by one because the extra:forbid argument does not appear to # be respected at this time. for key, value in key_value_updates.items(): try: - setattr(config_copy,key,value) + setattr(config_copy, key, value) except Exception as e: errors.append(e) if len(errors) > 0: - errors_string = '\n'.join([str(e) for e in errors]) + errors_string = "\n".join([str(e) for e in errors]) raise Exception(f"Error(s) updaitng configuration:\n{errors_string}") - + return config_copy - -def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]: - output_dict:dict[str,list[dict[str,Any]]] = {} - for contentType in ['detections','stories','baselines','investigations', - 'playbooks','macros','lookups','deployments',]: - +def content_to_dict(director: DirectorOutputDto) -> dict[str, list[dict[str, Any]]]: + output_dict: dict[str, list[dict[str, Any]]] = {} + for contentType in [ + "detections", + "stories", + "baselines", + "investigations", + "playbooks", + "macros", + "lookups", + "deployments", + ]: output_dict[contentType] = [] - t:list[SecurityContentObject] = getattr(director,contentType) - + t: list[SecurityContentObject] = getattr(director, contentType) + for item in t: output_dict[contentType].append(item.model_dump()) return output_dict - diff --git a/contentctl/enrichments/attack_enrichment.py b/contentctl/enrichments/attack_enrichment.py index 47ed6d78..b437860d 100644 --- a/contentctl/enrichments/attack_enrichment.py +++ b/contentctl/enrichments/attack_enrichment.py @@ -1,4 +1,3 @@ - from __future__ import annotations from attackcti import attack_client import logging @@ -6,115 +5,157 @@ from dataclasses import field from typing import Any from pathlib import Path -from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment, MitreTactics +from contentctl.objects.mitre_attack_enrichment import ( + MitreAttackEnrichment, + MitreTactics, +) from contentctl.objects.config import validate from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE -logging.getLogger('taxii2client').setLevel(logging.CRITICAL) + +logging.getLogger("taxii2client").setLevel(logging.CRITICAL) class AttackEnrichment(BaseModel): data: dict[str, MitreAttackEnrichment] = field(default_factory=dict) - use_enrichment:bool = True - + use_enrichment: bool = True + @staticmethod - def getAttackEnrichment(config:validate)->AttackEnrichment: + def getAttackEnrichment(config: validate) -> AttackEnrichment: enrichment = AttackEnrichment(use_enrichment=config.enrichments) _ = enrichment.get_attack_lookup(config.mitre_cti_repo_path, config.enrichments) return enrichment - - def getEnrichmentByMitreID(self, mitre_id:MITRE_ATTACK_ID_TYPE)->MitreAttackEnrichment: + + def getEnrichmentByMitreID( + self, mitre_id: MITRE_ATTACK_ID_TYPE + ) -> MitreAttackEnrichment: if not self.use_enrichment: - raise Exception("Error, trying to add Mitre Enrichment, but use_enrichment was set to False") - + raise Exception( + "Error, trying to add Mitre Enrichment, but use_enrichment was set to False" + ) + enrichment = self.data.get(mitre_id, None) if enrichment is not None: return enrichment else: - raise Exception(f"Error, Unable to find Mitre Enrichment for MitreID {mitre_id}") - - def addMitreIDViaGroupNames(self, technique:dict[str,Any], tactics:list[str], groupNames:list[str])->None: - technique_id = technique['technique_id'] - technique_obj = technique['technique'] + raise Exception( + f"Error, Unable to find Mitre Enrichment for MitreID {mitre_id}" + ) + + def addMitreIDViaGroupNames( + self, technique: dict[str, Any], tactics: list[str], groupNames: list[str] + ) -> None: + technique_id = technique["technique_id"] + technique_obj = technique["technique"] tactics.sort() - + if technique_id in self.data: raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'") - self.data[technique_id] = MitreAttackEnrichment.model_validate({'mitre_attack_id':technique_id, - 'mitre_attack_technique':technique_obj, - 'mitre_attack_tactics':tactics, - 'mitre_attack_groups':groupNames, - 'mitre_attack_group_objects':[]}) - - def addMitreIDViaGroupObjects(self, technique:dict[str,Any], tactics:list[MitreTactics], groupDicts:list[dict[str,Any]])->None: - technique_id = technique['technique_id'] - technique_obj = technique['technique'] + self.data[technique_id] = MitreAttackEnrichment.model_validate( + { + "mitre_attack_id": technique_id, + "mitre_attack_technique": technique_obj, + "mitre_attack_tactics": tactics, + "mitre_attack_groups": groupNames, + "mitre_attack_group_objects": [], + } + ) + + def addMitreIDViaGroupObjects( + self, + technique: dict[str, Any], + tactics: list[MitreTactics], + groupDicts: list[dict[str, Any]], + ) -> None: + technique_id = technique["technique_id"] + technique_obj = technique["technique"] tactics.sort() - - groupNames:list[str] = sorted([group['group'] for group in groupDicts]) - + + groupNames: list[str] = sorted([group["group"] for group in groupDicts]) + if technique_id in self.data: raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'") - - self.data[technique_id] = MitreAttackEnrichment.model_validate({'mitre_attack_id': technique_id, - 'mitre_attack_technique': technique_obj, - 'mitre_attack_tactics': tactics, - 'mitre_attack_groups': groupNames, - 'mitre_attack_group_objects': groupDicts}) - - - def get_attack_lookup(self, input_path: Path, enrichments:bool = False) -> dict[str,MitreAttackEnrichment]: - attack_lookup:dict[str,MitreAttackEnrichment] = {} + + self.data[technique_id] = MitreAttackEnrichment.model_validate( + { + "mitre_attack_id": technique_id, + "mitre_attack_technique": technique_obj, + "mitre_attack_tactics": tactics, + "mitre_attack_groups": groupNames, + "mitre_attack_group_objects": groupDicts, + } + ) + + def get_attack_lookup( + self, input_path: Path, enrichments: bool = False + ) -> dict[str, MitreAttackEnrichment]: + attack_lookup: dict[str, MitreAttackEnrichment] = {} if not enrichments: return attack_lookup - + try: - print(f"Performing MITRE Enrichment using the repository at {input_path}...",end="", flush=True) - # The existence of the input_path is validated during cli argument validation, but it is + print( + f"Performing MITRE Enrichment using the repository at {input_path}...", + end="", + flush=True, + ) + # The existence of the input_path is validated during cli argument validation, but it is # possible that the repo is in the wrong format. If the following directories do not - # exist, then attack_client will fall back to resolving via REST API. We do not - # want this as it is slow and error prone, so we will force an exception to + # exist, then attack_client will fall back to resolving via REST API. We do not + # want this as it is slow and error prone, so we will force an exception to # be generated. - enterprise_path = input_path/"enterprise-attack" - mobile_path = input_path/"ics-attack" - ics_path = input_path/"mobile-attack" - if not (enterprise_path.is_dir() and mobile_path.is_dir() and ics_path.is_dir()): - raise FileNotFoundError("One or more of the following paths does not exist: " - f"{[str(enterprise_path),str(mobile_path),str(ics_path)]}. " - f"Please ensure that the {input_path} directory " - "has been git cloned correctly.") + enterprise_path = input_path / "enterprise-attack" + mobile_path = input_path / "ics-attack" + ics_path = input_path / "mobile-attack" + if not ( + enterprise_path.is_dir() and mobile_path.is_dir() and ics_path.is_dir() + ): + raise FileNotFoundError( + "One or more of the following paths does not exist: " + f"{[str(enterprise_path), str(mobile_path), str(ics_path)]}. " + f"Please ensure that the {input_path} directory " + "has been git cloned correctly." + ) lift = attack_client( - local_paths= { - "enterprise":str(enterprise_path), - "mobile":str(mobile_path), - "ics":str(ics_path) + local_paths={ + "enterprise": str(enterprise_path), + "mobile": str(mobile_path), + "ics": str(ics_path), } ) - - all_enterprise_techniques = lift.get_enterprise_techniques(stix_format=False) - enterprise_relationships = lift.get_enterprise_relationships(stix_format=False) + + all_enterprise_techniques = lift.get_enterprise_techniques( + stix_format=False + ) + enterprise_relationships = lift.get_enterprise_relationships( + stix_format=False + ) enterprise_groups = lift.get_enterprise_groups(stix_format=False) - + for technique in all_enterprise_techniques: - apt_groups:list[dict[str,Any]] = [] + apt_groups: list[dict[str, Any]] = [] for relationship in enterprise_relationships: - if (relationship['target_object'] == technique['id']) and relationship['source_object'].startswith('intrusion-set'): + if ( + relationship["target_object"] == technique["id"] + ) and relationship["source_object"].startswith("intrusion-set"): for group in enterprise_groups: - if relationship['source_object'] == group['id']: + if relationship["source_object"] == group["id"]: apt_groups.append(group) - #apt_groups.append(group['group']) + # apt_groups.append(group['group']) tactics = [] - if ('tactic' in technique): - for tactic in technique['tactic']: - tactics.append(tactic.replace('-',' ').title()) + if "tactic" in technique: + for tactic in technique["tactic"]: + tactics.append(tactic.replace("-", " ").title()) self.addMitreIDViaGroupObjects(technique, tactics, apt_groups) - attack_lookup[technique['technique_id']] = {'technique': technique['technique'], 'tactics': tactics, 'groups': apt_groups} - + attack_lookup[technique["technique_id"]] = { + "technique": technique["technique"], + "tactics": tactics, + "groups": apt_groups, + } - except Exception as err: raise Exception(f"Error getting MITRE Enrichment: {str(err)}") - + print("Done!") - return attack_lookup \ No newline at end of file + return attack_lookup diff --git a/contentctl/enrichments/cve_enrichment.py b/contentctl/enrichments/cve_enrichment.py index d9d71546..75ffbd5a 100644 --- a/contentctl/enrichments/cve_enrichment.py +++ b/contentctl/enrichments/cve_enrichment.py @@ -1,59 +1,70 @@ from __future__ import annotations from pycvesearch import CVESearch from typing import Annotated, Union, TYPE_CHECKING -from pydantic import ConfigDict, BaseModel,Field, computed_field +from pydantic import ConfigDict, BaseModel, Field, computed_field from decimal import Decimal from contentctl.objects.annotated_types import CVE_TYPE + if TYPE_CHECKING: from contentctl.objects.config import validate - -CVESSEARCH_API_URL = 'https://cve.circl.lu' +CVESSEARCH_API_URL = "https://cve.circl.lu" class CveEnrichmentObj(BaseModel): id: CVE_TYPE - cvss: Annotated[Decimal, Field(ge=.1, le=10, decimal_places=1)] + cvss: Annotated[Decimal, Field(ge=0.1, le=10, decimal_places=1)] summary: str - + @computed_field @property - def url(self)->str: + def url(self) -> str: BASE_NVD_URL = "https://nvd.nist.gov/vuln/detail/" return f"{BASE_NVD_URL}{self.id}" class CveEnrichment(BaseModel): use_enrichment: bool = True - cve_api_obj: Union[CVESearch,None] = None + cve_api_obj: Union[CVESearch, None] = None # Arbitrary_types are allowed to let us use the CVESearch Object - model_config = ConfigDict( - arbitrary_types_allowed=True, - frozen=True - ) + model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True) @staticmethod - def getCveEnrichment(config:validate, timeout_seconds:int=10, force_disable_enrichment:bool=True)->CveEnrichment: + def getCveEnrichment( + config: validate, + timeout_seconds: int = 10, + force_disable_enrichment: bool = True, + ) -> CveEnrichment: if force_disable_enrichment: - return CveEnrichment(use_enrichment=False, cve_api_obj=None) - + return CveEnrichment(use_enrichment=False, cve_api_obj=None) + if config.enrichments: try: cve_api_obj = CVESearch(CVESSEARCH_API_URL, timeout=timeout_seconds) return CveEnrichment(use_enrichment=True, cve_api_obj=cve_api_obj) except Exception as e: - raise Exception(f"Error setting CVE_SEARCH API to: {CVESSEARCH_API_URL}: {str(e)}") - - return CveEnrichment(use_enrichment=False, cve_api_obj=None) - + raise Exception( + f"Error setting CVE_SEARCH API to: {CVESSEARCH_API_URL}: {str(e)}" + ) - def enrich_cve(self, cve_id:str, raise_exception_on_failure:bool=True)->CveEnrichmentObj: + return CveEnrichment(use_enrichment=False, cve_api_obj=None) + def enrich_cve( + self, cve_id: str, raise_exception_on_failure: bool = True + ) -> CveEnrichmentObj: if not self.use_enrichment: - return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME") + return CveEnrichmentObj( + id=cve_id, + cvss=Decimal(5.0), + summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME", + ) else: print("WARNING - Dynamic enrichment not supported at this time.") - return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME") - # Depending on needs, we may add dynamic enrichment functionality back to the tool \ No newline at end of file + return CveEnrichmentObj( + id=cve_id, + cvss=Decimal(5.0), + summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME", + ) + # Depending on needs, we may add dynamic enrichment functionality back to the tool diff --git a/contentctl/enrichments/splunk_app_enrichment.py b/contentctl/enrichments/splunk_app_enrichment.py index 161ec91c..00d7c1be 100644 --- a/contentctl/enrichments/splunk_app_enrichment.py +++ b/contentctl/enrichments/splunk_app_enrichment.py @@ -10,15 +10,16 @@ NON_PERSISTENT_CACHE = {} + @functools.cache -def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes: +def requests_get_helper(url: str, force_cached_or_offline: bool = False) -> bytes: if force_cached_or_offline: if not os.path.exists(APP_ENRICHMENT_CACHE_FILENAME): print(f"Cache at {APP_ENRICHMENT_CACHE_FILENAME} not found - Creating it.") - cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag='c', writeback=True) + cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag="c", writeback=True) else: cache = NON_PERSISTENT_CACHE - + if url in cache: req_content = cache[url] else: @@ -27,61 +28,65 @@ def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes: req_content = req.content cache[url] = req_content except Exception: - raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}")) - + raise ( + Exception( + f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}" + ) + ) + if isinstance(cache, shelve.Shelf): - #close the cache if it is a shelf + # close the cache if it is a shelf cache.close() - + return req_content -class SplunkAppEnrichment(): +class SplunkAppEnrichment: @classmethod - def enrich_splunk_app(self, splunk_ta: str, force_cached_or_offline: bool = False) -> dict: - + def enrich_splunk_app( + self, splunk_ta: str, force_cached_or_offline: bool = False + ) -> dict: appurl = SPLUNKBASE_API_URL + splunk_ta splunk_app_enriched = dict() - + try: - content = requests_get_helper(appurl, force_cached_or_offline) response_dict = xmltodict.parse(content) - + # check if list since data changes depending on answer url, results = self._parse_splunkbase_response(response_dict) # grab the app name for i in results: - if i['@name'] == 'appName': - splunk_app_enriched['name'] = i['#text'] - # grab out the splunkbase url - if 'entriesbyid' in url: + if i["@name"] == "appName": + splunk_app_enriched["name"] = i["#text"] + # grab out the splunkbase url + if "entriesbyid" in url: content = requests_get_helper(url, force_cached_or_offline) response_dict = xmltodict.parse(content) - - #print(json.dumps(response_dict, indent=2)) + + # print(json.dumps(response_dict, indent=2)) url, results = self._parse_splunkbase_response(response_dict) # chop the url so we grab the splunkbase portion but not direct download - splunk_app_enriched['url'] = url.rsplit('/', 4)[0] + splunk_app_enriched["url"] = url.rsplit("/", 4)[0] except requests.exceptions.ConnectionError as connErr: print(f"There was a connErr for ta {splunk_ta}: {connErr}") # there was a connection error lets just capture the name - splunk_app_enriched['name'] = splunk_ta - splunk_app_enriched['url'] = '' + splunk_app_enriched["name"] = splunk_ta + splunk_app_enriched["url"] = "" except Exception as e: - print(f"There was an unknown error enriching the Splunk TA [{splunk_ta}]: {str(e)}") - splunk_app_enriched['name'] = splunk_ta - splunk_app_enriched['url'] = '' - + print( + f"There was an unknown error enriching the Splunk TA [{splunk_ta}]: {str(e)}" + ) + splunk_app_enriched["name"] = splunk_ta + splunk_app_enriched["url"] = "" return splunk_app_enriched def _parse_splunkbase_response(response_dict): - if isinstance(response_dict['feed']['entry'], list): - url = response_dict['feed']['entry'][0]['link']['@href'] - results = response_dict['feed']['entry'][0]['content']['s:dict']['s:key'] + if isinstance(response_dict["feed"]["entry"], list): + url = response_dict["feed"]["entry"][0]["link"]["@href"] + results = response_dict["feed"]["entry"][0]["content"]["s:dict"]["s:key"] else: - url = response_dict['feed']['entry']['link']['@href'] - results = response_dict['feed']['entry']['content']['s:dict']['s:key'] + url = response_dict["feed"]["entry"]["link"]["@href"] + results = response_dict["feed"]["entry"]["content"]["s:dict"]["s:key"] return url, results - diff --git a/contentctl/helper/link_validator.py b/contentctl/helper/link_validator.py index 4419c32c..181642ec 100644 --- a/contentctl/helper/link_validator.py +++ b/contentctl/helper/link_validator.py @@ -11,88 +11,96 @@ DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36" ALLOWED_HTTP_CODES = [200] -class LinkStats(BaseModel): - #Static Values + +class LinkStats(BaseModel): + # Static Values method: Callable = requests.get - allowed_http_codes: list[int] = ALLOWED_HTTP_CODES - access_count: int = 1 #when constructor is called, it has been accessed once! + allowed_http_codes: list[int] = ALLOWED_HTTP_CODES + access_count: int = 1 # when constructor is called, it has been accessed once! timeout_seconds: int = 15 allow_redirects: bool = True headers: dict = {"User-Agent": DEFAULT_USER_AGENT_STRING} verify_ssl: bool = False if verify_ssl is False: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - #Values generated at runtime. - #We need to assign these some default values to get the - #validation working since ComputedField has not yet been - #introduced to Pydantic + + # Values generated at runtime. + # We need to assign these some default values to get the + # validation working since ComputedField has not yet been + # introduced to Pydantic reference: str referencing_files: set[str] - redirect: Union[str,None] = None + redirect: Union[str, None] = None status_code: int = 0 valid: bool = False - resolution_time: float = 0 - - - def is_link_valid(self, referencing_file:str)->bool: + resolution_time: float = 0 + + def is_link_valid(self, referencing_file: str) -> bool: self.access_count += 1 self.referencing_files.add(referencing_file) return self.valid - + @model_validator(mode="before") - def check_reference(cls, data:Any)->Any: + def check_reference(cls, data: Any) -> Any: start_time = time.time() - #Get out all the fields names to make them easier to reference - method = data['method'] - reference = data['reference'] - timeout_seconds = data['timeout_seconds'] - headers = data['headers'] - allow_redirects = data['allow_redirects'] - verify_ssl = data['verify_ssl'] - allowed_http_codes = data['allowed_http_codes'] + # Get out all the fields names to make them easier to reference + method = data["method"] + reference = data["reference"] + timeout_seconds = data["timeout_seconds"] + headers = data["headers"] + allow_redirects = data["allow_redirects"] + verify_ssl = data["verify_ssl"] + allowed_http_codes = data["allowed_http_codes"] if not (reference.startswith("http://") or reference.startswith("https://")): - raise(ValueError(f"Reference {reference} does not begin with http(s). Only http(s) references are supported")) - + raise ( + ValueError( + f"Reference {reference} does not begin with http(s). Only http(s) references are supported" + ) + ) + try: - get = method(reference, timeout=timeout_seconds, - headers = headers, - allow_redirects=allow_redirects, verify=verify_ssl) + get = method( + reference, + timeout=timeout_seconds, + headers=headers, + allow_redirects=allow_redirects, + verify=verify_ssl, + ) resolution_time = time.time() - start_time - data['status_code'] = get.status_code - data['resolution_time'] = resolution_time + data["status_code"] = get.status_code + data["resolution_time"] = resolution_time if reference != get.url: - data['redirect'] = get.url + data["redirect"] = get.url else: - data['redirect'] = None #None is also already the default + data["redirect"] = None # None is also already the default - #Returns the updated values and sets them for the object + # Returns the updated values and sets them for the object if get.status_code in allowed_http_codes: - data['valid'] = True + data["valid"] = True else: - #print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}") - data['valid'] = False - return data + # print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}") + data["valid"] = False + return data except Exception: resolution_time = time.time() - start_time - #print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds") - data['status_code'] = 0 - data['valid'] = False - data['redirect'] = None - data['resolution_time'] = resolution_time + # print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds") + data["status_code"] = 0 + data["valid"] = False + data["redirect"] = None + data["resolution_time"] = resolution_time return data class LinkValidator(abc.ABC): - cache: Union[dict[str,LinkStats], shelve.Shelf] = {} + cache: Union[dict[str, LinkStats], shelve.Shelf] = {} uncached_checks: int = 0 total_checks: int = 0 - #cache: dict[str,LinkStats] = {} + # cache: dict[str,LinkStats] = {} use_file_cache: bool = False - reference_cache_file: str ="lookups/REFERENCE_CACHE.db" + reference_cache_file: str = "lookups/REFERENCE_CACHE.db" @staticmethod def initialize_cache(use_file_cache: bool = False): @@ -100,74 +108,88 @@ def initialize_cache(use_file_cache: bool = False): if use_file_cache is False: return if not os.path.exists(LinkValidator.reference_cache_file): - print(f"Cache at {LinkValidator.reference_cache_file} not found - Creating it.") - + print( + f"Cache at {LinkValidator.reference_cache_file} not found - Creating it." + ) + try: - LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True) + LinkValidator.cache = shelve.open( + LinkValidator.reference_cache_file, flag="c", writeback=True + ) except Exception: - print(f"Failed to create the cache file {LinkValidator.reference_cache_file}. Reference info will not be cached.") + print( + f"Failed to create the cache file {LinkValidator.reference_cache_file}. Reference info will not be cached." + ) LinkValidator.cache = {} - #Remove all of the failures to force those resources to be resolved again + # Remove all of the failures to force those resources to be resolved again failed_refs = [] for ref in LinkValidator.cache.keys(): if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES: failed_refs.append(ref) - #can't remove it here because this will throw an error: - #cannot change size of dictionary while iterating over it + # can't remove it here because this will throw an error: + # cannot change size of dictionary while iterating over it else: - #Set the reference count to 0 and referencing files to empty set + # Set the reference count to 0 and referencing files to empty set LinkValidator.cache[ref].access_count = 0 LinkValidator.cache[ref].referencing_files = set() - - for ref in failed_refs: - del(LinkValidator.cache[ref]) + for ref in failed_refs: + del LinkValidator.cache[ref] - - @staticmethod def close_cache(): if LinkValidator.use_file_cache: LinkValidator.cache.close() @staticmethod - def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool: + def validate_reference( + reference: str, referencing_file: str, raise_exception_if_failure: bool = False + ) -> bool: LinkValidator.total_checks += 1 if reference not in LinkValidator.cache: LinkValidator.uncached_checks += 1 - LinkValidator.cache[reference] = LinkStats(reference=reference, referencing_files = set([referencing_file])) + LinkValidator.cache[reference] = LinkStats( + reference=reference, referencing_files=set([referencing_file]) + ) result = LinkValidator.cache[reference].is_link_valid(referencing_file) - #print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}") + # print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}") if result is True: return True elif raise_exception_if_failure is True: - raise(Exception(f"Reference Link Failed: {reference}")) + raise (Exception(f"Reference Link Failed: {reference}")) else: return False + @staticmethod def print_link_validation_errors(): - failures = [LinkValidator.cache[k] for k in LinkValidator.cache if LinkValidator.cache[k].valid is False] + failures = [ + LinkValidator.cache[k] + for k in LinkValidator.cache + if LinkValidator.cache[k].valid is False + ] failures.sort(key=lambda d: d.status_code) for failure in failures: - print(f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:") + print( + f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:" + ) for ref in failure.referencing_files: print(f"\t* {ref}") @staticmethod - def SecurityContentObject_validate_references(v:list, values: dict)->list: - if 'check_references' not in values: - raise(Exception("Member 'check_references' missing from Baseline!")) - elif values['check_references'] is False: - #Reference checking is enabled + def SecurityContentObject_validate_references(v: list, values: dict) -> list: + if "check_references" not in values: + raise (Exception("Member 'check_references' missing from Baseline!")) + elif values["check_references"] is False: + # Reference checking is enabled pass - elif values['check_references'] is True: + elif values["check_references"] is True: for reference in v: - LinkValidator.validate_reference(reference, values['name']) - #Remove the check_references key from the values dict so that it is not - #output by the serialization code - del values['check_references'] + LinkValidator.validate_reference(reference, values["name"]) + # Remove the check_references key from the values dict so that it is not + # output by the serialization code + del values["check_references"] return v diff --git a/contentctl/helper/splunk_app.py b/contentctl/helper/splunk_app.py index 0f95f593..34920e54 100644 --- a/contentctl/helper/splunk_app.py +++ b/contentctl/helper/splunk_app.py @@ -39,6 +39,7 @@ class RetryConstant: class SplunkBaseError(requests.HTTPError): """An error raise in communicating with Splunkbase""" + pass @@ -50,6 +51,7 @@ class SplunkApp: class InitializationError(Exception): """An initialization error during SplunkApp setup""" + pass @staticmethod @@ -68,16 +70,16 @@ def requests_retry_session( status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) return session def __init__( - self, - app_uid: Optional[int] = None, - app_name_id: Optional[str] = None, - manual_setup: bool = False, - ) -> None: + self, + app_uid: Optional[int] = None, + app_name_id: Optional[str] = None, + manual_setup: bool = False, + ) -> None: if app_uid is None and app_name_id is None: raise SplunkApp.InitializationError( "Either app_uid (the numeric app UID e.g. 742) or app_name_id (the app name " @@ -123,18 +125,22 @@ def get_app_info_by_uid(self) -> dict: if self._app_info_cache is not None: return self._app_info_cache elif self.app_uid is None: - raise SplunkApp.InitializationError("app_uid must be set in order to fetch app info") + raise SplunkApp.InitializationError( + "app_uid must be set in order to fetch app info" + ) # NOTE: auth not required # Get app info by uid try: response = self.requests_retry_session().get( APIEndPoint.SPLUNK_BASE_APP_INFO.format(app_uid=self.app_uid), - timeout=RetryConstant.RETRY_INTERVAL + timeout=RetryConstant.RETRY_INTERVAL, ) response.raise_for_status() except requests.exceptions.RequestException as e: - raise SplunkBaseError(f"Error fetching app info for app_uid {self.app_uid}: {str(e)}") + raise SplunkBaseError( + f"Error fetching app info for app_uid {self.app_uid}: {str(e)}" + ) # parse JSON and set cache self._app_info_cache: dict = json.loads(response.content) @@ -156,7 +162,9 @@ def set_app_name_id(self) -> None: if "appid" in app_info: self.app_name_id = app_info["appid"] else: - raise SplunkBaseError(f"Invalid response from Splunkbase; missing key 'appid': {app_info}") + raise SplunkBaseError( + f"Invalid response from Splunkbase; missing key 'appid': {app_info}" + ) def set_app_uid(self) -> None: """ @@ -166,19 +174,25 @@ def set_app_uid(self) -> None: if self.app_uid is not None: return elif self.app_name_id is None: - raise SplunkApp.InitializationError("app_name_id must be set in order to fetch app_uid") + raise SplunkApp.InitializationError( + "app_name_id must be set in order to fetch app_uid" + ) # NOTE: auth not required # Get app_uid by app_name_id via a redirect try: response = self.requests_retry_session().get( - APIEndPoint.SPLUNK_BASE_GET_UID_REDIRECT.format(app_name_id=self.app_name_id), + APIEndPoint.SPLUNK_BASE_GET_UID_REDIRECT.format( + app_name_id=self.app_name_id + ), allow_redirects=False, - timeout=RetryConstant.RETRY_INTERVAL + timeout=RetryConstant.RETRY_INTERVAL, ) response.raise_for_status() except requests.exceptions.RequestException as e: - raise SplunkBaseError(f"Error fetching app_uid for app_name_id '{self.app_name_id}': {str(e)}") + raise SplunkBaseError( + f"Error fetching app_uid for app_name_id '{self.app_name_id}': {str(e)}" + ) # Extract the app_uid from the redirect path if "Location" in response.headers: @@ -199,7 +213,9 @@ def set_app_title(self) -> None: if "title" in app_info: self.app_title = app_info["title"] else: - raise SplunkBaseError(f"Invalid response from Splunkbase; missing key 'title': {app_info}") + raise SplunkBaseError( + f"Invalid response from Splunkbase; missing key 'title': {app_info}" + ) def __fetch_url_latest_version_info(self) -> str: """ @@ -209,12 +225,16 @@ def __fetch_url_latest_version_info(self) -> str: # retrieve app entries using the app_name_id try: response = self.requests_retry_session().get( - APIEndPoint.SPLUNK_BASE_FETCH_APP_BY_ENTRY_ID.format(app_name_id=self.app_name_id), - timeout=RetryConstant.RETRY_INTERVAL + APIEndPoint.SPLUNK_BASE_FETCH_APP_BY_ENTRY_ID.format( + app_name_id=self.app_name_id + ), + timeout=RetryConstant.RETRY_INTERVAL, ) response.raise_for_status() except requests.exceptions.RequestException as e: - raise SplunkBaseError(f"Error fetching app entries for app_name_id '{self.app_name_id}': {str(e)}") + raise SplunkBaseError( + f"Error fetching app entries for app_name_id '{self.app_name_id}': {str(e)}" + ) # parse xml app_xml = xmltodict.parse(response.content) @@ -231,7 +251,9 @@ def __fetch_url_latest_version_info(self) -> str: return entry.get("link").get("@href") # raise if no entry was found - raise SplunkBaseError(f"No app entry found with 'islatest' tag set to True: {self.app_name_id}") + raise SplunkBaseError( + f"No app entry found with 'islatest' tag set to True: {self.app_name_id}" + ) def __fetch_url_latest_version_download(self, info_url: str) -> str: """ @@ -241,10 +263,14 @@ def __fetch_url_latest_version_download(self, info_url: str) -> str: """ # fetch download info try: - response = self.requests_retry_session().get(info_url, timeout=RetryConstant.RETRY_INTERVAL) + response = self.requests_retry_session().get( + info_url, timeout=RetryConstant.RETRY_INTERVAL + ) response.raise_for_status() except requests.exceptions.RequestException as e: - raise SplunkBaseError(f"Error fetching download info for app_name_id '{self.app_name_id}': {str(e)}") + raise SplunkBaseError( + f"Error fetching download info for app_name_id '{self.app_name_id}': {str(e)}" + ) # parse XML and extract download URL build_xml = xmltodict.parse(response.content) @@ -254,14 +280,18 @@ def __fetch_url_latest_version_download(self, info_url: str) -> str: def set_latest_version_info(self) -> None: # raise if app_name_id not set if self.app_name_id is None: - raise SplunkApp.InitializationError("app_name_id must be set in order to fetch latest version info") + raise SplunkApp.InitializationError( + "app_name_id must be set in order to fetch latest version info" + ) # fetch the info URL info_url = self.__fetch_url_latest_version_info() # parse out the version number and fetch the download URL self.latest_version = info_url.split("/")[-1] - self.latest_version_download_url = self.__fetch_url_latest_version_download(info_url) + self.latest_version_download_url = self.__fetch_url_latest_version_download( + info_url + ) def __get_splunk_base_session_token(self, username: str, password: str) -> str: """ @@ -309,12 +339,12 @@ def __get_splunk_base_session_token(self, username: str, password: str) -> str: return token_value def download( - self, - out: Path, - username: str, - password: str, - is_dir: bool = False, - overwrite: bool = False + self, + out: Path, + username: str, + password: str, + is_dir: bool = False, + overwrite: bool = False, ) -> Path: """ Given an output path, download the app to the specified location @@ -336,11 +366,7 @@ def download( # Get the Splunkbase session token token = self.__get_splunk_base_session_token(username, password) response = requests.request( - "GET", - self.latest_version_download_url, - cookies={ - "sessionid": token - } + "GET", self.latest_version_download_url, cookies={"sessionid": token} ) # If the provided output path was a directory we need to try and pull the filename from the @@ -348,17 +374,21 @@ def download( if is_dir: try: # Pull 'Content-Disposition' from the headers - content_disposition: str = response.headers['Content-Disposition'] + content_disposition: str = response.headers["Content-Disposition"] # Attempt to parse the filename as a KV key, value = content_disposition.strip().split("=") if key != "attachment;filename": - raise ValueError(f"Unexpected key in 'Content-Disposition' KV pair: {key}") + raise ValueError( + f"Unexpected key in 'Content-Disposition' KV pair: {key}" + ) # Validate the filename is the expected .tgz file filename = Path(value.strip().strip('"')) if filename.suffixes != [".tgz"]: - raise ValueError(f"Filename has unexpected extension(s): {filename.suffixes}") + raise ValueError( + f"Filename has unexpected extension(s): {filename.suffixes}" + ) out = Path(out, filename) except KeyError as e: raise KeyError( @@ -371,9 +401,7 @@ def download( # Ensure the output path is not already occupied if out.exists() and not overwrite: - msg = ( - f"File already exists at {out}, cannot download the app." - ) + msg = f"File already exists at {out}, cannot download the app." raise Exception(msg) # Make any parent directories as needed diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index f003e2fc..ba458b8b 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -12,6 +12,7 @@ from math import ceil from typing import TYPE_CHECKING + if TYPE_CHECKING: from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.security_content_object import SecurityContentObject @@ -24,26 +25,29 @@ class Utils: @staticmethod def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]: - listOfFiles:list[pathlib.Path] = [] + listOfFiles: list[pathlib.Path] = [] base_path = pathlib.Path(path) if not base_path.exists(): return listOfFiles - for (dirpath, dirnames, filenames) in os.walk(path): + for dirpath, dirnames, filenames in os.walk(path): for file in filenames: if file.endswith(".yml"): listOfFiles.append(pathlib.Path(os.path.join(dirpath, file))) - + return sorted(listOfFiles) - + @staticmethod - def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExtensions:list[str]=[".yml"], fileExtensionsToReturn:list[str]=[".yml"]) -> list[pathlib.Path]: - + def get_security_content_files_from_directory( + path: pathlib.Path, + allowedFileExtensions: list[str] = [".yml"], + fileExtensionsToReturn: list[str] = [".yml"], + ) -> list[pathlib.Path]: """ Get all of the Security Content Object Files rooted in a given directory. These will almost certain be YML files, but could be other file types as specified by the user Args: - path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. + path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. allowedFileExtensions (set[str], optional): File extensions which are allowed to be present in this directory. In most cases, we do not want to allow the presence of non-YML files. Defaults to [".yml"]. fileExtensionsToReturn (set[str], optional): Filenames with extensions that should be returned from this function. For example, the lookups/ directory contains YML, CSV, and MLMODEL directories, but only the YMLs are Security Content Objects for constructing Lookyps. Defaults to[".yml"]. @@ -56,14 +60,18 @@ def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExt list[pathlib.Path]: list of files with an extension in fileExtensionsToReturn found in path """ if not set(fileExtensionsToReturn).issubset(set(allowedFileExtensions)): - raise Exception(f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not") - + raise Exception( + f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not" + ) + if not path.exists() or not path.is_dir(): - raise Exception(f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory") - - allowedFiles:list[pathlib.Path] = [] - erroneousFiles:list[pathlib.Path] = [] - #Get every single file extension + raise Exception( + f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory" + ) + + allowedFiles: list[pathlib.Path] = [] + erroneousFiles: list[pathlib.Path] = [] + # Get every single file extension for filePath in path.glob("**/*.*"): if filePath.suffix in allowedFileExtensions: # Yes these are allowed @@ -73,58 +81,75 @@ def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExt erroneousFiles.append(filePath) if len(erroneousFiles): - raise Exception(f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}") - + raise Exception( + f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}" + ) + # There were no errorneous files, so return the requested files - return sorted([filePath for filePath in allowedFiles if filePath.suffix in fileExtensionsToReturn]) + return sorted( + [ + filePath + for filePath in allowedFiles + if filePath.suffix in fileExtensionsToReturn + ] + ) @staticmethod - def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]: + def get_all_yml_files_from_directory_one_layer_deep( + path: str, + ) -> list[pathlib.Path]: listOfFiles: list[pathlib.Path] = [] base_path = pathlib.Path(path) if not base_path.exists(): return listOfFiles # Check the base directory for item in base_path.iterdir(): - if item.is_file() and item.suffix == '.yml': + if item.is_file() and item.suffix == ".yml": listOfFiles.append(item) # Check one subfolder level deep for subfolder in base_path.iterdir(): if subfolder.is_dir() and subfolder.name != "cim": for item in subfolder.iterdir(): - if item.is_file() and item.suffix == '.yml': + if item.is_file() and item.suffix == ".yml": listOfFiles.append(item) return sorted(listOfFiles) - @staticmethod - def add_id(id_dict:dict[str, list[pathlib.Path]], obj:SecurityContentObject, path:pathlib.Path) -> None: + def add_id( + id_dict: dict[str, list[pathlib.Path]], + obj: SecurityContentObject, + path: pathlib.Path, + ) -> None: if hasattr(obj, "id"): obj_id = obj.id if obj_id in id_dict: id_dict[obj_id].append(path) else: id_dict[obj_id] = [path] + # Otherwise, no ID so nothing to add.... @staticmethod - def check_ids_for_duplicates(id_dict:dict[str, list[pathlib.Path]])->list[Tuple[pathlib.Path, ValueError]]: - validation_errors:list[Tuple[pathlib.Path, ValueError]] = [] - + def check_ids_for_duplicates( + id_dict: dict[str, list[pathlib.Path]], + ) -> list[Tuple[pathlib.Path, ValueError]]: + validation_errors: list[Tuple[pathlib.Path, ValueError]] = [] + for key, values in id_dict.items(): if len(values) > 1: error_file_path = pathlib.Path("MULTIPLE") - all_files = '\n\t'.join(str(pathlib.Path(p)) for p in values) - exception = ValueError(f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}") + all_files = "\n\t".join(str(pathlib.Path(p)) for p in values) + exception = ValueError( + f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}" + ) validation_errors.append((error_file_path, exception)) - + return validation_errors @staticmethod def validate_git_hash( repo_path: str, repo_url: str, commit_hash: str, branch_name: Union[str, None] ) -> bool: - # Get a list of all branches repo = git.Repo(repo_path) if commit_hash is None: @@ -251,7 +276,6 @@ def validate_git_pull_request(repo_path: str, pr_number: int) -> str: def verify_file_exists( file_path: str, verbose_print=False, timeout_seconds: int = 10 ) -> None: - try: if pathlib.Path(file_path).is_file(): # This is a file and we know it exists @@ -261,18 +285,13 @@ def verify_file_exists( # Try to make a head request to verify existence of the file try: - req = requests.head( file_path, timeout=timeout_seconds, verify=True, allow_redirects=True ) if req.status_code > 400: raise (Exception(f"Return code={req.status_code}")) except Exception as e: - raise ( - Exception( - f"HTTP Resolution Failed: {str(e)}" - ) - ) + raise (Exception(f"HTTP Resolution Failed: {str(e)}")) @staticmethod def copy_local_file( diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 8462d61e..1b637101 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -29,7 +29,7 @@ @dataclass class DirectorOutputDto: - # Atomic Tests are first because parsing them + # Atomic Tests are first because parsing them # is far quicker than attack_enrichment atomic_enrichment: AtomicEnrichment attack_enrichment: AttackEnrichment @@ -50,20 +50,20 @@ class DirectorOutputDto: def addContentToDictMappings(self, content: SecurityContentObject): content_name = content.name - - + if content_name in self.name_to_content_map: raise ValueError( f"Duplicate name '{content_name}' with paths:\n" f" - {content.file_path}\n" f" - {self.name_to_content_map[content_name].file_path}" ) - + if content.id in self.uuid_to_content_map: raise ValueError( f"Duplicate id '{content.id}' with paths:\n" f" - {content.file_path}\n" - f" - {self.uuid_to_content_map[content.id].file_path}") + f" - {self.uuid_to_content_map[content.id].file_path}" + ) if isinstance(content, Lookup): self.lookups.append(content) @@ -82,7 +82,7 @@ def addContentToDictMappings(self, content: SecurityContentObject): elif isinstance(content, Detection): self.detections.append(content) elif isinstance(content, Dashboard): - self.dashboards.append(content) + self.dashboards.append(content) elif isinstance(content, DataSource): self.data_sources.append(content) @@ -93,7 +93,7 @@ def addContentToDictMappings(self, content: SecurityContentObject): self.uuid_to_content_map[content.id] = content -class Director(): +class Director: input_dto: validate output_dto: DirectorOutputDto @@ -112,13 +112,18 @@ def execute(self, input_dto: validate) -> None: self.createSecurityContent(SecurityContentType.playbooks) self.createSecurityContent(SecurityContentType.detections) self.createSecurityContent(SecurityContentType.dashboards) - - from contentctl.objects.abstract_security_content_objects.detection_abstract import MISSING_SOURCES + + from contentctl.objects.abstract_security_content_objects.detection_abstract import ( + MISSING_SOURCES, + ) + if len(MISSING_SOURCES) > 0: missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES))) - print("WARNING: The following data_sources have been used in detections, but are not yet defined.\n" - "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 " - f"{missing_sources_string}") + print( + "WARNING: The following data_sources have been used in detections, but are not yet defined.\n" + "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 " + f"{missing_sources_string}" + ) else: print("No missing data_sources!") @@ -133,18 +138,20 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None: SecurityContentType.playbooks, SecurityContentType.detections, SecurityContentType.data_sources, - SecurityContentType.dashboards + SecurityContentType.dashboards, ]: files = Utils.get_all_yml_files_from_directory( os.path.join(self.input_dto.path, str(contentType.name)) ) - security_content_files = [ - f for f in files - ] + security_content_files = [f for f in files] else: - raise (Exception(f"Cannot createSecurityContent for unknown product {contentType}.")) + raise ( + Exception( + f"Cannot createSecurityContent for unknown product {contentType}." + ) + ) - validation_errors:list[tuple[Path,ValueError]] = [] + validation_errors: list[tuple[Path, ValueError]] = [] already_ran = False progress_percent = 0 @@ -156,41 +163,67 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None: modelDict = YmlReader.load_file(file) if contentType == SecurityContentType.lookups: - lookup = LookupAdapter.validate_python(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) - #lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) + lookup = LookupAdapter.validate_python( + modelDict, + context={ + "output_dto": self.output_dto, + "config": self.input_dto, + }, + ) + # lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto}) self.output_dto.addContentToDictMappings(lookup) - + elif contentType == SecurityContentType.macros: - macro = Macro.model_validate(modelDict, context={"output_dto":self.output_dto}) + macro = Macro.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) self.output_dto.addContentToDictMappings(macro) - + elif contentType == SecurityContentType.deployments: - deployment = Deployment.model_validate(modelDict, context={"output_dto":self.output_dto}) + deployment = Deployment.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) self.output_dto.addContentToDictMappings(deployment) elif contentType == SecurityContentType.playbooks: - playbook = Playbook.model_validate(modelDict, context={"output_dto":self.output_dto}) - self.output_dto.addContentToDictMappings(playbook) - + playbook = Playbook.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) + self.output_dto.addContentToDictMappings(playbook) + elif contentType == SecurityContentType.baselines: - baseline = Baseline.model_validate(modelDict, context={"output_dto":self.output_dto}) + baseline = Baseline.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) self.output_dto.addContentToDictMappings(baseline) - + elif contentType == SecurityContentType.investigations: - investigation = Investigation.model_validate(modelDict, context={"output_dto":self.output_dto}) + investigation = Investigation.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) self.output_dto.addContentToDictMappings(investigation) elif contentType == SecurityContentType.stories: - story = Story.model_validate(modelDict, context={"output_dto":self.output_dto}) + story = Story.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) self.output_dto.addContentToDictMappings(story) - + elif contentType == SecurityContentType.detections: - detection = Detection.model_validate(modelDict, context={"output_dto":self.output_dto, "app":self.input_dto.app}) + detection = Detection.model_validate( + modelDict, + context={ + "output_dto": self.output_dto, + "app": self.input_dto.app, + }, + ) self.output_dto.addContentToDictMappings(detection) - + elif contentType == SecurityContentType.dashboards: - dashboard = Dashboard.model_validate(modelDict,context={"output_dto":self.output_dto}) - self.output_dto.addContentToDictMappings(dashboard) + dashboard = Dashboard.model_validate( + modelDict, context={"output_dto": self.output_dto} + ) + self.output_dto.addContentToDictMappings(dashboard) elif contentType == SecurityContentType.data_sources: data_source = DataSource.model_validate( @@ -237,4 +270,3 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None: raise Exception( f"The following {len(validation_errors)} error(s) were found during validation:\n\n{errors_string}\n\nVALIDATION FAILED" ) - diff --git a/contentctl/input/new_content_questions.py b/contentctl/input/new_content_questions.py index c3ee5343..a7ce0e56 100644 --- a/contentctl/input/new_content_questions.py +++ b/contentctl/input/new_content_questions.py @@ -3,9 +3,8 @@ class NewContentQuestions: - @classmethod - def get_questions_detection(cls) -> list[dict[str,Any]]: + def get_questions_detection(cls) -> list[dict[str, Any]]: questions = [ { "type": "text", @@ -14,22 +13,16 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: "default": "Powershell Encoded Command", }, { - 'type': 'select', - 'message': 'what kind of detection is this', - 'name': 'detection_kind', - 'choices': [ - 'endpoint', - 'cloud', - 'application', - 'network', - 'web' - ], - 'default': 'endpoint' + "type": "select", + "message": "what kind of detection is this", + "name": "detection_kind", + "choices": ["endpoint", "cloud", "application", "network", "web"], + "default": "endpoint", }, { - 'type': 'text', - 'message': 'enter author name', - 'name': 'detection_author', + "type": "text", + "message": "enter author name", + "name": "detection_author", }, { "type": "select", @@ -46,12 +39,11 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: "default": "TTP", }, { - 'type': 'checkbox', - 'message': 'Your data source', - 'name': 'data_sources', - #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory - 'choices': sorted(DataSource._value2member_map_ ) - + "type": "checkbox", + "message": "Your data source", + "name": "data_sources", + # In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory + "choices": sorted(DataSource._value2member_map_), }, { "type": "text", @@ -66,24 +58,24 @@ def get_questions_detection(cls) -> list[dict[str,Any]]: "default": "T1003.002", }, { - 'type': 'select', - 'message': 'security_domain for detection', - 'name': 'security_domain', - 'choices': [ - 'access', - 'endpoint', - 'network', - 'threat', - 'identity', - 'audit' + "type": "select", + "message": "security_domain for detection", + "name": "security_domain", + "choices": [ + "access", + "endpoint", + "network", + "threat", + "identity", + "audit", ], - 'default': 'endpoint' + "default": "endpoint", }, ] return questions @classmethod - def get_questions_story(cls)-> list[dict[str,Any]]: + def get_questions_story(cls) -> list[dict[str, Any]]: questions = [ { "type": "text", diff --git a/contentctl/input/yml_reader.py b/contentctl/input/yml_reader.py index bc3219b5..65e74069 100644 --- a/contentctl/input/yml_reader.py +++ b/contentctl/input/yml_reader.py @@ -3,36 +3,43 @@ import sys import pathlib -class YmlReader(): +class YmlReader: @staticmethod - def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING:bool=False) -> Dict[str,Any]: + def load_file( + file_path: pathlib.Path, + add_fields: bool = True, + STRICT_YML_CHECKING: bool = False, + ) -> Dict[str, Any]: try: - file_handler = open(file_path, 'r', encoding="utf-8") - - # The following code can help diagnose issues with duplicate keys or + file_handler = open(file_path, "r", encoding="utf-8") + + # The following code can help diagnose issues with duplicate keys or # poorly-formatted but still "compliant" YML. This code should be - # enabled manually for debugging purposes. As such, strictyaml + # enabled manually for debugging purposes. As such, strictyaml # library is intentionally excluded from the contentctl requirements if STRICT_YML_CHECKING: import strictyaml + try: - strictyaml.dirty_load(file_handler.read(), allow_flow_style = True) + strictyaml.dirty_load(file_handler.read(), allow_flow_style=True) file_handler.seek(0) except Exception as e: print(f"Error loading YML file {file_path}: {str(e)}") sys.exit(1) try: - #Ideally we should use - # from contentctl.actions.new_content import NewContent - # and use NewContent.UPDATE_PREFIX, + # Ideally we should use + # from contentctl.actions.new_content import NewContent + # and use NewContent.UPDATE_PREFIX, # but there is a circular dependency right now which makes that difficult. # We have instead hardcoded UPDATE_PREFIX UPDATE_PREFIX = "__UPDATE__" data = file_handler.read() if UPDATE_PREFIX in data: - raise Exception(f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required.") + raise Exception( + f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required." + ) yml_obj = yaml.load(data, Loader=yaml.CSafeLoader) except yaml.YAMLError as exc: print(exc) @@ -41,12 +48,10 @@ def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING except OSError as exc: print(exc) sys.exit(1) - + if add_fields is False: return yml_obj - - - yml_obj['file_path'] = str(file_path) - + + yml_obj["file_path"] = str(file_path) return yml_obj diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 0a0fceff..bd9a78ec 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -11,16 +11,17 @@ Field, computed_field, model_serializer, - FilePath + FilePath, ) from contentctl.objects.macro import Macro from contentctl.objects.lookup import Lookup, FileBackedLookup, KVStoreLookup + if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.baseline import Baseline from contentctl.objects.config import CustomApp - + from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.enums import AnalyticsType from contentctl.objects.enums import DataModel @@ -46,20 +47,18 @@ ES_MAX_STANZA_LENGTH, ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE, CONTENTCTL_MAX_SEARCH_NAME_LENGTH, - CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE + CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE, ) MISSING_SOURCES: set[str] = set() # Those AnalyticsTypes that we do not test via contentctl -SKIPPED_ANALYTICS_TYPES: set[str] = { - AnalyticsType.Correlation -} +SKIPPED_ANALYTICS_TYPES: set[str] = {AnalyticsType.Correlation} class Detection_Abstract(SecurityContentObject): - name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) - #contentType: SecurityContentType = SecurityContentType.detections + name: str = Field(..., max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) + # contentType: SecurityContentType = SecurityContentType.detections type: AnalyticsType = Field(...) status: DetectionStatus = Field(...) data_source: list[str] = [] @@ -70,16 +69,15 @@ class Detection_Abstract(SecurityContentObject): rba: Optional[RBAObject] = Field(default=None) explanation: None | str = Field( default=None, - exclude=True, #Don't serialize this value when dumping the object + exclude=True, # Don't serialize this value when dumping the object description="Provide an explanation to be included " "in the 'Explanation' field of the Detection in " "the Use Case Library. If this field is not " "defined in the YML, it will default to the " - "value of the 'description' field when " + "value of the 'description' field when " "serialized in analyticstories_detections.j2", ) - enabled_by_default: bool = False file_path: FilePath = Field(...) # For model construction to first attempt construction of the leftmost object. @@ -87,36 +85,49 @@ class Detection_Abstract(SecurityContentObject): # default mode, 'smart' # https://docs.pydantic.dev/latest/concepts/unions/#left-to-right-mode # https://github.com/pydantic/pydantic/issues/9101#issuecomment-2019032541 - tests: List[Annotated[Union[UnitTest, IntegrationTest, ManualTest], Field(union_mode='left_to_right')]] = [] + tests: List[ + Annotated[ + Union[UnitTest, IntegrationTest, ManualTest], + Field(union_mode="left_to_right"), + ] + ] = [] # A list of groups of tests, relying on the same data test_groups: list[TestGroup] = [] data_source_objects: list[DataSource] = [] - drilldown_searches: list[Drilldown] = Field(default=[], description="A list of Drilldowns that should be included with this search") + drilldown_searches: list[Drilldown] = Field( + default=[], + description="A list of Drilldowns that should be included with this search", + ) - def get_conf_stanza_name(self, app:CustomApp)->str: - stanza_name = CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name) + def get_conf_stanza_name(self, app: CustomApp) -> str: + stanza_name = CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format( + app_label=app.label, detection_name=self.name + ) self.check_conf_stanza_max_length(stanza_name) return stanza_name - - def get_action_dot_correlationsearch_dot_label(self, app:CustomApp, max_stanza_length:int=ES_MAX_STANZA_LENGTH)->str: + def get_action_dot_correlationsearch_dot_label( + self, app: CustomApp, max_stanza_length: int = ES_MAX_STANZA_LENGTH + ) -> str: stanza_name = self.get_conf_stanza_name(app) - stanza_name_after_saving_in_es = ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format( - security_domain_value = self.tags.security_domain, - search_name = stanza_name + stanza_name_after_saving_in_es = ( + ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format( + security_domain_value=self.tags.security_domain, search_name=stanza_name ) - - + ) + if len(stanza_name_after_saving_in_es) > max_stanza_length: - raise ValueError(f"label may only be {max_stanza_length} characters to allow updating in-product, " - f"but stanza was actually {len(stanza_name_after_saving_in_es)} characters: '{stanza_name_after_saving_in_es}' ") - - return stanza_name + raise ValueError( + f"label may only be {max_stanza_length} characters to allow updating in-product, " + f"but stanza was actually {len(stanza_name_after_saving_in_es)} characters: '{stanza_name_after_saving_in_es}' " + ) + + return stanza_name @field_validator("search", mode="before") @classmethod - def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str: + def validate_presence_of_filter_macro(cls, value: str, info: ValidationInfo) -> str: """ Validates that, if required to be present, the filter macro is present with the proper name. The filter macro MUST be derived from the name of the detection @@ -130,7 +141,7 @@ def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str: Returns: str: The search, as an SPL formatted string. """ - + # Otherwise, the search is SPL. # In the future, we will may add support that makes the inclusion of the @@ -170,7 +181,7 @@ def adjust_tests_and_groups(self) -> None: the model from the list of unit tests. Also, preemptively skips all manual tests, as well as tests for experimental/deprecated detections and Correlation type detections. """ - + # Since ManualTest and UnitTest are not differentiable without looking at the manual_test # tag, Pydantic builds all tests as UnitTest objects. If we see the manual_test flag, we # convert these to ManualTest @@ -183,10 +194,7 @@ def adjust_tests_and_groups(self) -> None: f"but encountered a {type(test)}." ) # Create the manual test and skip it upon creation (cannot test via contentctl) - manual_test = ManualTest( - name=test.name, - attack_data=test.attack_data - ) + manual_test = ManualTest(name=test.name, attack_data=test.attack_data) tmp.append(manual_test) self.tests = tmp @@ -212,8 +220,10 @@ def adjust_tests_and_groups(self) -> None: # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name # Skip tests for non-production detections - if self.status != DetectionStatus.production: - self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})") + if self.status != DetectionStatus.production: + self.skip_all_tests( + f"TEST SKIPPED: Detection is non-production ({self.status})" + ) # Skip tests for detecton types like Correlation which are not supported via contentctl if self.type in SKIPPED_ANALYTICS_TYPES: @@ -240,7 +250,10 @@ def test_status(self) -> TestResultStatus | None: # If the result/status of any test has not yet been set, return None if test.result is None or test.result.status is None: return None - elif test.result.status == TestResultStatus.ERROR or test.result.status == TestResultStatus.FAIL: + elif ( + test.result.status == TestResultStatus.ERROR + or test.result.status == TestResultStatus.FAIL + ): # If any test failed or errored, return fail (we don't return the error state at # the aggregate detection level) return TestResultStatus.FAIL @@ -266,24 +279,21 @@ def test_status(self) -> TestResultStatus | None: @property def datamodel(self) -> List[DataModel]: return [dm for dm in DataModel if dm in self.search] - - - @computed_field @property def source(self) -> str: return self.file_path.absolute().parent.name - deployment: Deployment = Field({}) @computed_field @property def annotations(self) -> dict[str, Union[List[str], int, str]]: - annotations_dict: dict[str, str | list[str] | int] = {} - annotations_dict["analytic_story"] = [story.name for story in self.tags.analytic_story] + annotations_dict["analytic_story"] = [ + story.name for story in self.tags.analytic_story + ] if len(self.tags.cve or []) > 0: annotations_dict["cve"] = self.tags.cve annotations_dict["type"] = self.type @@ -310,11 +320,13 @@ def mappings(self) -> dict[str, List[str]]: if len(self.tags.cis20) > 0: mappings["cis20"] = [tag for tag in self.tags.cis20] if len(self.tags.kill_chain_phases) > 0: - mappings['kill_chain_phases'] = [phase for phase in self.tags.kill_chain_phases] + mappings["kill_chain_phases"] = [ + phase for phase in self.tags.kill_chain_phases + ] if len(self.tags.mitre_attack_id) > 0: - mappings['mitre_attack'] = self.tags.mitre_attack_id + mappings["mitre_attack"] = self.tags.mitre_attack_id if len(self.tags.nist) > 0: - mappings['nist'] = [category for category in self.tags.nist] + mappings["nist"] = [category for category in self.tags.nist] # No need to sort the dict! It has been constructed in-order. # However, if this logic is changed, then consider reordering or @@ -329,8 +341,10 @@ def mappings(self) -> dict[str, List[str]]: def cve_enrichment_func(self, __context: Any): if len(self.cve_enrichment) > 0: - raise ValueError(f"Error, field 'cve_enrichment' should be empty and " - f"dynamically populated at runtime. Instead, this field contained: {self.cve_enrichment}") + raise ValueError( + f"Error, field 'cve_enrichment' should be empty and " + f"dynamically populated at runtime. Instead, this field contained: {self.cve_enrichment}" + ) output_dto: Union[DirectorOutputDto, None] = __context.get("output_dto", None) if output_dto is None: @@ -340,7 +354,11 @@ def cve_enrichment_func(self, __context: Any): for cve_id in self.tags.cve: try: - enriched_cves.append(output_dto.cve_enrichment.enrich_cve(cve_id, raise_exception_on_failure=False)) + enriched_cves.append( + output_dto.cve_enrichment.enrich_cve( + cve_id, raise_exception_on_failure=False + ) + ) except Exception as e: raise ValueError(f"{e}") self.cve_enrichment = enriched_cves @@ -352,7 +370,7 @@ def cve_enrichment_func(self, __context: Any): @property def nes_fields(self) -> Optional[str]: if self.deployment.alert_action.notable is not None: - return ','.join(self.deployment.alert_action.notable.nes_fields) + return ",".join(self.deployment.alert_action.notable.nes_fields) else: return None @@ -361,7 +379,6 @@ def nes_fields(self) -> Optional[str]: def providing_technologies(self) -> List[ProvidingTechnology]: return ProvidingTechnology.getProvidingTechFromSearch(self.search) - @computed_field @property def risk(self) -> list[dict[str, Any]]: @@ -369,19 +386,18 @@ def risk(self) -> list[dict[str, Any]]: for entity in self.rba.risk_objects: risk_object: dict[str, str | int] = dict() - risk_object['risk_object_type'] = entity.type - risk_object['risk_object_field'] = entity.field - risk_object['risk_score'] = entity.score + risk_object["risk_object_type"] = entity.type + risk_object["risk_object_field"] = entity.field + risk_object["risk_score"] = entity.score risk_objects.append(risk_object) - + for entity in self.rba.threat_objects: threat_object: dict[str, str] = dict() - threat_object['threat_object_field'] = entity.field - threat_object['threat_object_type'] = entity.type + threat_object["threat_object_field"] = entity.field + threat_object["threat_object_type"] = entity.type risk_objects.append(threat_object) return risk_objects - # TODO Remove observable code # @computed_field # @property @@ -428,7 +444,7 @@ def risk(self) -> list[dict[str, Any]]: # risk_object['threat_object_field'] = entity.name # risk_object['threat_object_type'] = "url" # risk_objects.append(risk_object) - + # elif 'Attacker' in entity.role: # risk_object['threat_object_field'] = entity.name # risk_object['threat_object_type'] = entity.type.lower() @@ -445,7 +461,7 @@ def risk(self) -> list[dict[str, Any]]: @computed_field @property - def metadata(self) -> dict[str, str|float]: + def metadata(self) -> dict[str, str | float]: # NOTE: we ignore the type error around self.status because we are using Pydantic's # use_enum_values configuration # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name @@ -455,10 +471,19 @@ def metadata(self) -> dict[str, str|float]: # dict below) should not have any impact, but renaming or removing any of these fields will # break the `inspect` action. return { - 'detection_id': str(self.id), - 'deprecated': '1' if self.status == DetectionStatus.deprecated else '0', # type: ignore - 'detection_version': str(self.version), - 'publish_time': datetime.datetime(self.date.year,self.date.month,self.date.day,0,0,0,0,tzinfo=datetime.timezone.utc).timestamp() + "detection_id": str(self.id), + "deprecated": "1" if self.status == DetectionStatus.deprecated else "0", # type: ignore + "detection_version": str(self.version), + "publish_time": datetime.datetime( + self.date.year, + self.date.month, + self.date.day, + 0, + 0, + 0, + 0, + tzinfo=datetime.timezone.utc, + ).timestamp(), } @model_serializer @@ -479,9 +504,9 @@ def serialize_model(self): } if self.rba is not None: model["risk_severity"] = self.rba.severity - model['tags']['risk_score'] = self.rba.risk_score + model["tags"]["risk_score"] = self.rba.risk_score else: - model['tags']['risk_score'] = 0 + model["tags"]["risk_score"] = 0 # Only a subset of macro fields are required: all_macros: list[dict[str, str | list[str]]] = [] @@ -489,13 +514,13 @@ def serialize_model(self): macro_dump: dict[str, str | list[str]] = { "name": macro.name, "definition": macro.definition, - "description": macro.description + "description": macro.description, } if len(macro.arguments) > 0: - macro_dump['arguments'] = macro.arguments + macro_dump["arguments"] = macro.arguments all_macros.append(macro_dump) - model['macros'] = all_macros # type: ignore + model["macros"] = all_macros # type: ignore all_lookups: list[dict[str, str | int | None]] = [] for lookup in self.lookups: @@ -506,7 +531,7 @@ def serialize_model(self): "description": lookup.description, "collection": lookup.collection, "case_sensitive_match": None, - "fields_list": lookup.fields_to_fields_list_conf_format + "fields_list": lookup.fields_to_fields_list_conf_format, } ) elif isinstance(lookup, FileBackedLookup): @@ -516,15 +541,17 @@ def serialize_model(self): "description": lookup.description, "filename": lookup.filename.name, "default_match": "true" if lookup.default_match else "false", - "case_sensitive_match": "true" if lookup.case_sensitive_match else "false", + "case_sensitive_match": "true" + if lookup.case_sensitive_match + else "false", "match_type": lookup.match_type_to_conf_format, - "min_matches": lookup.min_matches + "min_matches": lookup.min_matches, } ) - model['lookups'] = all_lookups # type: ignore + model["lookups"] = all_lookups # type: ignore # Combine fields from this model with fields from parent - super_fields.update(model) # type: ignore + super_fields.update(model) # type: ignore # return the model return super_fields @@ -557,7 +584,7 @@ def model_post_init(self, __context: Any) -> None: updated_data_source_names: set[str] = set() for ds in self.data_source: - split_data_sources = {d.strip() for d in ds.split('AND')} + split_data_sources = {d.strip() for d in ds.split("AND")} updated_data_source_names.update(split_data_sources) sources = sorted(list(updated_data_source_names)) @@ -566,7 +593,9 @@ def model_post_init(self, __context: Any) -> None: missing_sources: list[str] = [] for source in sources: try: - matched_data_sources += DataSource.mapNamesToSecurityContentObjects([source], director) + matched_data_sources += DataSource.mapNamesToSecurityContentObjects( + [source], director + ) except Exception: # We gobble this up and add it to a global set so that we # can print it ONCE at the end of the build of datasources. @@ -583,7 +612,7 @@ def model_post_init(self, __context: Any) -> None: self.data_source_objects = matched_data_sources for story in self.tags.analytic_story: - story.detections.append(self) + story.detections.append(self) self.cve_enrichment_func(__context) @@ -594,32 +623,39 @@ def model_post_init(self, __context: Any) -> None: # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting - - if self.type == AnalyticsType.Hunting or self.status != DetectionStatus.production: - #No additional check need to happen on the potential drilldowns. + + if ( + self.type == AnalyticsType.Hunting + or self.status != DetectionStatus.production + ): + # No additional check need to happen on the potential drilldowns. pass else: found_placeholder = False if len(self.drilldown_searches) < 2: - raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + raise ValueError( + f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]" + ) for drilldown in self.drilldown_searches: if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: found_placeholder = True if not found_placeholder: - raise ValueError("Detection has one or more drilldown_searches, but none of them " - f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " - "if drilldown_searches are defined.'") - + raise ValueError( + "Detection has one or more drilldown_searches, but none of them " + f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + "if drilldown_searches are defined.'" + ) + # Update the search fields with the original search, if required for drilldown in self.drilldown_searches: drilldown.perform_search_substitutions(self) - #For experimental purposes, add the default drilldowns - #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) + # For experimental purposes, add the default drilldowns + # self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) @property - def drilldowns_in_JSON(self) -> list[dict[str,str]]: - """This function is required for proper JSON + def drilldowns_in_JSON(self) -> list[dict[str, str]]: + """This function is required for proper JSON serializiation of drilldowns to occur in savedsearches.conf. It returns the list[Drilldown] as a list[dict]. Without this function, the jinja template is unable @@ -627,24 +663,26 @@ def drilldowns_in_JSON(self) -> list[dict[str,str]]: Returns: list[dict[str,str]]: List of Drilldowns dumped to dict format - """ + """ return [drilldown.model_dump() for drilldown in self.drilldown_searches] - @field_validator('lookups', mode="before") + @field_validator("lookups", mode="before") @classmethod - def getDetectionLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]: - director:DirectorOutputDto = info.context.get("output_dto",None) - - search:Union[str,None] = info.data.get("search",None) + def getDetectionLookups(cls, v: list[str], info: ValidationInfo) -> list[Lookup]: + director: DirectorOutputDto = info.context.get("output_dto", None) + + search: Union[str, None] = info.data.get("search", None) if search is None: raise ValueError("Search was None - is this file missing the search field?") - + lookups = Lookup.get_lookups(search, director) return lookups - @field_validator('baselines', mode="before") + @field_validator("baselines", mode="before") @classmethod - def mapDetectionNamesToBaselineObjects(cls, v: list[str], info: ValidationInfo) -> List[Baseline]: + def mapDetectionNamesToBaselineObjects( + cls, v: list[str], info: ValidationInfo + ) -> List[Baseline]: if len(v) > 0: raise ValueError( "Error, baselines are constructed automatically at runtime. Please do not include this field." @@ -652,7 +690,9 @@ def mapDetectionNamesToBaselineObjects(cls, v: list[str], info: ValidationInfo) name: Union[str, None] = info.data.get("name", None) if name is None: - raise ValueError("Error, cannot get Baselines because the Detection does not have a 'name' defined.") + raise ValueError( + "Error, cannot get Baselines because the Detection does not have a 'name' defined." + ) if info.context is None: raise ValueError("ValidationInfo.context unexpectedly null") @@ -663,14 +703,16 @@ def mapDetectionNamesToBaselineObjects(cls, v: list[str], info: ValidationInfo) # This matching is a bit strange, because baseline.tags.detections starts as a list of strings, but # is eventually updated to a list of Detections as we construct all of the detection objects. detection_names = [ - detection_name for detection_name in baseline.tags.detections if isinstance(detection_name, str) + detection_name + for detection_name in baseline.tags.detections + if isinstance(detection_name, str) ] if name in detection_names: baselines.append(baseline) return baselines - @field_validator('macros', mode="before") + @field_validator("macros", mode="before") @classmethod def getDetectionMacros(cls, v: list[str], info: ValidationInfo) -> list[Macro]: if info.context is None: @@ -686,21 +728,25 @@ def getDetectionMacros(cls, v: list[str], info: ValidationInfo) -> list[Macro]: message = f"Expected 'search_name' to be a string, instead it was [{type(search_name)}]" assert isinstance(search_name, str), message - filter_macro_name = search_name.replace(' ', '_')\ - .replace('-', '_')\ - .replace('.', '_')\ - .replace('/', '_')\ - .lower()\ - + '_filter' + filter_macro_name = ( + search_name.replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + + "_filter" + ) try: - filter_macro = Macro.mapNamesToSecurityContentObjects([filter_macro_name], director)[0] + filter_macro = Macro.mapNamesToSecurityContentObjects( + [filter_macro_name], director + )[0] except Exception: # Filter macro did not exist, so create one at runtime filter_macro = Macro.model_validate( { "name": filter_macro_name, - "definition": 'search *', - "description": 'Update this macro to limit the output results to filter out false positives.' + "definition": "search *", + "description": "Update this macro to limit the output results to filter out false positives.", } ) director.addContentToDictMappings(filter_macro) @@ -723,12 +769,12 @@ def getDeployment(cls, v: Any, info: ValidationInfo) -> Deployment: @field_validator("enabled_by_default", mode="before") def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool: - ''' + """ A detection can ONLY be enabled by default if it is a PRODUCTION detection. If not (for example, it is EXPERIMENTAL or DEPRECATED) then we will throw an exception. Similarly, a detection MUST be schedulable, meaning that it must be Anomaly, Correleation, or TTP. We will not allow Hunting searches to be enabled by default. - ''' + """ if v is False: return v @@ -739,16 +785,23 @@ def only_enabled_if_production_status(cls, v: Any, info: ValidationInfo) -> bool errors.append( f"status is '{status.name}'. Detections that are enabled by default MUST be " f"'{DetectionStatus.production}'" - ) + ) - if searchType not in [AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]: + if searchType not in [ + AnalyticsType.Anomaly, + AnalyticsType.Correlation, + AnalyticsType.TTP, + ]: errors.append( f"type is '{searchType}'. Detections that are enabled by default MUST be one" " of the following types: " - f"{[AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]}") + f"{[AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]}" + ) if len(errors) > 0: error_message = "\n - ".join(errors) - raise ValueError(f"Detection is 'enabled_by_default: true' however \n - {error_message}") + raise ValueError( + f"Detection is 'enabled_by_default: true' however \n - {error_message}" + ) return v @@ -759,39 +812,42 @@ def addTags_nist(self): else: self.tags.nist = [NistCategory.DE_AE] return self - @model_validator(mode="after") def ensureThrottlingFieldsExist(self): - ''' + """ For throttling to work properly, the fields to throttle on MUST exist in the search itself. If not, then we cannot apply the throttling - ''' + """ if self.tags.throttling is None: # No throttling configured for this detection return self - missing_fields:list[str] = [field for field in self.tags.throttling.fields if field not in self.search] + missing_fields: list[str] = [ + field for field in self.tags.throttling.fields if field not in self.search + ] if len(missing_fields) > 0: - raise ValueError(f"The following throttle fields were missing from the search: {missing_fields}") + raise ValueError( + f"The following throttle fields were missing from the search: {missing_fields}" + ) else: # All throttling fields present in search return self - - @model_validator(mode="after") def ensureProperRBAConfig(self): """ If a detection has an RBA deployment and is PRODUCTION, then it must have an RBA config, with at least one risk object - + Returns: self: Returns itself if the validation passes """ - - if self.deployment.alert_action.rba is None or self.deployment.alert_action.rba.enabled is False: + if ( + self.deployment.alert_action.rba is None + or self.deployment.alert_action.rba.enabled is False + ): # confirm we don't have an RBA config if self.rba is None: return self @@ -805,14 +861,13 @@ def ensureProperRBAConfig(self): "Detection is expected to have an RBA object based on its deployment config" ) else: - if len(self.rba.risk_objects) > 0: # type: ignore + if len(self.rba.risk_objects) > 0: # type: ignore return self else: raise ValueError( "Detection expects an RBA config with at least one risk object." ) - # TODO - Remove old observable code # @model_validator(mode="after") # def ensureProperObservablesExist(self): @@ -850,10 +905,13 @@ def ensureProperRBAConfig(self): @model_validator(mode="after") def search_rba_fields_exist_validate(self): # Return immediately if RBA isn't required - if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None) and self.rba is None: #type: ignore + if ( + self.deployment.alert_action.rba.enabled is False + or self.deployment.alert_action.rba is None + ) and self.rba is None: # type: ignore return self - - # Raise error if RBA isn't present + + # Raise error if RBA isn't present if self.rba is None: raise ValueError( "RBA is required for this detection based on its deployment config" @@ -868,7 +926,9 @@ def search_rba_fields_exist_validate(self): if self.rba.message: matches = re.findall(field_match_regex, self.rba.message.lower()) message_fields = [match.replace("$", "").lower() for match in matches] - missing_fields = set([field for field in rba_fields if field not in self.search.lower()]) + missing_fields = set( + [field for field in rba_fields if field not in self.search.lower()] + ) else: message_fields = [] missing_fields = set() @@ -879,15 +939,16 @@ def search_rba_fields_exist_validate(self): "The following fields are declared in the rba config, but do not exist in the " f"search: {missing_fields}" ) - missing_fields = set([field for field in message_fields if field not in self.search.lower()]) + missing_fields = set( + [field for field in message_fields if field not in self.search.lower()] + ) if len(missing_fields) > 0: error_messages.append( "The following fields are used as fields in the message, but do not exist in " f"the search: {missing_fields}" ) - if len(error_messages) > 0 and self.status == DetectionStatus.production: - + if len(error_messages) > 0 and self.status == DetectionStatus.production: msg = ( "Use of fields in rba/messages that do not appear in search:\n\t- " "\n\t- ".join(error_messages) @@ -940,7 +1001,7 @@ def search_rba_fields_exist_validate(self): # return self @field_validator("tests", mode="before") - def ensure_yml_test_is_unittest(cls, v:list[dict]): + def ensure_yml_test_is_unittest(cls, v: list[dict]): """The typing for the tests field allows it to be one of a number of different types of tests. However, ONLY UnitTest should be allowed to be defined in the YML @@ -956,17 +1017,17 @@ def ensure_yml_test_is_unittest(cls, v:list[dict]): it into a different type of test Args: - v (list[dict]): list of dicts read from the yml. + v (list[dict]): list of dicts read from the yml. Each one SHOULD be a valid UnitTest. If we cannot construct a valid unitTest from it, a ValueError should be raised Returns: - _type_: The input of the function, assuming no + _type_: The input of the function, assuming no ValueError is raised. - """ - valueErrors:list[ValueError] = [] + """ + valueErrors: list[ValueError] = [] for unitTest in v: - #This raises a ValueError on a failed UnitTest. + # This raises a ValueError on a failed UnitTest. try: UnitTest.model_validate(unitTest) except ValueError as e: @@ -976,13 +1037,10 @@ def ensure_yml_test_is_unittest(cls, v:list[dict]): # All of these can be constructred as UnitTests with no # Exceptions, so let the normal flow continue return v - @field_validator("tests") def tests_validate( - cls, - v: list[UnitTest | IntegrationTest | ManualTest], - info: ValidationInfo + cls, v: list[UnitTest | IntegrationTest | ManualTest], info: ValidationInfo ) -> list[UnitTest | IntegrationTest | ManualTest]: # Only production analytics require tests if info.data.get("status", "") != DetectionStatus.production: @@ -1002,7 +1060,8 @@ def tests_validate( # Ensure that there is at least 1 test if len(v) == 0: raise ValueError( - "At least one test is REQUIRED for production detection: " + info.data.get("name", "NO NAME FOUND") + "At least one test is REQUIRED for production detection: " + + info.data.get("name", "NO NAME FOUND") ) # No issues - at least one test provided for production type requiring testing @@ -1074,13 +1133,29 @@ def all_tests_successful(self) -> bool: def get_summary( self, detection_fields: list[str] = [ - "name", "type", "status", "test_status", "source", "data_source", "search", "file_path" + "name", + "type", + "status", + "test_status", + "source", + "data_source", + "search", + "file_path", ], detection_field_aliases: dict[str, str] = { - "status": "production_status", "test_status": "status", "source": "source_category" + "status": "production_status", + "test_status": "status", + "source": "source_category", }, tags_fields: list[str] = ["manual_test"], - test_result_fields: list[str] = ["success", "message", "exception", "status", "duration", "wait_duration"], + test_result_fields: list[str] = [ + "success", + "message", + "exception", + "status", + "duration", + "wait_duration", + ], test_job_fields: list[str] = ["resultCount", "runDuration"], ) -> dict[str, Any]: """ @@ -1120,7 +1195,7 @@ def get_summary( # Initialize the dict as a mapping of strings to str/bool result: dict[str, Union[str, bool]] = { "name": test.name, - "test_type": test.test_type + "test_type": test.test_type, } # If result is not None, get a summary of the test result w/ the requested fields @@ -1137,7 +1212,7 @@ def get_summary( result["message"] = "NO RESULT - Test not run" # Add the result to our list - summary_dict["tests"].append(result) # type: ignore + summary_dict["tests"].append(result) # type: ignore # Return the summary diff --git a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py index 2ddb3124..bc90d750 100644 --- a/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py @@ -21,7 +21,7 @@ HttpUrl, NonNegativeInt, ConfigDict, - model_serializer + model_serializer, ) from typing import Tuple, Optional, List, Union import pathlib @@ -31,13 +31,13 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC): - model_config = ConfigDict(validate_default=True,extra="forbid") - name: str = Field(...,max_length=99) - author: str = Field(...,max_length=255) + model_config = ConfigDict(validate_default=True, extra="forbid") + name: str = Field(..., max_length=99) + author: str = Field(..., max_length=255) date: datetime.date = Field(...) version: NonNegativeInt = Field(...) - id: uuid.UUID = Field(...) #we set a default here until all content has a uuid - description: str = Field(...,max_length=10000) + id: uuid.UUID = Field(...) # we set a default here until all content has a uuid + description: str = Field(..., max_length=10000) file_path: Optional[FilePath] = None references: Optional[List[HttpUrl]] = None @@ -53,15 +53,18 @@ def serialize_model(self): "version": self.version, "id": str(self.id), "description": self.description, - "references": [str(url) for url in self.references or []] + "references": [str(url) for url in self.references or []], } - - - def check_conf_stanza_max_length(self, stanza_name:str, max_stanza_length:int=CONTENTCTL_MAX_STANZA_LENGTH) -> None: + + def check_conf_stanza_max_length( + self, stanza_name: str, max_stanza_length: int = CONTENTCTL_MAX_STANZA_LENGTH + ) -> None: if len(stanza_name) > max_stanza_length: - raise ValueError(f"conf stanza may only be {max_stanza_length} characters, " - f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' ") - + raise ValueError( + f"conf stanza may only be {max_stanza_length} characters, " + f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' " + ) + @staticmethod def objectListToNameList(objects: list[SecurityContentObject]) -> list[str]: return [object.getName() for object in objects] @@ -73,17 +76,19 @@ def getName(self) -> str: @classmethod def contentNameToFileName(cls, content_name: str) -> str: - return content_name \ - .replace(' ', '_') \ - .replace('-', '_') \ - .replace('.', '_') \ - .replace('/', '_') \ - .lower() + ".yml" + return ( + content_name.replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + + ".yml" + ) def ensureFileNameMatchesSearchName(self): file_name = self.contentNameToFileName(self.name) - if (self.file_path is not None and file_name != self.file_path.name): + if self.file_path is not None and file_name != self.file_path.name: raise ValueError( f"The file name MUST be based off the content 'name' field:\n" f"\t- Expected File Name: {file_name}\n" @@ -92,7 +97,7 @@ def ensureFileNameMatchesSearchName(self): return self - @field_validator('file_path') + @field_validator("file_path") @classmethod def file_path_valid(cls, v: Optional[pathlib.PosixPath], info: ValidationInfo): if not v: @@ -109,7 +114,9 @@ def getReferencesListForJson(self) -> List[str]: return [str(url) for url in self.references or []] @classmethod - def mapNamesToSecurityContentObjects(cls, v: list[str], director: Union[DirectorOutputDto, None]) -> list[Self]: + def mapNamesToSecurityContentObjects( + cls, v: list[str], director: Union[DirectorOutputDto, None] + ) -> list[Self]: if director is not None: name_map = director.name_to_content_map else: @@ -129,7 +136,9 @@ def mapNamesToSecurityContentObjects(cls, v: list[str], director: Union[Director errors: list[str] = [] if len(missing_objects) > 0: - errors.append(f"Failed to find the following '{cls.__name__}': {missing_objects}") + errors.append( + f"Failed to find the following '{cls.__name__}': {missing_objects}" + ) if len(mistyped_objects) > 0: for mistyped_object in mistyped_objects: errors.append( @@ -141,13 +150,16 @@ def mapNamesToSecurityContentObjects(cls, v: list[str], director: Union[Director error_string = "\n - ".join(errors) raise ValueError( f"Found {len(errors)} issues when resolving references Security Content Object " - f"names:\n - {error_string}") + f"names:\n - {error_string}" + ) # Sort all objects sorted by name return sorted(mappedObjects, key=lambda o: o.name) @staticmethod - def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> Deployment: + def getDeploymentFromType( + typeField: Union[str, None], info: ValidationInfo + ) -> Deployment: if typeField is None: raise ValueError("'type:' field is missing from YML.") @@ -156,21 +168,25 @@ def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> director: Optional[DirectorOutputDto] = info.context.get("output_dto", None) if not director: - raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context") + raise ValueError( + "Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context" + ) type_to_deployment_name_map = { AnalyticsType.TTP: "ESCU Default Configuration TTP", AnalyticsType.Hunting: "ESCU Default Configuration Hunting", AnalyticsType.Correlation: "ESCU Default Configuration Correlation", AnalyticsType.Anomaly: "ESCU Default Configuration Anomaly", - "Baseline": "ESCU Default Configuration Baseline" + "Baseline": "ESCU Default Configuration Baseline", } converted_type_field = type_to_deployment_name_map[typeField] # TODO: This is clunky, but is imported here to resolve some circular import errors from contentctl.objects.deployment import Deployment - deployments = Deployment.mapNamesToSecurityContentObjects([converted_type_field], director) + deployments = Deployment.mapNamesToSecurityContentObjects( + [converted_type_field], director + ) if len(deployments) == 1: return deployments[0] elif len(deployments) == 0: @@ -186,18 +202,19 @@ def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> @staticmethod def get_objects_by_name( - names_to_find: set[str], - objects_to_search: list[SecurityContentObject_Abstract] + names_to_find: set[str], objects_to_search: list[SecurityContentObject_Abstract] ) -> Tuple[list[SecurityContentObject_Abstract], set[str]]: raise Exception("get_objects_by_name deprecated") - found_objects = list(filter(lambda obj: obj.name in names_to_find, objects_to_search)) + found_objects = list( + filter(lambda obj: obj.name in names_to_find, objects_to_search) + ) found_names = set([obj.name for obj in found_objects]) missing_names = names_to_find - found_names return found_objects, missing_names @staticmethod def create_filename_to_content_dict( - all_objects: list[SecurityContentObject_Abstract] + all_objects: list[SecurityContentObject_Abstract], ) -> dict[str, SecurityContentObject_Abstract]: name_dict: dict[str, SecurityContentObject_Abstract] = dict() for object in all_objects: @@ -205,7 +222,9 @@ def create_filename_to_content_dict( # SecurityContentObject (e.g. filter macros that are created at runtime but have no # actual file associated) if object.file_path is None: - raise ValueError(f"SecurityContentObject is missing a file_path: {object.name}") + raise ValueError( + f"SecurityContentObject is missing a file_path: {object.name}" + ) name_dict[str(pathlib.Path(object.file_path))] = object return name_dict @@ -222,12 +241,16 @@ def __str__(self) -> str: def __lt__(self, other: object) -> bool: if not isinstance(other, SecurityContentObject_Abstract): - raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}") + raise Exception( + f"SecurityContentObject can only be compared to each other, not to {type(other)}" + ) return self.name < other.name def __eq__(self, other: object) -> bool: if not isinstance(other, SecurityContentObject_Abstract): - raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}") + raise Exception( + f"SecurityContentObject can only be compared to each other, not to {type(other)}" + ) if id(self) == id(other) and self.name == other.name and self.id == other.id: # Yes, this is the same object diff --git a/contentctl/objects/alert_action.py b/contentctl/objects/alert_action.py index d2855292..c50e9bdb 100644 --- a/contentctl/objects/alert_action.py +++ b/contentctl/objects/alert_action.py @@ -8,6 +8,7 @@ from contentctl.objects.deployment_slack import DeploymentSlack from contentctl.objects.deployment_phantom import DeploymentPhantom + class AlertAction(BaseModel): model_config = ConfigDict(extra="forbid") email: Optional[DeploymentEmail] = None @@ -16,26 +17,25 @@ class AlertAction(BaseModel): slack: Optional[DeploymentSlack] = None phantom: Optional[DeploymentPhantom] = None - @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent model = {} if self.email is not None: raise Exception("Email not implemented") if self.notable is not None: - model['notable'] = self.notable + model["notable"] = self.notable if self.rba is not None and self.rba.enabled: - model['rba'] = {'enabled': "true"} + model["rba"] = {"enabled": "true"} if self.slack is not None: raise Exception("Slack not implemented") - + if self.phantom is not None: raise Exception("Phantom not implemented") - - #return the model - return model \ No newline at end of file + + # return the model + return model diff --git a/contentctl/objects/annotated_types.py b/contentctl/objects/annotated_types.py index 9a7a60a3..f1291af5 100644 --- a/contentctl/objects/annotated_types.py +++ b/contentctl/objects/annotated_types.py @@ -3,4 +3,4 @@ CVE_TYPE = Annotated[str, Field(pattern=r"^CVE-[1|2]\d{3}-\d+$")] MITRE_ATTACK_ID_TYPE = Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})?$")] -APPID_TYPE = Annotated[str,Field(pattern="^[a-zA-Z0-9_-]+$")] \ No newline at end of file +APPID_TYPE = Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")] diff --git a/contentctl/objects/atomic.py b/contentctl/objects/atomic.py index 7e79227c..49ac443b 100644 --- a/contentctl/objects/atomic.py +++ b/contentctl/objects/atomic.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING + if TYPE_CHECKING: from contentctl.objects.config import validate @@ -11,12 +12,13 @@ from enum import StrEnum, auto import uuid -class SupportedPlatform(StrEnum): + +class SupportedPlatform(StrEnum): windows = auto() linux = auto() macos = auto() containers = auto() - # Because the following fields contain special characters + # Because the following fields contain special characters # (which cannot be field names) we must specifiy them manually google_workspace = "google-workspace" iaas_gcp = "iaas:gcp" @@ -24,7 +26,6 @@ class SupportedPlatform(StrEnum): iaas_aws = "iaas:aws" azure_ad = "azure-ad" office_365 = "office-365" - class InputArgumentType(StrEnum): @@ -40,29 +41,33 @@ class InputArgumentType(StrEnum): Path = "Path" Url = "Url" + class AtomicExecutor(BaseModel): model_config = ConfigDict(extra="forbid") name: str - elevation_required: Optional[bool] = False #Appears to be optional + elevation_required: Optional[bool] = False # Appears to be optional command: Optional[str] = None steps: Optional[str] = None cleanup_command: Optional[str] = None - @model_validator(mode='after') - def ensure_mutually_exclusive_fields(self)->Self: + @model_validator(mode="after") + def ensure_mutually_exclusive_fields(self) -> Self: if self.command is not None and self.steps is not None: - raise ValueError("command and steps cannot both be defined in the executor section. Exactly one must be defined.") + raise ValueError( + "command and steps cannot both be defined in the executor section. Exactly one must be defined." + ) elif self.command is None and self.steps is None: - raise ValueError("Neither command nor steps were defined in the executor section. Exactly one must be defined.") + raise ValueError( + "Neither command nor steps were defined in the executor section. Exactly one must be defined." + ) return self - class InputArgument(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") description: str type: InputArgumentType - default: Union[str,int,float,None] = None + default: Union[str, int, float, None] = None class DependencyExecutorType(StrEnum): @@ -71,43 +76,51 @@ class DependencyExecutorType(StrEnum): bash = auto() command_prompt = auto() + class AtomicDependency(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") description: str prereq_command: str get_prereq_command: str + class AtomicTest(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") name: str auto_generated_guid: UUID4 description: str supported_platforms: List[SupportedPlatform] executor: AtomicExecutor - input_arguments: Optional[Dict[str,InputArgument]] = None + input_arguments: Optional[Dict[str, InputArgument]] = None dependencies: Optional[List[AtomicDependency]] = None dependency_executor_name: Optional[DependencyExecutorType] = None @staticmethod def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4) -> AtomicTest: - return AtomicTest(name="Missing Atomic", - auto_generated_guid=auto_generated_guid, - description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile.", - supported_platforms=[], - executor=AtomicExecutor(name="Placeholder Executor (failed to find auto_generated_guid)", - command="Placeholder command (failed to find auto_generated_guid)")) - + return AtomicTest( + name="Missing Atomic", + auto_generated_guid=auto_generated_guid, + description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile.", + supported_platforms=[], + executor=AtomicExecutor( + name="Placeholder Executor (failed to find auto_generated_guid)", + command="Placeholder command (failed to find auto_generated_guid)", + ), + ) + @classmethod - def parseArtRepo(cls, repo_path:pathlib.Path)->dict[uuid.UUID, AtomicTest]: + def parseArtRepo(cls, repo_path: pathlib.Path) -> dict[uuid.UUID, AtomicTest]: test_mapping: dict[uuid.UUID, AtomicTest] = {} - atomics_path = repo_path/"atomics" + atomics_path = repo_path / "atomics" if not atomics_path.is_dir(): - raise FileNotFoundError(f"WARNING: Atomic Red Team repo exists at {repo_path}, " - f"but atomics directory does NOT exist at {atomics_path}. " - "Was it deleted or renamed?") - - atomic_files:List[AtomicFile] = [] - error_messages:List[str] = [] + raise FileNotFoundError( + f"WARNING: Atomic Red Team repo exists at {repo_path}, " + f"but atomics directory does NOT exist at {atomics_path}. " + "Was it deleted or renamed?" + ) + + atomic_files: List[AtomicFile] = [] + error_messages: List[str] = [] for obj_path in atomics_path.glob("**/T*.yaml"): try: atomic_files.append(cls.constructAtomicFile(obj_path)) @@ -115,14 +128,16 @@ def parseArtRepo(cls, repo_path:pathlib.Path)->dict[uuid.UUID, AtomicTest]: error_messages.append(f"File [{obj_path}]\n{str(e)}") if len(error_messages) > 0: - exceptions_string = '\n\n'.join(error_messages) - print(f"WARNING: The following [{len(error_messages)}] ERRORS were generated when parsing the Atomic Red Team Repo.\n" - "Please raise an issue so that they can be fixed at https://github.com/redcanaryco/atomic-red-team/issues.\n" - "Note that this is only a warning and contentctl will ignore Atomics contained in these files.\n" - f"However, if you have written a detection that references them, 'contentctl build --enrichments' will fail:\n\n{exceptions_string}") - + exceptions_string = "\n\n".join(error_messages) + print( + f"WARNING: The following [{len(error_messages)}] ERRORS were generated when parsing the Atomic Red Team Repo.\n" + "Please raise an issue so that they can be fixed at https://github.com/redcanaryco/atomic-red-team/issues.\n" + "Note that this is only a warning and contentctl will ignore Atomics contained in these files.\n" + f"However, if you have written a detection that references them, 'contentctl build --enrichments' will fail:\n\n{exceptions_string}" + ) + # Now iterate over all the files, collect all the tests, and return the dict mapping - redefined_guids:set[uuid.UUID] = set() + redefined_guids: set[uuid.UUID] = set() for atomic_file in atomic_files: for atomic_test in atomic_file.atomic_tests: if atomic_test.auto_generated_guid in test_mapping: @@ -130,23 +145,25 @@ def parseArtRepo(cls, repo_path:pathlib.Path)->dict[uuid.UUID, AtomicTest]: else: test_mapping[atomic_test.auto_generated_guid] = atomic_test if len(redefined_guids) > 0: - guids_string = '\n\t'.join([str(guid) for guid in redefined_guids]) - raise Exception(f"The following [{len(redefined_guids)}] Atomic Test" - " auto_generated_guid(s) were defined more than once. " - f"auto_generated_guids MUST be unique:\n\t{guids_string}") + guids_string = "\n\t".join([str(guid) for guid in redefined_guids]) + raise Exception( + f"The following [{len(redefined_guids)}] Atomic Test" + " auto_generated_guid(s) were defined more than once. " + f"auto_generated_guids MUST be unique:\n\t{guids_string}" + ) print(f"Successfully parsed [{len(test_mapping)}] Atomic Red Team Tests!") return test_mapping - + @classmethod - def constructAtomicFile(cls, file_path:pathlib.Path)->AtomicFile: - yml_dict = YmlReader.load_file(file_path) + def constructAtomicFile(cls, file_path: pathlib.Path) -> AtomicFile: + yml_dict = YmlReader.load_file(file_path) atomic_file = AtomicFile.model_validate(yml_dict) return atomic_file class AtomicFile(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") file_path: FilePath attack_technique: str display_name: str @@ -154,18 +171,18 @@ class AtomicFile(BaseModel): class AtomicEnrichment(BaseModel): - data: dict[uuid.UUID,AtomicTest] = dataclasses.field(default_factory = dict) + data: dict[uuid.UUID, AtomicTest] = dataclasses.field(default_factory=dict) use_enrichment: bool = False @classmethod - def getAtomicEnrichment(cls, config:validate)->AtomicEnrichment: + def getAtomicEnrichment(cls, config: validate) -> AtomicEnrichment: enrichment = AtomicEnrichment(use_enrichment=config.enrichments) if config.enrichments: enrichment.data = AtomicTest.parseArtRepo(config.atomic_red_team_repo_path) return enrichment - def getAtomic(self, atomic_guid: uuid.UUID)->AtomicTest: + def getAtomic(self, atomic_guid: uuid.UUID) -> AtomicTest: if self.use_enrichment: if atomic_guid in self.data: return self.data[atomic_guid] @@ -175,10 +192,3 @@ def getAtomic(self, atomic_guid: uuid.UUID)->AtomicTest: # If enrichment is not enabled, for the sake of compatability # return a stub test with no useful or meaningful information. return AtomicTest.AtomicTestWhenTestIsMissing(atomic_guid) - - - - - - - \ No newline at end of file diff --git a/contentctl/objects/base_test.py b/contentctl/objects/base_test.py index a47ed574..4505b4d3 100644 --- a/contentctl/objects/base_test.py +++ b/contentctl/objects/base_test.py @@ -2,7 +2,7 @@ from typing import Union from abc import ABC, abstractmethod -from pydantic import BaseModel,ConfigDict +from pydantic import BaseModel, ConfigDict from contentctl.objects.base_test_result import BaseTestResult @@ -11,6 +11,7 @@ class TestType(StrEnum): """ Types of tests """ + UNIT = "unit" INTEGRATION = "integration" MANUAL = "manual" diff --git a/contentctl/objects/base_test_result.py b/contentctl/objects/base_test_result.py index 6f9ce11a..c528969a 100644 --- a/contentctl/objects/base_test_result.py +++ b/contentctl/objects/base_test_result.py @@ -2,7 +2,7 @@ from enum import StrEnum from pydantic import ConfigDict, BaseModel -from splunklib.data import Record # type: ignore +from splunklib.data import Record # type: ignore from contentctl.helper.utils import Utils @@ -12,6 +12,7 @@ # type; remove mypy ignores associated w/ these typing issues once we do class TestResultStatus(StrEnum): """Enum for test status (e.g. pass/fail)""" + # Test failed (detection did NOT fire appropriately) FAIL = "fail" @@ -35,6 +36,7 @@ class BaseTestResult(BaseModel): """ Base class for test results """ + # Message for the result message: Union[None, str] = None @@ -54,10 +56,7 @@ class BaseTestResult(BaseModel): sid_link: Union[None, str] = None # Needed to allow for embedding of Exceptions in the model - model_config = ConfigDict( - validate_assignment=True, - arbitrary_types_allowed=True - ) + model_config = ConfigDict(validate_assignment=True, arbitrary_types_allowed=True) @property def passed(self) -> bool: @@ -81,7 +80,10 @@ def failed(self) -> bool: Property returning True if status is FAIL or ERROR; False otherwise (PASS, SKIP) :returns: bool indicating fialure if True """ - return self.status == TestResultStatus.FAIL or self.status == TestResultStatus.ERROR + return ( + self.status == TestResultStatus.FAIL + or self.status == TestResultStatus.ERROR + ) @property def complete(self) -> bool: @@ -94,7 +96,13 @@ def complete(self) -> bool: def get_summary_dict( self, model_fields: list[str] = [ - "success", "exception", "message", "sid_link", "status", "duration", "wait_duration" + "success", + "exception", + "message", + "sid_link", + "status", + "duration", + "wait_duration", ], job_fields: list[str] = ["search", "resultCount", "runDuration"], ) -> dict[str, Any]: @@ -125,7 +133,7 @@ def get_summary_dict( # Grab the job content fields required for field in job_fields: if self.job_content is not None: - value: Any = self.job_content.get(field, None) # type: ignore + value: Any = self.job_content.get(field, None) # type: ignore # convert runDuration to a fixed width string representation of a float if field == "runDuration": diff --git a/contentctl/objects/baseline.py b/contentctl/objects/baseline.py index f66b5b2b..4fe2f1ad 100644 --- a/contentctl/objects/baseline.py +++ b/contentctl/objects/baseline.py @@ -1,10 +1,16 @@ - from __future__ import annotations -from typing import Annotated, List,Any, TYPE_CHECKING +from typing import Annotated, List, Any, TYPE_CHECKING + if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto -from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field +from pydantic import ( + field_validator, + ValidationInfo, + Field, + model_serializer, + computed_field, +) from contentctl.objects.deployment import Deployment from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.enums import DataModel @@ -13,11 +19,15 @@ from contentctl.objects.config import CustomApp from contentctl.objects.lookup import Lookup -from contentctl.objects.constants import CONTENTCTL_MAX_SEARCH_NAME_LENGTH,CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE +from contentctl.objects.constants import ( + CONTENTCTL_MAX_SEARCH_NAME_LENGTH, + CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE, +) + class Baseline(SecurityContentObject): - name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) - type: Annotated[str,Field(pattern="^Baseline$")] = Field(...) + name: str = Field(..., max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) + type: Annotated[str, Field(pattern="^Baseline$")] = Field(...) search: str = Field(..., min_length=4) how_to_implement: str = Field(..., min_length=4) known_false_positives: str = Field(..., min_length=4) @@ -26,30 +36,31 @@ class Baseline(SecurityContentObject): # enrichment deployment: Deployment = Field({}) - - @field_validator('lookups', mode="before") + @field_validator("lookups", mode="before") @classmethod - def getBaselineLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]: - ''' + def getBaselineLookups(cls, v: list[str], info: ValidationInfo) -> list[Lookup]: + """ This function has been copied and renamed from the Detection_Abstract class - ''' - director:DirectorOutputDto = info.context.get("output_dto",None) - search: str | None = info.data.get("search",None) + """ + director: DirectorOutputDto = info.context.get("output_dto", None) + search: str | None = info.data.get("search", None) if search is None: raise ValueError("Search was None - is this file missing the search field?") - + lookups = Lookup.get_lookups(search, director) return lookups - def get_conf_stanza_name(self, app:CustomApp)->str: - stanza_name = CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name) + def get_conf_stanza_name(self, app: CustomApp) -> str: + stanza_name = CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE.format( + app_label=app.label, detection_name=self.name + ) self.check_conf_stanza_max_length(stanza_name) return stanza_name @field_validator("deployment", mode="before") - def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment: - return Deployment.getDeployment(v,info) - + def getDeployment(cls, v: Any, info: ValidationInfo) -> Deployment: + return Deployment.getDeployment(v, info) + @computed_field @property def datamodel(self) -> List[DataModel]: @@ -57,21 +68,21 @@ def datamodel(self) -> List[DataModel]: @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - - #All fields custom to this model - model= { + + # All fields custom to this model + model = { "tags": self.tags.model_dump(), "type": self.type, "search": self.search, - "how_to_implement":self.how_to_implement, - "known_false_positives":self.known_false_positives, + "how_to_implement": self.how_to_implement, + "known_false_positives": self.known_false_positives, "datamodel": self.datamodel, } - - #Combine fields from this model with fields from parent + + # Combine fields from this model with fields from parent super_fields.update(model) - - #return the model - return super_fields \ No newline at end of file + + # return the model + return super_fields diff --git a/contentctl/objects/baseline_tags.py b/contentctl/objects/baseline_tags.py index db5f8048..c8911cc9 100644 --- a/contentctl/objects/baseline_tags.py +++ b/contentctl/objects/baseline_tags.py @@ -1,5 +1,12 @@ from __future__ import annotations -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer, ConfigDict +from pydantic import ( + BaseModel, + Field, + field_validator, + ValidationInfo, + model_serializer, + ConfigDict, +) from typing import List, Any, Union from contentctl.objects.story import Story @@ -8,35 +15,35 @@ from contentctl.objects.enums import SecurityDomain - - - class BaselineTags(BaseModel): model_config = ConfigDict(extra="forbid") analytic_story: list[Story] = Field(...) - #deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION') + # deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION') # TODO (#223): can we remove str from the possible types here? - detections: List[Union[Detection,str]] = Field(...) - product: List[SecurityContentProductName] = Field(...,min_length=1) + detections: List[Union[Detection, str]] = Field(...) + product: List[SecurityContentProductName] = Field(..., min_length=1) security_domain: SecurityDomain = Field(...) + @field_validator("analytic_story", mode="before") + def getStories(cls, v: Any, info: ValidationInfo) -> List[Story]: + return Story.mapNamesToSecurityContentObjects( + v, info.context.get("output_dto", None) + ) - @field_validator("analytic_story",mode="before") - def getStories(cls, v:Any, info:ValidationInfo)->List[Story]: - return Story.mapNamesToSecurityContentObjects(v, info.context.get("output_dto",None)) - - @model_serializer - def serialize_model(self): - #All fields custom to this model - model= { + def serialize_model(self): + # All fields custom to this model + model = { "analytic_story": [story.name for story in self.analytic_story], - "detections": [detection.name for detection in self.detections if isinstance(detection,Detection)], + "detections": [ + detection.name + for detection in self.detections + if isinstance(detection, Detection) + ], "product": self.product, - "security_domain":self.security_domain, - "deployments": None + "security_domain": self.security_domain, + "deployments": None, } - - - #return the model - return model \ No newline at end of file + + # return the model + return model diff --git a/contentctl/objects/constants.py b/contentctl/objects/constants.py index f0f0d8f3..c20184da 100644 --- a/contentctl/objects/constants.py +++ b/contentctl/objects/constants.py @@ -15,7 +15,7 @@ "Collection": "Exploitation", "Command And Control": "Command and Control", "Exfiltration": "Actions on Objectives", - "Impact": "Actions on Objectives" + "Impact": "Actions on Objectives", } SES_CONTEXT_MAPPING = { @@ -65,7 +65,7 @@ "Other:Policy Violation": 82, "Other:Threat Intelligence": 83, "Other:Flight Risk": 84, - "Other:Removable Storage": 85 + "Other:Removable Storage": 85, } SES_KILL_CHAIN_MAPPINGS = { @@ -76,7 +76,7 @@ "Exploitation": 4, "Installation": 5, "Command and Control": 6, - "Actions on Objectives": 7 + "Actions on Objectives": 7, } # TODO (cmcginley): @ljstella should this be removed? also referenced in new_content.py @@ -91,7 +91,7 @@ "Child Process": 6, "Known Bad": 7, "Data Loss": 8, - "Observer": 9 + "Observer": 9, } # TODO (cmcginley): @ljstella should this be removed? also referenced in new_content.py @@ -117,7 +117,7 @@ "Container": 27, "Registry Key": 28, "Registry Value": 29, - "Other": 99 + "Other": 99, } SES_ATTACK_TACTICS_ID_MAPPING = { @@ -134,24 +134,21 @@ "Collection": "TA0009", "Command_and_Control": "TA0011", "Exfiltration": "TA0010", - "Impact": "TA0040" + "Impact": "TA0040", } # TODO (cmcginley): is this just for the transition testing? -RBA_OBSERVABLE_ROLE_MAPPING = { - "Attacker": 0, - "Victim": 1 -} +RBA_OBSERVABLE_ROLE_MAPPING = {"Attacker": 0, "Victim": 1} # The relative path to the directory where any apps/packages will be downloaded DOWNLOADS_DIRECTORY = "downloads" # Maximum length of the name field for a search. -# This number is derived from a limitation that exists in +# This number is derived from a limitation that exists in # ESCU where a search cannot be edited, due to validation # errors, if its name is longer than 99 characters. # When an saved search is cloned in Enterprise Security User Interface, -# it is wrapped in the following: +# it is wrapped in the following: # {Detection.tags.security_domain} - {SEARCH_STANZA_NAME} - Rule # Similarly, when we generate the search stanza name in contentctl, it # is app.label - detection.name - Rule @@ -160,16 +157,32 @@ # or in ESCU: # ESCU - {detection.name} - Rule, # this gives us a maximum length below. -# When an ESCU search is cloned, it will +# When an ESCU search is cloned, it will # have a full name like (the following is NOT a typo): # Endpoint - ESCU - Name of Search From YML File - Rule - Rule # The math below accounts for all these caveats ES_MAX_STANZA_LENGTH = 99 -CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Rule" +CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE = ( + "{app_label} - {detection_name} - Rule" +) CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name}" -CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Response Task" +CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE = ( + "{app_label} - {detection_name} - Response Task" +) -ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE = "{security_domain_value} - {search_name} - Rule" -SECURITY_DOMAIN_MAX_LENGTH = max([len(SecurityDomain[value]) for value in SecurityDomain._member_map_]) -CONTENTCTL_MAX_STANZA_LENGTH = ES_MAX_STANZA_LENGTH - len(ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format(security_domain_value="X"*SECURITY_DOMAIN_MAX_LENGTH,search_name="")) -CONTENTCTL_MAX_SEARCH_NAME_LENGTH = CONTENTCTL_MAX_STANZA_LENGTH - len(CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label="ESCU", detection_name="")) \ No newline at end of file +ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE = ( + "{security_domain_value} - {search_name} - Rule" +) +SECURITY_DOMAIN_MAX_LENGTH = max( + [len(SecurityDomain[value]) for value in SecurityDomain._member_map_] +) +CONTENTCTL_MAX_STANZA_LENGTH = ES_MAX_STANZA_LENGTH - len( + ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format( + security_domain_value="X" * SECURITY_DOMAIN_MAX_LENGTH, search_name="" + ) +) +CONTENTCTL_MAX_SEARCH_NAME_LENGTH = CONTENTCTL_MAX_STANZA_LENGTH - len( + CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format( + app_label="ESCU", detection_name="" + ) +) diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 9ce66a76..0cebf5cc 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -6,25 +6,25 @@ from functools import cached_property from pydantic import ConfigDict, BaseModel, computed_field, Field, PrivateAttr -from splunklib.results import JSONResultsReader, Message # type: ignore -from splunklib.binding import HTTPError, ResponseReader # type: ignore -import splunklib.client as splunklib # type: ignore -from tqdm import tqdm # type: ignore +from splunklib.results import JSONResultsReader, Message # type: ignore +from splunklib.binding import HTTPError, ResponseReader # type: ignore +import splunklib.client as splunklib # type: ignore +from tqdm import tqdm # type: ignore from contentctl.objects.risk_analysis_action import RiskAnalysisAction from contentctl.objects.notable_action import NotableAction from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.integration_test_result import IntegrationTestResult from contentctl.actions.detection_testing.progress_bar import ( - format_pbar_string, # type: ignore + format_pbar_string, # type: ignore TestReportingType, - TestingStates + TestingStates, ) from contentctl.objects.errors import ( IntegrationTestingError, ServerError, ClientError, - ValidationFailed + ValidationFailed, ) from contentctl.objects.detection import Detection from contentctl.objects.risk_event import RiskEvent @@ -65,7 +65,9 @@ def get_logger() -> logging.Logger: handler = logging.NullHandler() # Format our output - formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(levelname)s:%(name)s - %(message)s" + ) handler.setFormatter(formatter) # Set handler level and add to logger @@ -79,6 +81,7 @@ class SavedSearchKeys(StrEnum): """ Various keys into the SavedSearch content """ + # setup the names of the keys we expect to access in content EARLIEST_TIME_KEY = "dispatch.earliest_time" LATEST_TIME_KEY = "dispatch.latest_time" @@ -92,6 +95,7 @@ class Indexes(StrEnum): """ Indexes we search against """ + # setup the names of the risk and notable indexes RISK_INDEX = "risk" NOTABLE_INDEX = "notable" @@ -101,6 +105,7 @@ class TimeoutConfig(IntEnum): """ Configuration values for the exponential backoff timer """ + # base amount to sleep for before beginning exponential backoff during testing BASE_SLEEP = 60 @@ -118,6 +123,7 @@ class ScheduleConfig(StrEnum): """ Configuraton values for the saved search schedule """ + EARLIEST_TIME = "-5y@y" LATEST_TIME = "-1m@m" CRON_SCHEDULE = "*/1 * * * *" @@ -132,11 +138,10 @@ class ResultIterator: :param response_reader: a ResponseReader object :param logger: a Logger object """ + def __init__(self, response_reader: ResponseReader) -> None: # init the results reader - self.results_reader: JSONResultsReader = JSONResultsReader( - response_reader - ) + self.results_reader: JSONResultsReader = JSONResultsReader(response_reader) # get logger self.logger: logging.Logger = get_logger() @@ -150,18 +155,18 @@ def __next__(self) -> dict[Any, Any]: # log messages, or raise if error if isinstance(result, Message): # convert level string to level int - level_name = result.type.strip().upper() # type: ignore + level_name = result.type.strip().upper() # type: ignore level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed - message = f"SPLUNK: {result.message}" # type: ignore + message = f"SPLUNK: {result.message}" # type: ignore self.logger.log(level, message) if level == logging.ERROR: raise ServerError(message) # if dict, just return elif isinstance(result, dict): - return result # type: ignore + return result # type: ignore # raise for any unexpected types else: @@ -178,14 +183,13 @@ class PbarData(BaseModel): :param fq_test_name: the fully qualifed (fq) test name (":") used for logging :param start_time: the start time used for logging """ - pbar: tqdm # type: ignore + + pbar: tqdm # type: ignore fq_test_name: str start_time: float # needed to support the tqdm type - model_config = ConfigDict( - arbitrary_types_allowed=True - ) + model_config = ConfigDict(arbitrary_types_allowed=True) class CorrelationSearch(BaseModel): @@ -198,6 +202,7 @@ class CorrelationSearch(BaseModel): :param pbar_data: the encapsulated info needed for logging w/ pbar :param test_index: the index attack data is forwarded to for testing (optionally used in cleanup) """ + # the detection associated with the correlation search (e.g. "Windows Modify Registry EnableLinkedConnections") detection: Detection = Field(...) @@ -232,10 +237,7 @@ class CorrelationSearch(BaseModel): # Need arbitrary types to allow fields w/ types like SavedSearch; we also want to forbid # unexpected fields - model_config = ConfigDict( - arbitrary_types_allowed=True, - extra='forbid' - ) + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") def model_post_init(self, __context: Any) -> None: super().model_post_init(__context) @@ -309,7 +311,7 @@ def earliest_time(self) -> str: The earliest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] # type: ignore + return self.saved_search.content[SavedSearchKeys.EARLIEST_TIME_KEY] # type: ignore else: raise ClientError( "Something unexpected went wrong in initialization; saved_search was not populated" @@ -321,7 +323,7 @@ def latest_time(self) -> str: The latest time configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] # type: ignore + return self.saved_search.content[SavedSearchKeys.LATEST_TIME_KEY] # type: ignore else: raise ClientError( "Something unexpected went wrong in initialization; saved_search was not populated" @@ -333,7 +335,7 @@ def cron_schedule(self) -> str: The cron schedule configured for the saved search """ if self.saved_search is not None: - return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] # type: ignore + return self.saved_search.content[SavedSearchKeys.CRON_SCHEDULE_KEY] # type: ignore else: raise ClientError( "Something unexpected went wrong in initialization; saved_search was not populated" @@ -345,7 +347,7 @@ def enabled(self) -> bool: Whether the saved search is enabled """ if self.saved_search is not None: - if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): # type: ignore + if int(self.saved_search.content[SavedSearchKeys.DISBALED_KEY]): # type: ignore return False else: return True @@ -354,7 +356,7 @@ def enabled(self) -> bool: "Something unexpected went wrong in initialization; saved_search was not populated" ) - @ property + @property def has_risk_analysis_action(self) -> bool: """Whether the correlation search has an associated risk analysis Adaptive Response Action :return: a boolean indicating whether it has a risk analysis Adaptive Response Action @@ -405,11 +407,13 @@ def _parse_risk_and_notable_actions(self) -> None: """ # grab risk details if present self._risk_analysis_action = CorrelationSearch._get_risk_analysis_action( - self.saved_search.content # type: ignore + self.saved_search.content # type: ignore ) # grab notable details if present - self._notable_action = CorrelationSearch._get_notable_action(self.saved_search.content) # type: ignore + self._notable_action = CorrelationSearch._get_notable_action( + self.saved_search.content + ) # type: ignore def refresh(self) -> None: """Refreshes the metadata in the SavedSearch entity, and re-parses the fields we care about @@ -417,10 +421,9 @@ def refresh(self) -> None: After operations we expect to alter the state of the SavedSearch, we call refresh so that we have a local representation of the new state; then we extrat what we care about into this instance """ - self.logger.debug( - f"Refreshing SavedSearch metadata for {self.name}...") + self.logger.debug(f"Refreshing SavedSearch metadata for {self.name}...") try: - self.saved_search.refresh() # type: ignore + self.saved_search.refresh() # type: ignore except HTTPError as e: raise ServerError(f"HTTP error encountered during refresh: {e}") self._parse_risk_and_notable_actions() @@ -434,7 +437,7 @@ def enable(self, refresh: bool = True) -> None: """ self.logger.debug(f"Enabling {self.name}...") try: - self.saved_search.enable() # type: ignore + self.saved_search.enable() # type: ignore except HTTPError as e: raise ServerError(f"HTTP error encountered while enabling detection: {e}") if refresh: @@ -449,7 +452,7 @@ def disable(self, refresh: bool = True) -> None: """ self.logger.debug(f"Disabling {self.name}...") try: - self.saved_search.disable() # type: ignore + self.saved_search.disable() # type: ignore except HTTPError as e: raise ServerError(f"HTTP error encountered while disabling detection: {e}") if refresh: @@ -460,7 +463,7 @@ def update_timeframe( earliest_time: str = ScheduleConfig.EARLIEST_TIME, latest_time: str = ScheduleConfig.LATEST_TIME, cron_schedule: str = ScheduleConfig.CRON_SCHEDULE, - refresh: bool = True + refresh: bool = True, ) -> None: """Updates the correlation search timeframe to work with test data @@ -477,12 +480,12 @@ def update_timeframe( data = { SavedSearchKeys.EARLIEST_TIME_KEY: earliest_time, SavedSearchKeys.LATEST_TIME_KEY: latest_time, - SavedSearchKeys.CRON_SCHEDULE_KEY: cron_schedule + SavedSearchKeys.CRON_SCHEDULE_KEY: cron_schedule, } self.logger.info(data) self.logger.info(f"Updating timeframe for '{self.name}': {data}") try: - self.saved_search.update(**data) # type: ignore + self.saved_search.update(**data) # type: ignore except HTTPError as e: raise ServerError(f"HTTP error encountered while updating timeframe: {e}") @@ -531,7 +534,9 @@ def get_risk_events(self, force_update: bool = False) -> list[RiskEvent]: # Use the cached risk_events unless we're forcing an update if self._risk_events is not None: - self.logger.debug(f"Using cached risk events ({len(self._risk_events)} total).") + self.logger.debug( + f"Using cached risk events ({len(self._risk_events)} total)." + ) return self._risk_events # TODO (#248): Refactor risk/notable querying to pin to a single savedsearch ID @@ -553,7 +558,9 @@ def get_risk_events(self, force_update: bool = False) -> list[RiskEvent]: parsed_raw = json.loads(result["_raw"]) event = RiskEvent.model_validate(parsed_raw) except Exception: - self.logger.error(f"Failed to parse RiskEvent from search result: {result}") + self.logger.error( + f"Failed to parse RiskEvent from search result: {result}" + ) raise events.append(event) self.logger.debug(f"Found risk event for '{self.name}': {event}") @@ -597,7 +604,9 @@ def get_notable_events(self, force_update: bool = False) -> list[NotableEvent]: # Use the cached notable_events unless we're forcing an update if self._notable_events is not None: - self.logger.debug(f"Using cached notable events ({len(self._notable_events)} total).") + self.logger.debug( + f"Using cached notable events ({len(self._notable_events)} total)." + ) return self._notable_events # Search for all notable events from a single scheduled search (indicated by orig_sid) @@ -618,7 +627,9 @@ def get_notable_events(self, force_update: bool = False) -> list[NotableEvent]: parsed_raw = json.loads(result["_raw"]) event = NotableEvent.model_validate(parsed_raw) except Exception: - self.logger.error(f"Failed to parse NotableEvent from search result: {result}") + self.logger.error( + f"Failed to parse NotableEvent from search result: {result}" + ) raise events.append(event) self.logger.debug(f"Found notable event for '{self.name}': {event}") @@ -653,7 +664,9 @@ def validate_risk_events(self) -> None: " with it; cannot validate." ) - risk_object_counts: dict[int, int] = {id(x): 0 for x in self.detection.rba.risk_objects} + risk_object_counts: dict[int, int] = { + id(x): 0 for x in self.detection.rba.risk_objects + } # Get the risk events; note that we use the cached risk events, expecting they were # saved by a prior call to risk_event_exists @@ -670,7 +683,9 @@ def validate_risk_events(self) -> None: event.validate_against_detection(self.detection) # Update risk object count based on match - matched_risk_object = event.get_matched_risk_object(self.detection.rba.risk_objects) + matched_risk_object = event.get_matched_risk_object( + self.detection.rba.risk_objects + ) self.logger.debug( f"Matched risk event (object={event.es_risk_object}, type={event.es_risk_object_type}) " f"to detection's risk object (name={matched_risk_object.field}, " @@ -740,7 +755,9 @@ def validate_notable_events(self) -> None: # NOTE: it would be more ideal to switch this to a system which gets the handle of the saved search job and polls # it for completion, but that seems more tricky - def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = False) -> IntegrationTestResult: + def test( + self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = False + ) -> IntegrationTestResult: """Execute the integration test Executes an integration test for this CorrelationSearch. First, ensures no matching risk/notables already exist @@ -772,9 +789,7 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = Fa try: # first make sure the indexes are currently empty and the detection is starting from a disabled state - self.logger.debug( - "Cleaning up any pre-existing risk/notable events..." - ) + self.logger.debug("Cleaning up any pre-existing risk/notable events...") self.update_pbar(TestingStates.PRE_CLEANUP) if self.risk_event_exists(): self.logger.warning( @@ -806,7 +821,9 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = Fa # loop so long as the elapsed time is less than max_sleep while elapsed_sleep_time < max_sleep: # sleep so the detection job can finish - self.logger.info(f"Waiting {time_to_sleep} for {self.name} so it can finish") + self.logger.info( + f"Waiting {time_to_sleep} for {self.name} so it can finish" + ) self.update_pbar(TestingStates.VALIDATING) time.sleep(time_to_sleep) elapsed_sleep_time += time_to_sleep @@ -895,7 +912,7 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = Fa wait_duration=elapsed_sleep_time, exception=e, ) - self.logger.exception(result.message) # type: ignore + self.logger.exception(result.message) # type: ignore else: raise e except Exception as e: @@ -905,7 +922,10 @@ def test(self, max_sleep: int = TimeoutConfig.MAX_SLEEP, raise_on_exc: bool = Fa # log based on result status if result is not None: - if result.status == TestResultStatus.PASS or result.status == TestResultStatus.SKIP: + if ( + result.status == TestResultStatus.PASS + or result.status == TestResultStatus.SKIP + ): self.logger.info(f"{result.status.name}: {result.message}") elif result.status == TestResultStatus.FAIL: self.logger.error(f"{result.status.name}: {result.message}") @@ -928,11 +948,11 @@ def _search(self, query: str) -> ResultIterator: :param query: the SPL string to run """ self.logger.debug(f"Executing query: `{query}`") - job = self.service.search(query, exec_mode="blocking") # type: ignore + job = self.service.search(query, exec_mode="blocking") # type: ignore # query the results, catching any HTTP status code errors try: - response_reader: ResponseReader = job.results(output_mode="json") # type: ignore + response_reader: ResponseReader = job.results(output_mode="json") # type: ignore except HTTPError as e: # e.g. -> HTTP 400 Bad Request -- b'{"messages":[{"type":"FATAL","text":"Error in \'delete\' command: You # have insufficient privileges to delete events."}]}' @@ -940,7 +960,7 @@ def _search(self, query: str) -> ResultIterator: self.logger.error(message) raise ServerError(message) - return ResultIterator(response_reader) # type: ignore + return ResultIterator(response_reader) # type: ignore def _delete_index(self, index: str) -> None: """Deletes events in a given index @@ -991,7 +1011,7 @@ def cleanup(self, delete_test_index: bool = False) -> None: # Add indexes to purge if delete_test_index: - self.indexes_to_purge.add(self.test_index) # type: ignore + self.indexes_to_purge.add(self.test_index) # type: ignore if self._risk_events is not None: self.indexes_to_purge.add(Indexes.RISK_INDEX) if self._notable_events is not None: @@ -1019,5 +1039,5 @@ def update_pbar(self, state: str) -> str: self.pbar_data.fq_test_name, state, self.pbar_data.start_time, - True + True, ) diff --git a/contentctl/objects/dashboard.py b/contentctl/objects/dashboard.py index 90bfc93f..d6ac49fa 100644 --- a/contentctl/objects/dashboard.py +++ b/contentctl/objects/dashboard.py @@ -8,7 +8,7 @@ from contentctl.objects.config import build from enum import StrEnum -DEFAULT_DASHBAORD_JINJA2_TEMPLATE = ''' +DEFAULT_DASHBAORD_JINJA2_TEMPLATE = """ -''' +""" + class DashboardTheme(StrEnum): light = "light" dark = "dark" + class Dashboard(SecurityContentObject): - j2_template: str = Field(default=DEFAULT_DASHBAORD_JINJA2_TEMPLATE, description="Jinja2 Template used to construct the dashboard") - description: str = Field(...,description="A description of the dashboard. This does not have to match " - "the description of the dashboard in the JSON file.", max_length=10000) - theme: DashboardTheme = Field(default=DashboardTheme.light, description="The theme of the dashboard. Choose between 'light' and 'dark'.") - json_obj: Json[dict[str,Any]] = Field(..., description="Valid JSON object that describes the dashboard") - - - - def label(self, config:build)->str: + j2_template: str = Field( + default=DEFAULT_DASHBAORD_JINJA2_TEMPLATE, + description="Jinja2 Template used to construct the dashboard", + ) + description: str = Field( + ..., + description="A description of the dashboard. This does not have to match " + "the description of the dashboard in the JSON file.", + max_length=10000, + ) + theme: DashboardTheme = Field( + default=DashboardTheme.light, + description="The theme of the dashboard. Choose between 'light' and 'dark'.", + ) + json_obj: Json[dict[str, Any]] = Field( + ..., description="Valid JSON object that describes the dashboard" + ) + + def label(self, config: build) -> str: return f"{config.app.label} - {self.name}" - + @model_validator(mode="before") @classmethod - def validate_fields_from_json(cls, data:Any)->Any: - yml_file_name:str|None = data.get("file_path", None) + def validate_fields_from_json(cls, data: Any) -> Any: + yml_file_name: str | None = data.get("file_path", None) if yml_file_name is None: raise ValueError("File name not passed to dashboard constructor") yml_file_path = pathlib.Path(yml_file_name) @@ -50,51 +62,53 @@ def validate_fields_from_json(cls, data:Any)->Any: if not json_file_path.is_file(): raise ValueError(f"Required file {json_file_path} does not exist.") - - with open(json_file_path,'r') as jsonFilePointer: + + with open(json_file_path, "r") as jsonFilePointer: try: - json_obj:dict[str,Any] = json.load(jsonFilePointer) + json_obj: dict[str, Any] = json.load(jsonFilePointer) except Exception as e: raise ValueError(f"Unable to load data from {json_file_path}: {str(e)}") - name_from_file = data.get("name",None) - name_from_json = json_obj.get("title",None) + name_from_file = data.get("name", None) + name_from_json = json_obj.get("title", None) - errors:list[str] = [] + errors: list[str] = [] if name_from_json is None: errors.append(f"'title' field is missing from {json_file_path}") elif name_from_json != name_from_file: - errors.append(f"The 'title' field in the JSON file [{json_file_path}] does not match the 'name' field in the YML object [{yml_file_path}]. These two MUST match:\n " - f"title in JSON : {name_from_json}\n " - f"title in YML : {name_from_file}\n ") - - description_from_json = json_obj.get("description",None) + errors.append( + f"The 'title' field in the JSON file [{json_file_path}] does not match the 'name' field in the YML object [{yml_file_path}]. These two MUST match:\n " + f"title in JSON : {name_from_json}\n " + f"title in YML : {name_from_file}\n " + ) + + description_from_json = json_obj.get("description", None) if description_from_json is None: errors.append("'description' field is missing from field 'json_object'") - - if len(errors) > 0 : + + if len(errors) > 0: err_string = "\n - ".join(errors) raise ValueError(f"Error(s) validating dashboard:\n - {err_string}") - - data['name'] = name_from_file - data['json_obj'] = json.dumps(json_obj) + + data["name"] = name_from_file + data["json_obj"] = json.dumps(json_obj) return data - def pretty_print_json_obj(self): return json.dumps(self.json_obj, indent=4) - - def getOutputFilepathRelativeToAppRoot(self, config:build)->pathlib.Path: + + def getOutputFilepathRelativeToAppRoot(self, config: build) -> pathlib.Path: filename = f"{self.file_path.stem}.xml".lower() - return pathlib.Path("default/data/ui/views")/filename - - - def writeDashboardFile(self, j2_env:Environment, config:build): + return pathlib.Path("default/data/ui/views") / filename + + def writeDashboardFile(self, j2_env: Environment, config: build): template = j2_env.from_string(self.j2_template) dashboard_text = template.render(config=config, dashboard=self) - with open(config.getPackageDirectoryPath()/self.getOutputFilepathRelativeToAppRoot(config), 'a') as f: - output_xml = dashboard_text.encode('utf-8', 'ignore').decode('utf-8') + with open( + config.getPackageDirectoryPath() + / self.getOutputFilepathRelativeToAppRoot(config), + "a", + ) as f: + output_xml = dashboard_text.encode("utf-8", "ignore").decode("utf-8") f.write(output_xml) - - diff --git a/contentctl/objects/data_source.py b/contentctl/objects/data_source.py index 920a19e2..ea2e4bb0 100644 --- a/contentctl/objects/data_source.py +++ b/contentctl/objects/data_source.py @@ -8,6 +8,8 @@ class TA(BaseModel): name: str url: HttpUrl | None = None version: str + + class DataSource(SecurityContentObject): source: str = Field(...) sourcetype: str = Field(...) @@ -19,14 +21,13 @@ class DataSource(SecurityContentObject): convert_to_log_source: None | list = None example_log: None | str = None - @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - - #All fields custom to this model - model:dict[str,Any] = { + + # All fields custom to this model + model: dict[str, Any] = { "source": self.source, "sourcetype": self.sourcetype, "separator": self.separator, @@ -35,12 +36,11 @@ def serialize_model(self): "fields": self.fields, "field_mappings": self.field_mappings, "convert_to_log_source": self.convert_to_log_source, - "example_log":self.example_log + "example_log": self.example_log, } - - - #Combine fields from this model with fields from parent + + # Combine fields from this model with fields from parent super_fields.update(model) - - #return the model - return super_fields \ No newline at end of file + + # return the model + return super_fields diff --git a/contentctl/objects/deployment.py b/contentctl/objects/deployment.py index d268670f..83e7422c 100644 --- a/contentctl/objects/deployment.py +++ b/contentctl/objects/deployment.py @@ -1,5 +1,11 @@ from __future__ import annotations -from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt +from pydantic import ( + Field, + computed_field, + ValidationInfo, + model_serializer, + NonNegativeInt, +) from typing import Any import uuid import datetime @@ -10,68 +16,69 @@ from contentctl.objects.enums import DeploymentType -class Deployment(SecurityContentObject): +class Deployment(SecurityContentObject): scheduling: DeploymentScheduling = Field(...) alert_action: AlertAction = AlertAction() type: DeploymentType = Field(...) - author: str = Field(...,max_length=255) + author: str = Field(..., max_length=255) version: NonNegativeInt = 1 - #Type was the only tag exposed and should likely be removed/refactored. - #For transitional reasons, provide this as a computed_field in prep for removal + # Type was the only tag exposed and should likely be removed/refactored. + # For transitional reasons, provide this as a computed_field in prep for removal @computed_field @property - def tags(self)->dict[str,DeploymentType]: + def tags(self) -> dict[str, DeploymentType]: return {"type": self.type} - @staticmethod - def getDeployment(v:dict[str,Any], info:ValidationInfo)->Deployment: + def getDeployment(v: dict[str, Any], info: ValidationInfo) -> Deployment: if v != {}: # If the user has defined a deployment, then allow it to be validated # and override the default deployment info defined in type:Baseline - v['type'] = DeploymentType.Embedded - + v["type"] = DeploymentType.Embedded + detection_name = info.data.get("name", None) if detection_name is None: - raise ValueError("Could not create inline deployment - Baseline or Detection lacking 'name' field,") + raise ValueError( + "Could not create inline deployment - Baseline or Detection lacking 'name' field," + ) - # Add a number of static values - v.update({ - 'name': f"{detection_name} - Inline Deployment", - 'id':uuid.uuid4(), - 'date': datetime.date.today(), - 'description': "Inline deployment created at runtime.", - 'author': "contentctl tool" - }) + # Add a number of static values + v.update( + { + "name": f"{detection_name} - Inline Deployment", + "id": uuid.uuid4(), + "date": datetime.date.today(), + "description": "Inline deployment created at runtime.", + "author": "contentctl tool", + } + ) - # This constructs a temporary in-memory deployment, - # allowing the deployment to be easily defined in the + # allowing the deployment to be easily defined in the # detection on a per detection basis. return Deployment.model_validate(v) - + else: - return SecurityContentObject.getDeploymentFromType(info.data.get("type",None), info) - + return SecurityContentObject.getDeploymentFromType( + info.data.get("type", None), info + ) + @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - - #All fields custom to this model - model= { - "scheduling": self.scheduling.model_dump(), - "tags": self.tags - } - #Combine fields from this model with fields from parent + # All fields custom to this model + model = {"scheduling": self.scheduling.model_dump(), "tags": self.tags} + + # Combine fields from this model with fields from parent model.update(super_fields) - + alert_action_fields = self.alert_action.model_dump() model.update(alert_action_fields) - del(model['references']) - - #return the model - return model \ No newline at end of file + del model["references"] + + # return the model + return model diff --git a/contentctl/objects/deployment_email.py b/contentctl/objects/deployment_email.py index 1d1269fe..a4f829f0 100644 --- a/contentctl/objects/deployment_email.py +++ b/contentctl/objects/deployment_email.py @@ -6,4 +6,4 @@ class DeploymentEmail(BaseModel): model_config = ConfigDict(extra="forbid") message: str subject: str - to: str \ No newline at end of file + to: str diff --git a/contentctl/objects/deployment_notable.py b/contentctl/objects/deployment_notable.py index 7f064b43..8ac77568 100644 --- a/contentctl/objects/deployment_notable.py +++ b/contentctl/objects/deployment_notable.py @@ -2,8 +2,9 @@ from pydantic import BaseModel, ConfigDict from typing import List + class DeploymentNotable(BaseModel): model_config = ConfigDict(extra="forbid") rule_description: str rule_title: str - nes_fields: List[str] \ No newline at end of file + nes_fields: List[str] diff --git a/contentctl/objects/deployment_phantom.py b/contentctl/objects/deployment_phantom.py index 1d4a9975..8bd96516 100644 --- a/contentctl/objects/deployment_phantom.py +++ b/contentctl/objects/deployment_phantom.py @@ -4,8 +4,8 @@ class DeploymentPhantom(BaseModel): model_config = ConfigDict(extra="forbid") - cam_workers : str - label : str - phantom_server : str - sensitivity : str - severity : str \ No newline at end of file + cam_workers: str + label: str + phantom_server: str + sensitivity: str + severity: str diff --git a/contentctl/objects/deployment_rba.py b/contentctl/objects/deployment_rba.py index 58917c70..53df26f4 100644 --- a/contentctl/objects/deployment_rba.py +++ b/contentctl/objects/deployment_rba.py @@ -4,4 +4,4 @@ class DeploymentRBA(BaseModel): model_config = ConfigDict(extra="forbid") - enabled: bool = False \ No newline at end of file + enabled: bool = False diff --git a/contentctl/objects/deployment_scheduling.py b/contentctl/objects/deployment_scheduling.py index b21673d8..177fbb30 100644 --- a/contentctl/objects/deployment_scheduling.py +++ b/contentctl/objects/deployment_scheduling.py @@ -7,4 +7,4 @@ class DeploymentScheduling(BaseModel): cron_schedule: str earliest_time: str latest_time: str - schedule_window: str \ No newline at end of file + schedule_window: str diff --git a/contentctl/objects/deployment_slack.py b/contentctl/objects/deployment_slack.py index 03cf5ebb..63f99767 100644 --- a/contentctl/objects/deployment_slack.py +++ b/contentctl/objects/deployment_slack.py @@ -5,4 +5,4 @@ class DeploymentSlack(BaseModel): model_config = ConfigDict(extra="forbid") channel: str - message: str \ No newline at end of file + message: str diff --git a/contentctl/objects/detection.py b/contentctl/objects/detection.py index d418b520..d6c1208b 100644 --- a/contentctl/objects/detection.py +++ b/contentctl/objects/detection.py @@ -1,5 +1,8 @@ from __future__ import annotations -from contentctl.objects.abstract_security_content_objects.detection_abstract import Detection_Abstract +from contentctl.objects.abstract_security_content_objects.detection_abstract import ( + Detection_Abstract, +) + class Detection(Detection_Abstract): # Customization to the Detection Class go here. @@ -12,4 +15,4 @@ class Detection(Detection_Abstract): # them or modifying their behavior may cause # undefined issues with the contentctl tooling # or output of the tooling. - pass \ No newline at end of file + pass diff --git a/contentctl/objects/detection_metadata.py b/contentctl/objects/detection_metadata.py index 46f07e78..3c8c0484 100644 --- a/contentctl/objects/detection_metadata.py +++ b/contentctl/objects/detection_metadata.py @@ -8,6 +8,7 @@ class DetectionMetadata(BaseModel): """ A model of the metadata line in a detection stanza in savedsearches.conf """ + # A bool indicating whether the detection is deprecated (serialized as an int, 1 or 0) deprecated: bool = Field(...) diff --git a/contentctl/objects/detection_stanza.py b/contentctl/objects/detection_stanza.py index 88f9c350..8d4b545d 100644 --- a/contentctl/objects/detection_stanza.py +++ b/contentctl/objects/detection_stanza.py @@ -11,6 +11,7 @@ class DetectionStanza(BaseModel): """ A model representing a stanza for a detection in savedsearches.conf """ + # The lines that comprise this stanza, in the order they appear in the conf lines: list[str] = Field(...) @@ -47,7 +48,9 @@ def metadata(self) -> DetectionMetadata: raise Exception(f"No metadata for detection '{self.name}' found in stanza.") # Parse the metadata JSON into a model - return DetectionMetadata.model_validate_json(meta_line[len(DetectionStanza.METADATA_LINE_PREFIX):]) + return DetectionMetadata.model_validate_json( + meta_line[len(DetectionStanza.METADATA_LINE_PREFIX) :] + ) @computed_field @cached_property @@ -76,4 +79,6 @@ def version_should_be_bumped(self, previous: "DetectionStanza") -> bool: :returns: True if the version still needs to be bumped :rtype: bool """ - return (self.hash != previous.hash) and (self.metadata.detection_version <= previous.metadata.detection_version) + return (self.hash != previous.hash) and ( + self.metadata.detection_version <= previous.metadata.detection_version + ) diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py index aea02bfe..17980060 100644 --- a/contentctl/objects/detection_tags.py +++ b/contentctl/objects/detection_tags.py @@ -11,10 +11,11 @@ field_validator, ValidationInfo, model_serializer, - model_validator + model_validator, ) from contentctl.objects.story import Story from contentctl.objects.throttling import Throttling + if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto @@ -27,7 +28,7 @@ SecurityDomain, KillChainPhase, NistCategory, - SecurityContentProductName + SecurityContentProductName, ) from contentctl.objects.atomic import AtomicEnrichment, AtomicTest from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE @@ -36,7 +37,7 @@ class DetectionTags(BaseModel): # detection spec - model_config = ConfigDict(validate_default=False, extra='forbid') + model_config = ConfigDict(validate_default=False, extra="forbid") analytic_story: list[Story] = Field(...) asset_type: AssetType = Field(...) group: list[str] = [] @@ -54,7 +55,9 @@ class DetectionTags(BaseModel): atomic_guid: List[AtomicTest] = [] # enrichment - mitre_attack_enrichments: List[MitreAttackEnrichment] = Field([], validate_default=True) + mitre_attack_enrichments: List[MitreAttackEnrichment] = Field( + [], validate_default=True + ) @computed_field @property @@ -127,7 +130,7 @@ def serialize_model(self): "nist": self.nist, "security_domain": self.security_domain, "mitre_attack_id": self.mitre_attack_id, - "mitre_attack_enrichments": self.mitre_attack_enrichments + "mitre_attack_enrichments": self.mitre_attack_enrichments, } @model_validator(mode="after") @@ -141,9 +144,13 @@ def addAttackEnrichment(self, info: ValidationInfo): f" at runtime. Instead, this field contained: {self.mitre_attack_enrichments}" ) - output_dto: Union[DirectorOutputDto, None] = info.context.get("output_dto", None) + output_dto: Union[DirectorOutputDto, None] = info.context.get( + "output_dto", None + ) if output_dto is None: - raise ValueError("Context not provided to detection.detection_tags model post validator") + raise ValueError( + "Context not provided to detection.detection_tags model post validator" + ) if output_dto.attack_enrichment.use_enrichment is False: return self @@ -152,7 +159,9 @@ def addAttackEnrichment(self, info: ValidationInfo): missing_tactics: list[str] = [] for mitre_attack_id in self.mitre_attack_id: try: - mitre_enrichments.append(output_dto.attack_enrichment.getEnrichmentByMitreID(mitre_attack_id)) + mitre_enrichments.append( + output_dto.attack_enrichment.getEnrichmentByMitreID(mitre_attack_id) + ) except Exception: missing_tactics.append(mitre_attack_id) @@ -163,7 +172,7 @@ def addAttackEnrichment(self, info: ValidationInfo): return self - ''' + """ @field_validator('mitre_attack_enrichments', mode="before") @classmethod def addAttackEnrichments(cls, v:list[MitreAttackEnrichment], info:ValidationInfo)->list[MitreAttackEnrichment]: @@ -181,31 +190,43 @@ def addAttackEnrichments(cls, v:list[MitreAttackEnrichment], info:ValidationInfo enrichments = [] return enrichments - ''' + """ - @field_validator('analytic_story', mode="before") + @field_validator("analytic_story", mode="before") @classmethod - def mapStoryNamesToStoryObjects(cls, v: list[str], info: ValidationInfo) -> list[Story]: + def mapStoryNamesToStoryObjects( + cls, v: list[str], info: ValidationInfo + ) -> list[Story]: if info.context is None: raise ValueError("ValidationInfo.context unexpectedly null") - return Story.mapNamesToSecurityContentObjects(v, info.context.get("output_dto", None)) + return Story.mapNamesToSecurityContentObjects( + v, info.context.get("output_dto", None) + ) def getAtomicGuidStringArray(self) -> List[str]: - return [str(atomic_guid.auto_generated_guid) for atomic_guid in self.atomic_guid] + return [ + str(atomic_guid.auto_generated_guid) for atomic_guid in self.atomic_guid + ] - @field_validator('atomic_guid', mode="before") + @field_validator("atomic_guid", mode="before") @classmethod - def mapAtomicGuidsToAtomicTests(cls, v: List[UUID4], info: ValidationInfo) -> List[AtomicTest]: + def mapAtomicGuidsToAtomicTests( + cls, v: List[UUID4], info: ValidationInfo + ) -> List[AtomicTest]: if len(v) == 0: return [] if info.context is None: raise ValueError("ValidationInfo.context unexpectedly null") - output_dto: Union[DirectorOutputDto, None] = info.context.get("output_dto", None) + output_dto: Union[DirectorOutputDto, None] = info.context.get( + "output_dto", None + ) if output_dto is None: - raise ValueError("Context not provided to detection.detection_tags.atomic_guid validator") + raise ValueError( + "Context not provided to detection.detection_tags.atomic_guid validator" + ) atomic_enrichment: AtomicEnrichment = output_dto.atomic_enrichment @@ -247,4 +268,6 @@ def mapAtomicGuidsToAtomicTests(cls, v: List[UUID4], info: ValidationInfo) -> Li elif len(missing_tests) > 0: raise ValueError(missing_tests_string) - return matched_tests + [AtomicTest.AtomicTestWhenTestIsMissing(test) for test in missing_tests] + return matched_tests + [ + AtomicTest.AtomicTestWhenTestIsMissing(test) for test in missing_tests + ] diff --git a/contentctl/objects/drilldown.py b/contentctl/objects/drilldown.py index b5604748..470ff37f 100644 --- a/contentctl/objects/drilldown.py +++ b/contentctl/objects/drilldown.py @@ -1,71 +1,95 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_serializer from typing import TYPE_CHECKING + if TYPE_CHECKING: from contentctl.objects.detection import Detection from contentctl.objects.enums import AnalyticsType + DRILLDOWN_SEARCH_PLACEHOLDER = "%original_detection_search%" EARLIEST_OFFSET = "$info_min_time$" LATEST_OFFSET = "$info_max_time$" RISK_SEARCH = "index = risk starthoursago = 168 endhoursago = 0 | stats count values(search_name) values(risk_message) values(analyticstories) values(annotations._all) values(annotations.mitre_attack.mitre_tactic) " + class Drilldown(BaseModel): name: str = Field(..., description="The name of the drilldown search", min_length=5) - search: str = Field(..., description="The text of a drilldown search. This must be valid SPL.", min_length=1) - earliest_offset:None | str = Field(..., - description="Earliest offset time for the drilldown search. " - f"The most common value for this field is '{EARLIEST_OFFSET}', " - "but it is NOT the default value and must be supplied explicitly.", - min_length= 1) - latest_offset:None | str = Field(..., - description="Latest offset time for the driolldown search. " - f"The most common value for this field is '{LATEST_OFFSET}', " - "but it is NOT the default value and must be supplied explicitly.", - min_length= 1) + search: str = Field( + ..., + description="The text of a drilldown search. This must be valid SPL.", + min_length=1, + ) + earliest_offset: None | str = Field( + ..., + description="Earliest offset time for the drilldown search. " + f"The most common value for this field is '{EARLIEST_OFFSET}', " + "but it is NOT the default value and must be supplied explicitly.", + min_length=1, + ) + latest_offset: None | str = Field( + ..., + description="Latest offset time for the driolldown search. " + f"The most common value for this field is '{LATEST_OFFSET}', " + "but it is NOT the default value and must be supplied explicitly.", + min_length=1, + ) # TODO (cmcginley): @ljstella the drilldowns will need to be updated @classmethod def constructDrilldownsFromDetection(cls, detection: Detection) -> list[Drilldown]: - victim_observables = [o for o in detection.tags.observable if o.role[0] == "Victim"] + victim_observables = [ + o for o in detection.tags.observable if o.role[0] == "Victim" + ] if len(victim_observables) == 0 or detection.type == AnalyticsType.Hunting: # No victims, so no drilldowns return [] print(f"Adding default drilldowns for [{detection.name}]") - variableNamesString = ' and '.join([f"${o.name}$" for o in victim_observables]) + variableNamesString = " and ".join([f"${o.name}$" for o in victim_observables]) nameField = f"View the detection results for {variableNamesString}" - appendedSearch = " | search " + ' '.join([f"{o.name} = ${o.name}$" for o in victim_observables]) + appendedSearch = " | search " + " ".join( + [f"{o.name} = ${o.name}$" for o in victim_observables] + ) search_field = f"{detection.search}{appendedSearch}" - detection_results = cls(name=nameField, earliest_offset=EARLIEST_OFFSET, latest_offset=LATEST_OFFSET, search=search_field) - - + detection_results = cls( + name=nameField, + earliest_offset=EARLIEST_OFFSET, + latest_offset=LATEST_OFFSET, + search=search_field, + ) + nameField = f"View risk events for the last 7 days for {variableNamesString}" - fieldNamesListString = ', '.join([o.name for o in victim_observables]) + fieldNamesListString = ", ".join([o.name for o in victim_observables]) search_field = f"{RISK_SEARCH}by {fieldNamesListString} {appendedSearch}" - risk_events_last_7_days = cls(name=nameField, earliest_offset=None, latest_offset=None, search=search_field) + risk_events_last_7_days = cls( + name=nameField, + earliest_offset=None, + latest_offset=None, + search=search_field, + ) - return [detection_results,risk_events_last_7_days] - + return [detection_results, risk_events_last_7_days] - def perform_search_substitutions(self, detection:Detection)->None: + def perform_search_substitutions(self, detection: Detection) -> None: """Replaces the field DRILLDOWN_SEARCH_PLACEHOLDER (%original_detection_search%) with the search contained in the detection. We do this so that the YML does not need the search copy/pasted from the search field into the drilldown object. Args: detection (Detection): Detection to be used to update the search field of the drilldown - """ - self.search = self.search.replace(DRILLDOWN_SEARCH_PLACEHOLDER, detection.search) - + """ + self.search = self.search.replace( + DRILLDOWN_SEARCH_PLACEHOLDER, detection.search + ) @model_serializer - def serialize_model(self) -> dict[str,str]: - #Call serializer for parent - model:dict[str,str] = {} + def serialize_model(self) -> dict[str, str]: + # Call serializer for parent + model: dict[str, str] = {} - model['name'] = self.name - model['search'] = self.search + model["name"] = self.name + model["search"] = self.search if self.earliest_offset is not None: - model['earliest_offset'] = self.earliest_offset + model["earliest_offset"] = self.earliest_offset if self.latest_offset is not None: - model['latest_offset'] = self.latest_offset - return model \ No newline at end of file + model["latest_offset"] = self.latest_offset + return model diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index 8070d4a4..50109239 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -9,6 +9,7 @@ class AnalyticsType(StrEnum): Hunting = "Hunting" Correlation = "Correlation" + class DeploymentType(StrEnum): TTP = "TTP" Anomaly = "Anomaly" @@ -20,7 +21,7 @@ class DeploymentType(StrEnum): class DataModel(StrEnum): ENDPOINT = "Endpoint" - NETWORK_TRAFFIC = "Network_Traffic" + NETWORK_TRAFFIC = "Network_Traffic" AUTHENTICATION = "Authentication" CHANGE = "Change" CHANGE_ANALYSIS = "Change_Analysis" @@ -31,11 +32,11 @@ class DataModel(StrEnum): UPDATES = "Updates" VULNERABILITIES = "Vulnerabilities" WEB = "Web" - #Should the following more specific DMs be added? - #Or should they just fall under endpoint? - #ENDPOINT_PROCESSES = "Endpoint_Processes" - #ENDPOINT_FILESYSTEM = "Endpoint_Filesystem" - #ENDPOINT_REGISTRY = "Endpoint_Registry" + # Should the following more specific DMs be added? + # Or should they just fall under endpoint? + # ENDPOINT_PROCESSES = "Endpoint_Processes" + # ENDPOINT_FILESYSTEM = "Endpoint_Filesystem" + # ENDPOINT_REGISTRY = "Endpoint_Registry" RISK = "Risk" SPLUNK_AUDIT = "Splunk_Audit" @@ -44,6 +45,7 @@ class PlaybookType(StrEnum): INVESTIGATION = "Investigation" RESPONSE = "Response" + class SecurityContentType(IntEnum): detections = 1 baselines = 2 @@ -68,7 +70,6 @@ class SecurityContentType(IntEnum): # json_objects = "json_objects" - class SecurityContentProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" @@ -76,6 +77,7 @@ class SecurityContentProductName(StrEnum): SPLUNK_SECURITY_ANALYTICS_FOR_AWS = "Splunk Security Analytics for AWS" SPLUNK_BEHAVIORAL_ANALYTICS = "Splunk Behavioral Analytics" + class SecurityContentInvestigationProductName(StrEnum): SPLUNK_ENTERPRISE = "Splunk Enterprise" SPLUNK_ENTERPRISE_SECURITY = "Splunk Enterprise Security" @@ -83,7 +85,7 @@ class SecurityContentInvestigationProductName(StrEnum): SPLUNK_SECURITY_ANALYTICS_FOR_AWS = "Splunk Security Analytics for AWS" SPLUNK_BEHAVIORAL_ANALYTICS = "Splunk Behavioral Analytics" SPLUNK_PHANTOM = "Splunk Phantom" - + class DetectionStatus(StrEnum): production = "production" @@ -147,7 +149,7 @@ class DetectionTestingMode(StrEnum): # "Actions on Objectives": 7 # } class KillChainPhase(StrEnum): - UNKNOWN ="Unknown" + UNKNOWN = "Unknown" RECONNAISSANCE = "Reconnaissance" WEAPONIZATION = "Weaponization" DELIVERY = "Delivery" @@ -197,6 +199,7 @@ class DataSource(StrEnum): WINDOWS_SECURITY_5145 = "Windows Security 5145" WINDOWS_SYSTEM_7045 = "Windows System 7045" + class ProvidingTechnology(StrEnum): AMAZON_SECURITY_LAKE = "Amazon Security Lake" AMAZON_WEB_SERVICES_CLOUDTRAIL = "Amazon Web Services - Cloudtrail" @@ -216,9 +219,9 @@ class ProvidingTechnology(StrEnum): SPLUNK_INTERNAL_LOGS = "Splunk Internal Logs" SYMANTEC_ENDPOINT_PROTECTION = "Symantec Endpoint Protection" ZEEK = "Zeek" - + @staticmethod - def getProvidingTechFromSearch(search_string:str)->List[ProvidingTechnology]: + def getProvidingTechFromSearch(search_string: str) -> List[ProvidingTechnology]: """_summary_ Args: @@ -230,34 +233,45 @@ def getProvidingTechFromSearch(search_string:str)->List[ProvidingTechnology]: Returns: List[ProvidingTechnology]: List of providing technologies (with no duplicates because it is derived from a set) calculated from the search string. - """ - matched_technologies:set[ProvidingTechnology] = set() - #As there are many different sources that use google logs, we define the set once - google_logs = set([ProvidingTechnology.GOOGLE_WORKSPACE, ProvidingTechnology.GOOGLE_CLOUD_PLATFORM]) + """ + matched_technologies: set[ProvidingTechnology] = set() + # As there are many different sources that use google logs, we define the set once + google_logs = set( + [ + ProvidingTechnology.GOOGLE_WORKSPACE, + ProvidingTechnology.GOOGLE_CLOUD_PLATFORM, + ] + ) providing_technologies_mapping = { - '`amazon_security_lake`': set([ProvidingTechnology.AMAZON_SECURITY_LAKE]), - 'audit_searches': set([ProvidingTechnology.SPLUNK_INTERNAL_LOGS]), - '`azure_monitor_aad`': set([ProvidingTechnology.AZURE_AD, ProvidingTechnology.ENTRA_ID]), - '`cloudtrail`': set([ProvidingTechnology.AMAZON_WEB_SERVICES_CLOUDTRAIL]), - #Endpoint is NOT a Macro (and this is intentional since it is to capture Endpoint Datamodel usage) - 'Endpoint': set([ProvidingTechnology.MICROSOFT_SYSMON, - ProvidingTechnology.MICROSOFT_WINDOWS, - ProvidingTechnology.CARBON_BLACK_RESPONSE, - ProvidingTechnology.CROWDSTRIKE_FALCON, - ProvidingTechnology.SYMANTEC_ENDPOINT_PROTECTION]), - '`google_': google_logs, - '`gsuite': google_logs, - '`gws_': google_logs, - '`kube': set([ProvidingTechnology.KUBERNETES]), - '`ms_defender`': set([ProvidingTechnology.MICROSOFT_DEFENDER]), - '`o365_': set([ProvidingTechnology.MICROSOFT_OFFICE_365]), - '`okta': set([ProvidingTechnology.OKTA]), - '`pingid`': set([ProvidingTechnology.PING_ID]), - '`powershell`': set(set([ProvidingTechnology.MICROSOFT_WINDOWS])), - '`splunkd_': set([ProvidingTechnology.SPLUNK_INTERNAL_LOGS]), - '`sysmon`': set([ProvidingTechnology.MICROSOFT_SYSMON]), - '`wineventlog_security`': set([ProvidingTechnology.MICROSOFT_WINDOWS]), - '`zeek_': set([ProvidingTechnology.ZEEK]), + "`amazon_security_lake`": set([ProvidingTechnology.AMAZON_SECURITY_LAKE]), + "audit_searches": set([ProvidingTechnology.SPLUNK_INTERNAL_LOGS]), + "`azure_monitor_aad`": set( + [ProvidingTechnology.AZURE_AD, ProvidingTechnology.ENTRA_ID] + ), + "`cloudtrail`": set([ProvidingTechnology.AMAZON_WEB_SERVICES_CLOUDTRAIL]), + # Endpoint is NOT a Macro (and this is intentional since it is to capture Endpoint Datamodel usage) + "Endpoint": set( + [ + ProvidingTechnology.MICROSOFT_SYSMON, + ProvidingTechnology.MICROSOFT_WINDOWS, + ProvidingTechnology.CARBON_BLACK_RESPONSE, + ProvidingTechnology.CROWDSTRIKE_FALCON, + ProvidingTechnology.SYMANTEC_ENDPOINT_PROTECTION, + ] + ), + "`google_": google_logs, + "`gsuite": google_logs, + "`gws_": google_logs, + "`kube": set([ProvidingTechnology.KUBERNETES]), + "`ms_defender`": set([ProvidingTechnology.MICROSOFT_DEFENDER]), + "`o365_": set([ProvidingTechnology.MICROSOFT_OFFICE_365]), + "`okta": set([ProvidingTechnology.OKTA]), + "`pingid`": set([ProvidingTechnology.PING_ID]), + "`powershell`": set(set([ProvidingTechnology.MICROSOFT_WINDOWS])), + "`splunkd_": set([ProvidingTechnology.SPLUNK_INTERNAL_LOGS]), + "`sysmon`": set([ProvidingTechnology.MICROSOFT_SYSMON]), + "`wineventlog_security`": set([ProvidingTechnology.MICROSOFT_WINDOWS]), + "`zeek_": set([ProvidingTechnology.ZEEK]), } for key in providing_technologies_mapping: if key in search_string: @@ -286,6 +300,7 @@ class Cis18Value(StrEnum): CIS_17 = "CIS 17" CIS_18 = "CIS 18" + class SecurityDomain(StrEnum): ENDPOINT = "endpoint" NETWORK = "network" @@ -294,6 +309,7 @@ class SecurityDomain(StrEnum): ACCESS = "access" AUDIT = "audit" + class AssetType(StrEnum): AWS_ACCOUNT = "AWS Account" AWS_EKS_KUBERNETES_CLUSTER = "AWS EKS Kubernetes cluster" @@ -303,9 +319,9 @@ class AssetType(StrEnum): AMAZON_EKS_KUBERNETES_CLUSTER = "Amazon EKS Kubernetes cluster" AMAZON_EKS_KUBERNETES_CLUSTER_POD = "Amazon EKS Kubernetes cluster Pod" AMAZON_ELASTIC_CONTAINER_REGISTRY = "Amazon Elastic Container Registry" - #AZURE = "Azure" - #AZURE_AD = "Azure AD" - #AZURE_AD_TENANT = "Azure AD Tenant" + # AZURE = "Azure" + # AZURE_AD = "Azure AD" + # AZURE_AD_TENANT = "Azure AD Tenant" AZURE_TENANT = "Azure Tenant" AZURE_AKS_KUBERNETES_CLUSTER = "Azure AKS Kubernetes cluster" AZURE_ACTIVE_DIRECTORY = "Azure Active Directory" @@ -332,8 +348,8 @@ class AssetType(StrEnum): INSTANCE = "Instance" KUBERNETES = "Kubernetes" NETWORK = "Network" - #OFFICE_365 = "Office 365" - #OFFICE_365_Tenant = "Office 365 Tenant" + # OFFICE_365 = "Office 365" + # OFFICE_365_Tenant = "Office 365 Tenant" O365_TENANT = "O365 Tenant" OKTA_TENANT = "Okta Tenant" PROXY = "Proxy" @@ -345,6 +361,7 @@ class AssetType(StrEnum): WEB_APPLICATION = "Web Application" WINDOWS = "Windows" + class NistCategory(StrEnum): ID_AM = "ID.AM" ID_BE = "ID.BE" @@ -369,6 +386,7 @@ class NistCategory(StrEnum): RC_IM = "RC.IM" RC_CO = "RC.CO" + class RiskSeverity(StrEnum): # Levels taken from the following documentation link # https://docs.splunk.com/Documentation/ES/7.3.2/User/RiskScoring diff --git a/contentctl/objects/errors.py b/contentctl/objects/errors.py index 06f7751a..9d81f546 100644 --- a/contentctl/objects/errors.py +++ b/contentctl/objects/errors.py @@ -4,21 +4,25 @@ class ValidationFailed(Exception): """Indicates not an error in execution, but a validation failure""" + pass class IntegrationTestingError(Exception): """Base exception class for integration testing""" + pass class ServerError(IntegrationTestingError): """An error encounterd during integration testing, as provided by the server (Splunk instance)""" + pass class ClientError(IntegrationTestingError): """An error encounterd during integration testing, on the client's side (locally)""" + pass @@ -26,6 +30,7 @@ class MetadataValidationError(Exception, ABC): """ Base class for any errors arising from savedsearches.conf detection metadata validation """ + # The name of the rule the error relates to rule_name: str @@ -52,11 +57,8 @@ class DetectionMissingError(MetadataValidationError): """ An error indicating a detection in the prior build could not be found in the current build """ - def __init__( - self, - rule_name: str, - *args: object - ) -> None: + + def __init__(self, rule_name: str, *args: object) -> None: self.rule_name = rule_name super().__init__(self.long_message, *args) @@ -77,15 +79,14 @@ def short_message(self) -> str: A short-form error message :returns: a str, the message """ - return ( - "Detection from previous build not found in current build." - ) + return "Detection from previous build not found in current build." class DetectionIDError(MetadataValidationError): """ An error indicating the detection ID may have changed between builds """ + # The ID from the current build current_id: UUID @@ -93,11 +94,7 @@ class DetectionIDError(MetadataValidationError): previous_id: UUID def __init__( - self, - rule_name: str, - current_id: UUID, - previous_id: UUID, - *args: object + self, rule_name: str, current_id: UUID, previous_id: UUID, *args: object ) -> None: self.rule_name = rule_name self.current_id = current_id @@ -122,15 +119,14 @@ def short_message(self) -> str: A short-form error message :returns: a str, the message """ - return ( - f"Detection ID {self.current_id} in current build does not match ID {self.previous_id} in previous build." - ) + return f"Detection ID {self.current_id} in current build does not match ID {self.previous_id} in previous build." class VersioningError(MetadataValidationError, ABC): """ A base class for any metadata validation errors relating to detection versioning """ + # The version in the current build current_version: int @@ -138,11 +134,7 @@ class VersioningError(MetadataValidationError, ABC): previous_version: int def __init__( - self, - rule_name: str, - current_version: int, - previous_version: int, - *args: object + self, rule_name: str, current_version: int, previous_version: int, *args: object ) -> None: self.rule_name = rule_name self.current_version = current_version @@ -154,6 +146,7 @@ class VersionDecrementedError(VersioningError): """ An error indicating the version number went down between builds """ + @property def long_message(self) -> str: """ @@ -182,6 +175,7 @@ class VersionBumpingError(VersioningError): """ An error indicating the detection changed but its version wasn't bumped appropriately """ + @property def long_message(self) -> str: """ @@ -200,6 +194,4 @@ def short_message(self) -> str: A short-form error message :returns: a str, the message """ - return ( - f"Detection version in current build should be bumped to at least {self.previous_version + 1}." - ) + return f"Detection version in current build should be bumped to at least {self.previous_version + 1}." diff --git a/contentctl/objects/integration_test.py b/contentctl/objects/integration_test.py index 4f88be70..c1078e89 100644 --- a/contentctl/objects/integration_test.py +++ b/contentctl/objects/integration_test.py @@ -10,6 +10,7 @@ class IntegrationTest(BaseTest): """ An integration test for a detection against ES """ + # The test type (integration) test_type: TestType = Field(default=TestType.INTEGRATION) @@ -34,7 +35,6 @@ def skip(self, message: str) -> None: Skip the test by setting its result status :param message: the reason for skipping """ - self.result = IntegrationTestResult( # type: ignore - message=message, - status=TestResultStatus.SKIP + self.result = IntegrationTestResult( # type: ignore + message=message, status=TestResultStatus.SKIP ) diff --git a/contentctl/objects/integration_test_result.py b/contentctl/objects/integration_test_result.py index 18390120..a03cb71c 100644 --- a/contentctl/objects/integration_test_result.py +++ b/contentctl/objects/integration_test_result.py @@ -5,5 +5,6 @@ class IntegrationTestResult(BaseTestResult): """ An integration test result """ + # the total time we slept waiting for the detection to fire after activating it wait_duration: int | None = None diff --git a/contentctl/objects/investigation.py b/contentctl/objects/investigation.py index 0d35a9db..c74b1b86 100644 --- a/contentctl/objects/investigation.py +++ b/contentctl/objects/investigation.py @@ -1,21 +1,22 @@ from __future__ import annotations import re from typing import List, Any -from pydantic import computed_field, Field, ConfigDict,model_serializer +from pydantic import computed_field, Field, ConfigDict, model_serializer from contentctl.objects.security_content_object import SecurityContentObject from contentctl.objects.enums import DataModel from contentctl.objects.investigation_tags import InvestigationTags from contentctl.objects.constants import ( CONTENTCTL_MAX_SEARCH_NAME_LENGTH, CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE, - CONTENTCTL_MAX_STANZA_LENGTH + CONTENTCTL_MAX_STANZA_LENGTH, ) from contentctl.objects.config import CustomApp + class Investigation(SecurityContentObject): model_config = ConfigDict(validate_default=False) - type: str = Field(...,pattern="^Investigation$") - name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) + type: str = Field(..., pattern="^Investigation$") + name: str = Field(..., max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH) search: str = Field(...) how_to_implement: str = Field(...) known_false_positives: str = Field(...) @@ -24,9 +25,9 @@ class Investigation(SecurityContentObject): # enrichment @computed_field @property - def inputs(self)->List[str]: - #Parse out and return all inputs from the searchj - inputs:List[str] = [] + def inputs(self) -> List[str]: + # Parse out and return all inputs from the searchj + inputs: List[str] = [] pattern = r"\$([^\s.]*)\$" for input in re.findall(pattern, self.search): @@ -41,27 +42,42 @@ def datamodel(self) -> List[DataModel]: @computed_field @property - def lowercase_name(self)->str: - return self.name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower().replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + def lowercase_name(self) -> str: + return ( + self.name.replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + .replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + ) - # This is a slightly modified version of the get_conf_stanza_name function from # SecurityContentObject_Abstract - def get_response_task_name(self, app:CustomApp, max_stanza_length:int=CONTENTCTL_MAX_STANZA_LENGTH)->str: - stanza_name = CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name) + def get_response_task_name( + self, app: CustomApp, max_stanza_length: int = CONTENTCTL_MAX_STANZA_LENGTH + ) -> str: + stanza_name = CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE.format( + app_label=app.label, detection_name=self.name + ) if len(stanza_name) > max_stanza_length: - raise ValueError(f"conf stanza may only be {max_stanza_length} characters, " - f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' ") + raise ValueError( + f"conf stanza may only be {max_stanza_length} characters, " + f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' " + ) return stanza_name - @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - - #All fields custom to this model - model= { + + # All fields custom to this model + model = { "type": self.type, "datamodel": self.datamodel, "search": self.search, @@ -69,17 +85,16 @@ def serialize_model(self): "known_false_positives": self.known_false_positives, "inputs": self.inputs, "tags": self.tags.model_dump(), - "lowercase_name":self.lowercase_name + "lowercase_name": self.lowercase_name, } - - #Combine fields from this model with fields from parent + + # Combine fields from this model with fields from parent super_fields.update(model) - - #return the model - return super_fields + # return the model + return super_fields - def model_post_init(self, ctx:dict[str,Any]): + def model_post_init(self, ctx: dict[str, Any]): # Ensure we link all stories this investigation references # back to itself for story in self.tags.analytic_story: diff --git a/contentctl/objects/investigation_tags.py b/contentctl/objects/investigation_tags.py index c4b812e6..a8ff6307 100644 --- a/contentctl/objects/investigation_tags.py +++ b/contentctl/objects/investigation_tags.py @@ -1,33 +1,45 @@ from __future__ import annotations from typing import List -from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer,ConfigDict +from pydantic import ( + BaseModel, + Field, + field_validator, + ValidationInfo, + model_serializer, + ConfigDict, +) from contentctl.objects.story import Story -from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain +from contentctl.objects.enums import ( + SecurityContentInvestigationProductName, + SecurityDomain, +) + class InvestigationTags(BaseModel): model_config = ConfigDict(extra="forbid") - analytic_story: List[Story] = Field([],min_length=1) - product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1) + analytic_story: List[Story] = Field([], min_length=1) + product: List[SecurityContentInvestigationProductName] = Field(..., min_length=1) security_domain: SecurityDomain = Field(...) - - @field_validator('analytic_story',mode="before") + @field_validator("analytic_story", mode="before") @classmethod - def mapStoryNamesToStoryObjects(cls, v:list[str], info:ValidationInfo)->list[Story]: - return Story.mapNamesToSecurityContentObjects(v, info.context.get("output_dto",None)) - + def mapStoryNamesToStoryObjects( + cls, v: list[str], info: ValidationInfo + ) -> list[Story]: + return Story.mapNamesToSecurityContentObjects( + v, info.context.get("output_dto", None) + ) @model_serializer def serialize_model(self): - #All fields custom to this model - model= { + # All fields custom to this model + model = { "analytic_story": [story.name for story in self.analytic_story], "product": self.product, "security_domain": self.security_domain, } - - #Combine fields from this model with fields from parent - - - #return the model - return model \ No newline at end of file + + # Combine fields from this model with fields from parent + + # return the model + return model diff --git a/contentctl/objects/lookup.py b/contentctl/objects/lookup.py index 9f0b1318..8cd0d0bd 100644 --- a/contentctl/objects/lookup.py +++ b/contentctl/objects/lookup.py @@ -1,6 +1,16 @@ from __future__ import annotations -from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field, TypeAdapter +from pydantic import ( + field_validator, + ValidationInfo, + model_validator, + FilePath, + model_serializer, + Field, + NonNegativeInt, + computed_field, + TypeAdapter, +) from enum import StrEnum, auto from typing import TYPE_CHECKING, Optional, Any, Literal, Annotated, Self import re @@ -8,6 +18,7 @@ import abc from functools import cached_property import pathlib + if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.config import validate @@ -15,21 +26,37 @@ # This section is used to ignore lookups that are NOT shipped with ESCU app but are used in the detections. Adding exclusions here will so that contentctl builds will not fail. LOOKUPS_TO_IGNORE = set(["outputlookup"]) -LOOKUPS_TO_IGNORE.add("ut_shannon_lookup") #In the URL toolbox app which is recommended for ESCU -LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Asset and Identity Framework -LOOKUPS_TO_IGNORE.add("cim_corporate_web_domain_lookup") #Shipped with the Asset and Identity Framework -LOOKUPS_TO_IGNORE.add("cim_corporate_email_domain_lookup") #Shipped with the Enterprise Security -LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup") #Shipped with the Enterprise Security - -LOOKUPS_TO_IGNORE.add("alexa_lookup_by_str") #Shipped with the Asset and Identity Framework -LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework -LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework -LOOKUPS_TO_IGNORE.add("admon_groups_def") #Shipped with the SA-admon addon -LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Enterprise Security - -#Special case for the Detection "Exploit Public Facing Application via Apache Commons Text" -LOOKUPS_TO_IGNORE.add("=") -LOOKUPS_TO_IGNORE.add("other_lookups") +LOOKUPS_TO_IGNORE.add( + "ut_shannon_lookup" +) # In the URL toolbox app which is recommended for ESCU +LOOKUPS_TO_IGNORE.add( + "identity_lookup_expanded" +) # Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add( + "cim_corporate_web_domain_lookup" +) # Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add( + "cim_corporate_email_domain_lookup" +) # Shipped with the Enterprise Security +LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup") # Shipped with the Enterprise Security + +LOOKUPS_TO_IGNORE.add( + "alexa_lookup_by_str" +) # Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add( + "interesting_ports_lookup" +) # Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add( + "asset_lookup_by_str" +) # Shipped with the Asset and Identity Framework +LOOKUPS_TO_IGNORE.add("admon_groups_def") # Shipped with the SA-admon addon +LOOKUPS_TO_IGNORE.add( + "identity_lookup_expanded" +) # Shipped with the Enterprise Security + +# Special case for the Detection "Exploit Public Facing Application via Apache Commons Text" +LOOKUPS_TO_IGNORE.add("=") +LOOKUPS_TO_IGNORE.add("other_lookups") class Lookup_Type(StrEnum): @@ -38,164 +65,201 @@ class Lookup_Type(StrEnum): mlmodel = auto() - # TODO (#220): Split Lookup into 2 classes -class Lookup(SecurityContentObject, abc.ABC): +class Lookup(SecurityContentObject, abc.ABC): default_match: Optional[bool] = None # Per the documentation for transforms.conf, EXACT should not be specified in this list, # so we include only WILDCARD and CIDR - match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field(default=[]) + match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field( + default=[] + ) min_matches: None | NonNegativeInt = Field(default=None) - max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field(default=None) + max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field( + default=None + ) case_sensitive_match: None | bool = Field(default=None) - - - - @model_serializer def serialize_model(self): - #Call parent serializer + # Call parent serializer super_fields = super().serialize_model() - #All fields custom to this model - model= { - + # All fields custom to this model + model = { "default_match": "true" if self.default_match is True else "false", "match_type": self.match_type_to_conf_format, "min_matches": self.min_matches, "max_matches": self.max_matches, - "case_sensitive_match": "true" if self.case_sensitive_match is True else "false", + "case_sensitive_match": "true" + if self.case_sensitive_match is True + else "false", } - - #return the model + + # return the model model.update(super_fields) return model @model_validator(mode="before") - def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any: + def fix_lookup_path(cls, data: Any, info: ValidationInfo) -> Any: if data.get("filename"): - config:validate = info.context.get("config",None) + config: validate = info.context.get("config", None) if config is not None: data["filename"] = config.path / "lookups/" / data["filename"] else: - raise ValueError("config required for constructing lookup filename, but it was not") + raise ValueError( + "config required for constructing lookup filename, but it was not" + ) return data - @computed_field @cached_property - def match_type_to_conf_format(self)->str: - return ', '.join(self.match_type) - - + def match_type_to_conf_format(self) -> str: + return ", ".join(self.match_type) + @staticmethod - def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]: + def get_lookups( + text_field: str, + director: DirectorOutputDto, + ignore_lookups: set[str] = LOOKUPS_TO_IGNORE, + ) -> list[Lookup]: # Comprehensively match all kinds of lookups, including inputlookup and outputlookup - inputLookupsToGet = set(re.findall(r'[^\w]inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([\w]+)', text_field, re.IGNORECASE)) - outputLookupsToGet = set(re.findall(r'[^\w]outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([\w]+)',text_field,re.IGNORECASE)) - lookupsToGet = set(re.findall(r'[^\w](?:(?Self: + def ensure_lookup_file_exists(self) -> Self: if not self.filename.exists(): raise ValueError(f"Expected lookup filename {self.filename} does not exist") return self @computed_field @cached_property - def filename(self)->FilePath: + def filename(self) -> FilePath: if self.file_path is None: - raise ValueError(f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None") #type: ignore - - csv_file = self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}" #type: ignore + raise ValueError( + f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None" + ) # type: ignore + + csv_file = self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}" # type: ignore return csv_file - + @computed_field @cached_property - def app_filename(self)->FilePath: - ''' + def app_filename(self) -> FilePath: + """ We may consider two options: 1. Always apply the datetime stamp to the end of the file. This makes the code easier 2. Only apply the datetime stamp if it is version > 1. This makes the code a small fraction more complicated, but preserves longstanding CSV that have not been modified in a long time - ''' - return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore + """ + return pathlib.Path( + f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}" + ) # type: ignore + class CSVLookup(FileBackedLookup): - lookup_type:Literal[Lookup_Type.csv] - + lookup_type: Literal[Lookup_Type.csv] + @model_serializer def serialize_model(self): - #Call parent serializer + # Call parent serializer super_fields = super().serialize_model() - #All fields custom to this model - model= { - "filename": self.app_filename.name - } - - #return the model + # All fields custom to this model + model = {"filename": self.app_filename.name} + + # return the model model.update(super_fields) return model - + @model_validator(mode="after") - def ensure_correct_csv_structure(self)->Self: + def ensure_correct_csv_structure(self) -> Self: # https://docs.python.org/3/library/csv.html#csv.DictReader # Column Names (fieldnames) determine by the number of columns in the first row. # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception - # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception. - csv_errors:list[str] = [] + # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception. + csv_errors: list[str] = [] with open(self.filename, "r") as csv_fp: RESTKEY = "extra_fields_in_a_row" - csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY) + csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY) if csv_dict.fieldnames is None: - raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t" - "Unable to read fieldnames from CSV. Is the CSV empty?\n" - " Please try opening the file with a CSV Editor to ensure that it is correct.") + raise ValueError( + f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t" + "Unable to read fieldnames from CSV. Is the CSV empty?\n" + " Please try opening the file with a CSV Editor to ensure that it is correct." + ) # Remember that row 1 has the headers and we do not iterate over it in the loop below # CSVs are typically indexed starting a row 1 for the header. for row_index, data_row in enumerate(csv_dict): - row_index+=2 - if len(data_row.get(RESTKEY,[])) > 0: - csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns," - f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY,[]))}].") - + row_index += 2 + if len(data_row.get(RESTKEY, [])) > 0: + csv_errors.append( + f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns," + f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY, []))}]." + ) + for column_index, column_name in enumerate(data_row): if data_row[column_name] is None: - csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, " - f"but instead had [{column_index}].") + csv_errors.append( + f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, " + f"but instead had [{column_index}]." + ) if len(csv_errors) > 0: - err_string = '\n\t'.join(csv_errors) - raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t{err_string}\n" - f" Please try opening the file with a CSV Editor to ensure that it is correct.") - - return self + err_string = "\n\t".join(csv_errors) + raise ValueError( + f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t{err_string}\n" + f" Please try opening the file with a CSV Editor to ensure that it is correct." + ) + return self class KVStoreLookup(Lookup): lookup_type: Literal[Lookup_Type.kvstore] - fields: list[str] = Field(description="The names of the fields/headings for the KVStore.", min_length=1) + fields: list[str] = Field( + description="The names of the fields/headings for the KVStore.", min_length=1 + ) - @field_validator("fields", mode='after') + @field_validator("fields", mode="after") @classmethod def ensure_key(cls, values: list[str]): if values[0] != "_key": @@ -204,32 +268,34 @@ def ensure_key(cls, values: list[str]): @computed_field @cached_property - def collection(self)->str: + def collection(self) -> str: return self.name @computed_field @cached_property - def fields_to_fields_list_conf_format(self)->str: - return ', '.join(self.fields) + def fields_to_fields_list_conf_format(self) -> str: + return ", ".join(self.fields) @model_serializer def serialize_model(self): - #Call parent serializer + # Call parent serializer super_fields = super().serialize_model() - #All fields custom to this model - model= { + # All fields custom to this model + model = { "collection": self.collection, - "fields_list": self.fields_to_fields_list_conf_format + "fields_list": self.fields_to_fields_list_conf_format, } - - #return the model + + # return the model model.update(super_fields) return model + class MlModel(FileBackedLookup): lookup_type: Literal[Lookup_Type.mlmodel] - -LookupAdapter = TypeAdapter(Annotated[CSVLookup | KVStoreLookup | MlModel, Field(discriminator="lookup_type")]) +LookupAdapter = TypeAdapter( + Annotated[CSVLookup | KVStoreLookup | MlModel, Field(discriminator="lookup_type")] +) diff --git a/contentctl/objects/macro.py b/contentctl/objects/macro.py index 8c25dff7..04237a2c 100644 --- a/contentctl/objects/macro.py +++ b/contentctl/objects/macro.py @@ -1,4 +1,4 @@ -# Used so that we can have a staticmethod that takes the class +# Used so that we can have a staticmethod that takes the class # type Macro as an argument from __future__ import annotations from typing import TYPE_CHECKING, List @@ -6,18 +6,21 @@ from pydantic import Field, model_serializer, NonNegativeInt import uuid import datetime + if TYPE_CHECKING: from contentctl.input.director import DirectorOutputDto from contentctl.objects.security_content_object import SecurityContentObject -#The following macros are included in commonly-installed apps. -#As such, we will ignore if they are missing from our app. -#Included in -MACROS_TO_IGNORE = set(["drop_dm_object_name"]) # Part of CIM/Splunk_SA_CIM -MACROS_TO_IGNORE.add("get_asset") #SA-IdentityManagement, part of Enterprise Security -MACROS_TO_IGNORE.add("get_risk_severity") #SA-ThreatIntelligence, part of Enterprise Security -MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") #Part of CIM/Splunk_SA_CIM -#MACROS_TO_IGNORE.add("prohibited_processes") +# The following macros are included in commonly-installed apps. +# As such, we will ignore if they are missing from our app. +# Included in +MACROS_TO_IGNORE = set(["drop_dm_object_name"]) # Part of CIM/Splunk_SA_CIM +MACROS_TO_IGNORE.add("get_asset") # SA-IdentityManagement, part of Enterprise Security +MACROS_TO_IGNORE.add( + "get_risk_severity" +) # SA-ThreatIntelligence, part of Enterprise Security +MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") # Part of CIM/Splunk_SA_CIM +# MACROS_TO_IGNORE.add("prohibited_processes") class Macro(SecurityContentObject): @@ -26,48 +29,62 @@ class Macro(SecurityContentObject): # TODO: Add id field to all macro ymls id: uuid.UUID = Field(default_factory=uuid.uuid4) date: datetime.date = Field(datetime.date.today()) - author: str = Field("NO AUTHOR DEFINED",max_length=255) + author: str = Field("NO AUTHOR DEFINED", max_length=255) version: NonNegativeInt = 1 - - @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - #All fields custom to this model - model= { + # All fields custom to this model + model = { "definition": self.definition, "description": self.description, } - - #return the model + + # return the model model.update(super_fields) - + return model - + @staticmethod - def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[str]=MACROS_TO_IGNORE)->list[Macro]: - #Remove any comments, allowing there to be macros (which have a single backtick) inside those comments - #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here```` - #then there is a small edge case where the regex below does not work properly. If that is - #the case, we edit the search slightly to insert a space + def get_macros( + text_field: str, + director: DirectorOutputDto, + ignore_macros: set[str] = MACROS_TO_IGNORE, + ) -> list[Macro]: + # Remove any comments, allowing there to be macros (which have a single backtick) inside those comments + # If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here```` + # then there is a small edge case where the regex below does not work properly. If that is + # the case, we edit the search slightly to insert a space if re.findall(r"\`\`\`\`", text_field): - raise ValueError("Search contained four or more '`' characters in a row which is invalid SPL" - "This may have occurred when a macro was commented out.\n" - "Please ammend your search to remove the substring '````'") + raise ValueError( + "Search contained four or more '`' characters in a row which is invalid SPL" + "This may have occurred when a macro was commented out.\n" + "Please ammend your search to remove the substring '````'" + ) + + # Replace all the comments with a space. This prevents a comment from looking like a macro to the parser below + text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) + + # Find all the macros, which start and end with a '`' character + macros_to_get = re.findall(r"`([^\s]+)`", text_field) + # If macros take arguments, stop at the first argument. We just want the name of the macro + macros_to_get = set( + [ + macro[: macro.find("(")] if macro.find("(") != -1 else macro + for macro in macros_to_get + ] + ) - # Replace all the comments with a space. This prevents a comment from looking like a macro to the parser below - text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) - - # Find all the macros, which start and end with a '`' character - macros_to_get = re.findall(r'`([^\s]+)`', text_field) - #If macros take arguments, stop at the first argument. We just want the name of the macro - macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get]) - - macros_to_ignore = set([macro for macro in macros_to_get if any(to_ignore in macro for to_ignore in ignore_macros)]) - #remove the ones that we will ignore + macros_to_ignore = set( + [ + macro + for macro in macros_to_get + if any(to_ignore in macro for to_ignore in ignore_macros) + ] + ) + # remove the ones that we will ignore macros_to_get -= macros_to_ignore return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director) - diff --git a/contentctl/objects/manual_test.py b/contentctl/objects/manual_test.py index cc6e71a5..d77f21fe 100644 --- a/contentctl/objects/manual_test.py +++ b/contentctl/objects/manual_test.py @@ -12,6 +12,7 @@ class ManualTest(BaseTest): """ A manual test for a detection """ + # The test type (manual) test_type: TestType = Field(default=TestType.MANUAL) @@ -26,7 +27,6 @@ def skip(self, message: str) -> None: Skip the test by setting its result status :param message: the reason for skipping """ - self.result = ManualTestResult( # type: ignore - message=message, - status=TestResultStatus.SKIP + self.result = ManualTestResult( # type: ignore + message=message, status=TestResultStatus.SKIP ) diff --git a/contentctl/objects/manual_test_result.py b/contentctl/objects/manual_test_result.py index dd6439cd..9314bd31 100644 --- a/contentctl/objects/manual_test_result.py +++ b/contentctl/objects/manual_test_result.py @@ -5,4 +5,5 @@ class ManualTestResult(BaseTestResult): """ A manual test result """ + pass diff --git a/contentctl/objects/mitre_attack_enrichment.py b/contentctl/objects/mitre_attack_enrichment.py index 6c532bd8..f915fcd7 100644 --- a/contentctl/objects/mitre_attack_enrichment.py +++ b/contentctl/objects/mitre_attack_enrichment.py @@ -5,6 +5,7 @@ import datetime from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE + class MitreTactics(StrEnum): RECONNAISSANCE = "Reconnaissance" RESOURCE_DEVELOPMENT = "Resource Development" @@ -31,16 +32,17 @@ class AttackGroupMatrix(StrEnum): class AttackGroupType(StrEnum): intrusion_set = "intrusion-set" + class MitreExternalReference(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") source_name: str - external_id: None | str = None + external_id: None | str = None url: None | HttpUrl = None description: None | str = None class MitreAttackGroup(BaseModel): - model_config = ConfigDict(extra='forbid') + model_config = ConfigDict(extra="forbid") contributors: list[str] = [] created: datetime.datetime created_by_ref: str @@ -53,45 +55,44 @@ class MitreAttackGroup(BaseModel): matrix: list[AttackGroupMatrix] mitre_attack_spec_version: None | str mitre_version: str - #assume that if the deprecated field is not present, then the group is not deprecated + # assume that if the deprecated field is not present, then the group is not deprecated mitre_deprecated: bool modified: datetime.datetime modified_by_ref: str object_marking_refs: list[str] type: AttackGroupType url: HttpUrl - @field_validator("mitre_deprecated", mode="before") - def standardize_mitre_deprecated(cls, mitre_deprecated:bool | None) -> bool: - ''' + def standardize_mitre_deprecated(cls, mitre_deprecated: bool | None) -> bool: + """ For some reason, the API will return either a bool for mitre_deprecated OR None. We simplify our typing by converting None to False, and assuming that if deprecated is None, then the group is not deprecated. - ''' + """ if mitre_deprecated is None: return False return mitre_deprecated @field_validator("contributors", mode="before") - def standardize_contributors(cls, contributors:list[str] | None) -> list[str]: - ''' + def standardize_contributors(cls, contributors: list[str] | None) -> list[str]: + """ For some reason, the API will return either a list of strings for contributors OR None. We simplify our typing by converting None to an empty list. - ''' + """ if contributors is None: return [] return contributors -class MitreAttackEnrichment(BaseModel): - ConfigDict(extra='forbid') +class MitreAttackEnrichment(BaseModel): + ConfigDict(extra="forbid") mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...) mitre_attack_technique: str = Field(...) mitre_attack_tactics: List[MitreTactics] = Field(...) mitre_attack_groups: List[str] = Field(...) - #Exclude this field from serialization - it is very large and not useful in JSON objects + # Exclude this field from serialization - it is very large and not useful in JSON objects mitre_attack_group_objects: list[MitreAttackGroup] = Field(..., exclude=True) + def __hash__(self) -> int: return id(self) - diff --git a/contentctl/objects/notable_action.py b/contentctl/objects/notable_action.py index 7492a3ec..1310875f 100644 --- a/contentctl/objects/notable_action.py +++ b/contentctl/objects/notable_action.py @@ -14,6 +14,7 @@ class NotableAction(BaseModel): :param security_domain: the domain associated with the notable action and related rule (detection/search) :param severity: severity (e.g. "high") associated with the notable action and related rule (detection/search) """ + rule_name: str rule_description: str security_domain: str @@ -32,5 +33,5 @@ def parse_from_dict(cls, dict_: dict[str, Any]) -> "NotableAction": rule_name=dict_["action.notable.param.rule_title"], rule_description=dict_["action.notable.param.rule_description"], security_domain=dict_["action.notable.param.security_domain"], - severity=dict_["action.notable.param.severity"] + severity=dict_["action.notable.param.severity"], ) diff --git a/contentctl/objects/notable_event.py b/contentctl/objects/notable_event.py index 51b9715d..31f8a959 100644 --- a/contentctl/objects/notable_event.py +++ b/contentctl/objects/notable_event.py @@ -13,9 +13,7 @@ class NotableEvent(BaseModel): # Allowing fields that aren't explicitly defined to be passed since some of the risk event's # fields vary depending on the SPL which generated them - model_config = ConfigDict( - extra='allow' - ) + model_config = ConfigDict(extra="allow") def validate_against_detection(self, detection: Detection) -> None: raise NotImplementedError() diff --git a/contentctl/objects/observable.py b/contentctl/objects/observable.py index 35eb535a..e7b1f58b 100644 --- a/contentctl/objects/observable.py +++ b/contentctl/objects/observable.py @@ -1,21 +1,25 @@ from pydantic import BaseModel, field_validator, ConfigDict -from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, RBA_OBSERVABLE_ROLE_MAPPING +from contentctl.objects.constants import ( + SES_OBSERVABLE_TYPE_MAPPING, + RBA_OBSERVABLE_ROLE_MAPPING, +) # TODO (cmcginley): should this class be removed? + class Observable(BaseModel): model_config = ConfigDict(extra="forbid") name: str type: str role: list[str] - @field_validator('name') + @field_validator("name") def check_name(cls, v: str): if v == "": raise ValueError("No name provided for observable") return v - @field_validator('type') + @field_validator("type") def check_type(cls, v: str): if v not in SES_OBSERVABLE_TYPE_MAPPING.keys(): raise ValueError( @@ -24,7 +28,7 @@ def check_type(cls, v: str): ) return v - @field_validator('role') + @field_validator("role") def check_roles(cls, v: list[str]): if len(v) == 0: raise ValueError("Error, at least 1 role must be listed for Observable.") diff --git a/contentctl/objects/playbook.py b/contentctl/objects/playbook.py index 6d9290bf..b3c9f483 100644 --- a/contentctl/objects/playbook.py +++ b/contentctl/objects/playbook.py @@ -10,57 +10,59 @@ class Playbook(SecurityContentObject): type: PlaybookType = Field(...) - + # Override the type definition for filePath. # This MUST be backed by a file and cannot be None file_path: FilePath - + how_to_implement: str = Field(min_length=4) playbook: str = Field(min_length=4) - app_list: list[str] = Field(...,min_length=0) + app_list: list[str] = Field(..., min_length=0) tags: PlaybookTag = Field(...) - - @model_validator(mode="after") - def ensureJsonAndPyFilesExist(self)->Self: + def ensureJsonAndPyFilesExist(self) -> Self: json_file_path = self.file_path.with_suffix(".json") python_file_path = self.file_path.with_suffix(".py") - missing:list[str] = [] + missing: list[str] = [] if not json_file_path.is_file(): - missing.append(f"Playbook file named '{self.file_path.name}' MUST "\ - f"have a .json file named '{json_file_path.name}', "\ - "but it does not exist") - + missing.append( + f"Playbook file named '{self.file_path.name}' MUST " + f"have a .json file named '{json_file_path.name}', " + "but it does not exist" + ) + if not python_file_path.is_file(): - missing.append(f"Playbook file named '{self.file_path.name}' MUST "\ - f"have a .py file named '{python_file_path.name}', "\ - "but it does not exist") - - + missing.append( + f"Playbook file named '{self.file_path.name}' MUST " + f"have a .py file named '{python_file_path.name}', " + "but it does not exist" + ) + if len(missing) == 0: return self else: - missing_files_string = '\n - '.join(missing) + missing_files_string = "\n - ".join(missing) raise ValueError(f"Playbook files missing:\n -{missing_files_string}") - - #Override playbook file name checking FOR NOW + # Override playbook file name checking FOR NOW @model_validator(mode="after") - def ensureFileNameMatchesSearchName(self)->Self: - file_name = self.name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ - .lower() + ".yml" - - #allow different capitalization FOR NOW in playbook file names - if (self.file_path is not None and file_name != self.file_path.name.lower()): - raise ValueError(f"The file name MUST be based off the content 'name' field:\n"\ - f"\t- Expected File Name: {file_name}\n"\ - f"\t- Actual File Name : {self.file_path.name}") + def ensureFileNameMatchesSearchName(self) -> Self: + file_name = ( + self.name.replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") + .lower() + + ".yml" + ) - return self + # allow different capitalization FOR NOW in playbook file names + if self.file_path is not None and file_name != self.file_path.name.lower(): + raise ValueError( + f"The file name MUST be based off the content 'name' field:\n" + f"\t- Expected File Name: {file_name}\n" + f"\t- Actual File Name : {self.file_path.name}" + ) - \ No newline at end of file + return self diff --git a/contentctl/objects/playbook_tags.py b/contentctl/objects/playbook_tags.py index 299618fd..c58cc2cd 100644 --- a/contentctl/objects/playbook_tags.py +++ b/contentctl/objects/playbook_tags.py @@ -1,26 +1,31 @@ from __future__ import annotations from typing import Optional, List -from pydantic import BaseModel, Field,ConfigDict +from pydantic import BaseModel, Field, ConfigDict import enum from contentctl.objects.detection import Detection -class PlaybookProduct(str,enum.Enum): +class PlaybookProduct(str, enum.Enum): SPLUNK_SOAR = "Splunk SOAR" -class PlaybookUseCase(str,enum.Enum): + +class PlaybookUseCase(str, enum.Enum): PHISHING = "Phishing" ENDPOINT = "Endpoint" ENRICHMENT = "Enrichment" - -class PlaybookType(str,enum.Enum): + + +class PlaybookType(str, enum.Enum): INPUT = "Input" AUTOMATION = "Automation" -class VpeType(str,enum.Enum): + +class VpeType(str, enum.Enum): MODERN = "Modern" CLASSIC = "Classic" -class DefendTechnique(str,enum.Enum): + + +class DefendTechnique(str, enum.Enum): D3_AL = "D3-AL" D3_DNSDL = "D3-DNSDL" D3_DA = "D3-DA" @@ -35,20 +40,21 @@ class DefendTechnique(str,enum.Enum): D3_FHRA = "D3-FHRA" D3_SRA = "D3-SRA" D3_RUAA = "D3-RUAA" + + class PlaybookTag(BaseModel): model_config = ConfigDict(extra="forbid") analytic_story: Optional[list] = None detections: Optional[list] = None - platform_tags: list[str] = Field(...,min_length=0) + platform_tags: list[str] = Field(..., min_length=0) playbook_type: PlaybookType = Field(...) vpe_type: VpeType = Field(...) playbook_fields: list[str] = Field([], min_length=0) - product: list[PlaybookProduct] = Field([],min_length=0) - use_cases: list[PlaybookUseCase] = Field([],min_length=0) + product: list[PlaybookProduct] = Field([], min_length=0) + use_cases: list[PlaybookUseCase] = Field([], min_length=0) defend_technique_id: Optional[List[DefendTechnique]] = None - - labels:list[str] = [] - playbook_outputs:list[str] = [] - + + labels: list[str] = [] + playbook_outputs: list[str] = [] + detection_objects: list[Detection] = [] - \ No newline at end of file diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index d1581e0f..d33da47c 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -7,11 +7,13 @@ RiskScoreValue_Type = Annotated[int, Field(ge=1, le=100)] + class RiskObjectType(str, Enum): SYSTEM = "system" USER = "user" OTHER = "other" + class ThreatObjectType(str, Enum): CERTIFICATE_COMMON_NAME = "certificate_common_name" CERTIFICATE_ORGANIZATION = "certificate_organization" @@ -40,6 +42,7 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" + class RiskObject(BaseModel): field: str type: RiskObjectType @@ -48,6 +51,7 @@ class RiskObject(BaseModel): def __hash__(self): return hash((self.field, self.type, self.score)) + class ThreatObject(BaseModel): field: str type: ThreatObjectType @@ -55,26 +59,27 @@ class ThreatObject(BaseModel): def __hash__(self): return hash((self.field, self.type)) + class RBAObject(BaseModel, ABC): message: str risk_objects: Annotated[Set[RiskObject], Field(min_length=1)] threat_objects: Set[ThreatObject] - - @computed_field @property - def risk_score(self)->RiskScoreValue_Type: + def risk_score(self) -> RiskScoreValue_Type: # First get the maximum score associated with # a risk object. If there are no objects, then # we should throw an exception. if len(self.risk_objects) == 0: - raise Exception("There must be at least one Risk Object present to get Severity.") + raise Exception( + "There must be at least one Risk Object present to get Severity." + ) return max([risk_object.score for risk_object in self.risk_objects]) - + @computed_field @property - def severity(self)->RiskSeverity: + def severity(self) -> RiskSeverity: if 0 <= self.risk_score <= 20: return RiskSeverity.INFORMATIONAL elif 20 < self.risk_score <= 40: @@ -86,5 +91,6 @@ def severity(self)->RiskSeverity: elif 80 < self.risk_score <= 100: return RiskSeverity.CRITICAL else: - raise Exception(f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}") - + raise Exception( + f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}" + ) diff --git a/contentctl/objects/risk_analysis_action.py b/contentctl/objects/risk_analysis_action.py index 2fa295e4..aa0d2c89 100644 --- a/contentctl/objects/risk_analysis_action.py +++ b/contentctl/objects/risk_analysis_action.py @@ -18,6 +18,7 @@ class RiskAnalysisAction(BaseModel): :param message: the message associated w/ the risk event (NOTE: may contain macros of the form $...$ which should be replaced with real values in the resulting risk events) """ + risk_objects: list[RiskObject] message: str @@ -80,21 +81,24 @@ def parse_from_dict(cls, dict_: dict[str, Any]) -> "RiskAnalysisAction": # TODO (#231): add validation ensuring at least 1 risk objects for entry in object_dicts: if "risk_object_field" in entry: - risk_objects.append(RiskObject( - field=entry["risk_object_field"], - type=entry["risk_object_type"], - score=int(entry["risk_score"]) - )) + risk_objects.append( + RiskObject( + field=entry["risk_object_field"], + type=entry["risk_object_type"], + score=int(entry["risk_score"]), + ) + ) elif "threat_object_field" in entry: - threat_objects.append(ThreatObject( - field=entry["threat_object_field"], - type=entry["threat_object_type"] - )) + threat_objects.append( + ThreatObject( + field=entry["threat_object_field"], + type=entry["threat_object_type"], + ) + ) else: raise ValueError( f"Unexpected object within 'action.risk.param._risk': {entry}" ) return cls( - risk_objects=risk_objects, - message=dict_["action.risk.param._risk_message"] + risk_objects=risk_objects, message=dict_["action.risk.param._risk_message"] ) diff --git a/contentctl/objects/risk_event.py b/contentctl/objects/risk_event.py index 71ef3ed0..faf74fe7 100644 --- a/contentctl/objects/risk_event.py +++ b/contentctl/objects/risk_event.py @@ -1,7 +1,14 @@ import re from functools import cached_property -from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field +from pydantic import ( + ConfigDict, + BaseModel, + Field, + PrivateAttr, + field_validator, + computed_field, +) from contentctl.objects.errors import ValidationFailed from contentctl.objects.detection import Detection from contentctl.objects.rba import RiskObject @@ -35,8 +42,7 @@ class RiskEvent(BaseModel): # The MITRE ATT&CK IDs annotations_mitre_attack: list[str] = Field( - alias="annotations.mitre_attack", - default=[] + alias="annotations.mitre_attack", default=[] ) # Contributing events search query (we use this to derive the corresponding field from the @@ -48,9 +54,7 @@ class RiskEvent(BaseModel): # Allowing fields that aren't explicitly defined to be passed since some of the risk event's # fields vary depending on the SPL which generated them - model_config = ConfigDict( - extra="allow" - ) + model_config = ConfigDict(extra="allow") @field_validator("annotations_mitre_attack", "analyticstories", mode="before") @classmethod @@ -72,7 +76,9 @@ def source_field_name(self) -> str: event(s). Useful for mapping back to a risk object in the detection. """ pattern = re.compile( - r"\| savedsearch \"" + self.search_name + r"\" \| search (?P[^=]+)=.+" + r"\| savedsearch \"" + + self.search_name + + r"\" \| search (?P[^=]+)=.+" ) match = pattern.search(self.contributing_events_search) if match is None: @@ -121,7 +127,9 @@ def validate_mitre_ids(self, detection: Detection) -> None: :param detection: the detection associated w/ this risk event :raises: ValidationFailed """ - if sorted(self.annotations_mitre_attack) != sorted(detection.tags.mitre_attack_id): + if sorted(self.annotations_mitre_attack) != sorted( + detection.tags.mitre_attack_id + ): raise ValidationFailed( f"MITRE ATT&CK IDs in risk event ({self.annotations_mitre_attack}) do not match those" f" in detection ({detection.tags.mitre_attack_id})." @@ -134,7 +142,9 @@ def validate_analyticstories(self, detection: Detection) -> None: :raises: ValidationFailed """ # Render the detection analytic_story to a list of strings before comparing - detection_analytic_story = [story.name for story in detection.tags.analytic_story] + detection_analytic_story = [ + story.name for story in detection.tags.analytic_story + ] if sorted(self.analyticstories) != sorted(detection_analytic_story): raise ValidationFailed( f"Analytic stories in risk event ({self.analyticstories}) do not match those" @@ -174,16 +184,12 @@ def validate_risk_message(self, detection: Detection) -> None: # placeholder tmp_placeholder = "PLACEHOLDERPATTERNFORESCAPING" escaped_source_message_with_placeholder: str = re.escape( - field_replacement_pattern.sub( - tmp_placeholder, - detection.rba.message - ) + field_replacement_pattern.sub(tmp_placeholder, detection.rba.message) ) placeholder_replacement_pattern = re.compile(tmp_placeholder) final_risk_message_pattern = re.compile( placeholder_replacement_pattern.sub( - r"[\\s\\S]*\\S[\\s\\S]*", - escaped_source_message_with_placeholder + r"[\\s\\S]*\\S[\\s\\S]*", escaped_source_message_with_placeholder ) ) @@ -191,8 +197,8 @@ def validate_risk_message(self, detection: Detection) -> None: if final_risk_message_pattern.match(self.risk_message) is None: raise ValidationFailed( "Risk message in event does not match the pattern set by the detection. Message in " - f"risk event: \"{self.risk_message}\". Message in detection: " - f"\"{detection.rba.message}\"." + f'risk event: "{self.risk_message}". Message in detection: ' + f'"{detection.rba.message}".' ) def validate_risk_against_risk_objects(self, risk_objects: set[RiskObject]) -> None: diff --git a/contentctl/objects/risk_object.py b/contentctl/objects/risk_object.py index 61955c93..fa965666 100644 --- a/contentctl/objects/risk_object.py +++ b/contentctl/objects/risk_object.py @@ -13,6 +13,7 @@ class RiskObject(BaseModel): :param type_: the type of the risk object (e.g. "system") :param score: the risk score associated with the obersevable (e.g. 64) """ + field: str type: str score: int diff --git a/contentctl/objects/savedsearches_conf.py b/contentctl/objects/savedsearches_conf.py index 79e559c8..582ec5d9 100644 --- a/contentctl/objects/savedsearches_conf.py +++ b/contentctl/objects/savedsearches_conf.py @@ -1,4 +1,3 @@ - from pathlib import Path from typing import Any, ClassVar import re @@ -17,6 +16,7 @@ class SavedsearchesConf(BaseModel): NOTE: At present, this model only parses the detections themselves from the .conf; thing like baselines or response tasks are left alone currently """ + # The path to the conf file path: Path = Field(...) @@ -112,8 +112,7 @@ def section_end(self) -> None: # Build the stanza model from the accumulated lines and adjust the state to end this section self.detection_stanzas[self._current_section_name] = DetectionStanza( - name=self._current_section_name, - lines=self._current_section_lines + name=self._current_section_name, lines=self._current_section_lines ) self._in_section = False @@ -170,7 +169,9 @@ def _parse_detection_stanzas(self) -> None: self._in_detections = True @staticmethod - def init_from_package(package_path: Path, app_name: str, appid: str) -> "SavedsearchesConf": + def init_from_package( + package_path: Path, app_name: str, appid: str + ) -> "SavedsearchesConf": """ Alternate constructor which can take an app package, and extract the savedsearches.conf from a temporary file. @@ -188,9 +189,10 @@ def init_from_package(package_path: Path, app_name: str, appid: str) -> "Savedse # Open the tar/gzip archive with tarfile.open(package_path) as package: # Extract the savedsearches.conf and use it to init the model - package_conf_path = SavedsearchesConf.PACKAGE_CONF_PATH_FMT_STR.format(appid=appid) + package_conf_path = SavedsearchesConf.PACKAGE_CONF_PATH_FMT_STR.format( + appid=appid + ) package.extract(package_conf_path, path=tmpdir) return SavedsearchesConf( - path=Path(tmpdir, package_conf_path), - app_label=app_name + path=Path(tmpdir, package_conf_path), app_label=app_name ) diff --git a/contentctl/objects/security_content_object.py b/contentctl/objects/security_content_object.py index a2f87364..02d94eb0 100644 --- a/contentctl/objects/security_content_object.py +++ b/contentctl/objects/security_content_object.py @@ -1,5 +1,8 @@ from __future__ import annotations -from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract +from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import ( + SecurityContentObject_Abstract, +) + class SecurityContentObject(SecurityContentObject_Abstract): - pass \ No newline at end of file + pass diff --git a/contentctl/objects/story.py b/contentctl/objects/story.py index 09d65287..9230e4b3 100644 --- a/contentctl/objects/story.py +++ b/contentctl/objects/story.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING,List +from typing import TYPE_CHECKING, List from contentctl.objects.story_tags import StoryTags -from pydantic import Field, model_serializer,computed_field, model_validator +from pydantic import Field, model_serializer, computed_field, model_validator import re + if TYPE_CHECKING: from contentctl.objects.detection import Detection from contentctl.objects.investigation import Investigation @@ -12,88 +13,90 @@ from contentctl.objects.security_content_object import SecurityContentObject + class Story(SecurityContentObject): narrative: str = Field(...) tags: StoryTags = Field(...) # These are updated when detection and investigation objects are created. # Specifically in the model_post_init functions - detections:List[Detection] = [] + detections: List[Detection] = [] investigations: List[Investigation] = [] baselines: List[Baseline] = [] - - + @computed_field @property - def data_sources(self)-> list[DataSource]: + def data_sources(self) -> list[DataSource]: # Only add a data_source if it does not already exist in the story - data_source_objects:set[DataSource] = set() + data_source_objects: set[DataSource] = set() for detection in self.detections: data_source_objects.update(set(detection.data_source_objects)) - + return sorted(list(data_source_objects)) + def storyAndInvestigationNamesWithApp(self, app: CustomApp) -> List[str]: + return [ + detection.get_conf_stanza_name(app) for detection in self.detections + ] + [ + investigation.get_response_task_name(app) + for investigation in self.investigations + ] - def storyAndInvestigationNamesWithApp(self, app:CustomApp)->List[str]: - return [detection.get_conf_stanza_name(app) for detection in self.detections] + \ - [investigation.get_response_task_name(app) for investigation in self.investigations] - @model_serializer def serialize_model(self): - #Call serializer for parent + # Call serializer for parent super_fields = super().serialize_model() - - #All fields custom to this model - model= { + + # All fields custom to this model + model = { "narrative": self.narrative, "tags": self.tags.model_dump(), "detection_names": self.detection_names, "investigation_names": self.investigation_names, "baseline_names": self.baseline_names, "author_company": self.author_company, - "author_name":self.author_name + "author_name": self.author_name, } detections = [] for detection in self.detections: new_detection = { - "name":detection.name, - "source":detection.source, - "type":detection.type + "name": detection.name, + "source": detection.source, + "type": detection.type, } if self.tags.mitre_attack_enrichments is not None: - new_detection['tags'] = {"mitre_attack_enrichments": [{"mitre_attack_technique": enrichment.mitre_attack_technique} for enrichment in detection.tags.mitre_attack_enrichments]} + new_detection["tags"] = { + "mitre_attack_enrichments": [ + {"mitre_attack_technique": enrichment.mitre_attack_technique} + for enrichment in detection.tags.mitre_attack_enrichments + ] + } detections.append(new_detection) - model['detections'] = detections - #Combine fields from this model with fields from parent + model["detections"] = detections + # Combine fields from this model with fields from parent super_fields.update(model) - - #return the model + + # return the model return super_fields @model_validator(mode="after") def setTagsFields(self): - enrichments = [] for detection in self.detections: enrichments.extend(detection.tags.mitre_attack_enrichments) self.tags.mitre_attack_enrichments = list(set(enrichments)) - tactics = [] for enrichment in self.tags.mitre_attack_enrichments: tactics.extend(enrichment.mitre_attack_tactics) self.tags.mitre_attack_tactics = set(tactics) - - datamodels = [] for detection in self.detections: datamodels.extend(detection.datamodel) self.tags.datamodels = set(datamodels) - - kill_chain_phases = [] for detection in self.detections: kill_chain_phases.extend(detection.tags.kill_chain_phases) @@ -101,42 +104,40 @@ def setTagsFields(self): return self - @computed_field @property - def author_name(self)->str: - match_author = re.search(r'^([^,]+)', self.author) + def author_name(self) -> str: + match_author = re.search(r"^([^,]+)", self.author) if match_author is None: - return 'no' + return "no" else: return match_author.group(1) @computed_field @property - def author_company(self)->str: - match_company = re.search(r',\s?(.*)$', self.author) + def author_company(self) -> str: + match_company = re.search(r",\s?(.*)$", self.author) if match_company is None: - return 'no' + return "no" else: return match_company.group(1) @computed_field @property - def author_email(self)->str: + def author_email(self) -> str: return "-" @computed_field @property - def detection_names(self)->List[str]: + def detection_names(self) -> List[str]: return [detection.name for detection in self.detections] - + @computed_field @property - def investigation_names(self)->List[str]: + def investigation_names(self) -> List[str]: return [investigation.name for investigation in self.investigations] @computed_field @property - def baseline_names(self)->List[str]: + def baseline_names(self) -> List[str]: return [baseline.name for baseline in self.baselines] - diff --git a/contentctl/objects/story_tags.py b/contentctl/objects/story_tags.py index 5f5e4dfb..e611c596 100644 --- a/contentctl/objects/story_tags.py +++ b/contentctl/objects/story_tags.py @@ -1,54 +1,66 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_serializer, ConfigDict -from typing import List,Set,Optional +from typing import List, Set, Optional from enum import Enum from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment -from contentctl.objects.enums import StoryCategory, DataModel, KillChainPhase, SecurityContentProductName -from contentctl.objects.annotated_types import CVE_TYPE,MITRE_ATTACK_ID_TYPE +from contentctl.objects.enums import ( + StoryCategory, + DataModel, + KillChainPhase, + SecurityContentProductName, +) +from contentctl.objects.annotated_types import CVE_TYPE, MITRE_ATTACK_ID_TYPE -class StoryUseCase(str,Enum): - FRAUD_DETECTION = "Fraud Detection" - COMPLIANCE = "Compliance" - APPLICATION_SECURITY = "Application Security" - SECURITY_MONITORING = "Security Monitoring" - ADVANCED_THREAD_DETECTION = "Advanced Threat Detection" - INSIDER_THREAT = "Insider Threat" - OTHER = "Other" + +class StoryUseCase(str, Enum): + FRAUD_DETECTION = "Fraud Detection" + COMPLIANCE = "Compliance" + APPLICATION_SECURITY = "Application Security" + SECURITY_MONITORING = "Security Monitoring" + ADVANCED_THREAD_DETECTION = "Advanced Threat Detection" + INSIDER_THREAT = "Insider Threat" + OTHER = "Other" class StoryTags(BaseModel): - model_config = ConfigDict(extra='forbid') - category: List[StoryCategory] = Field(...,min_length=1) - product: List[SecurityContentProductName] = Field(...,min_length=1) - usecase: StoryUseCase = Field(...) - - # enrichment - mitre_attack_enrichments: Optional[List[MitreAttackEnrichment]] = None - mitre_attack_tactics: Optional[Set[MITRE_ATTACK_ID_TYPE]] = None - datamodels: Optional[Set[DataModel]] = None - kill_chain_phases: Optional[Set[KillChainPhase]] = None - cve: List[CVE_TYPE] = [] - group: List[str] = Field([], description="A list of groups who leverage the techniques list in this Analytic Story.") - - def getCategory_conf(self) -> str: - #if len(self.category) > 1: - # print("Story with more than 1 category. We can only have 1 category, fix it!") - return list(self.category)[0] - - @model_serializer - def serialize_model(self): - #no super to call - return { - "category": list(self.category), - "product": list(self.product), - "usecase": self.usecase, - "mitre_attack_enrichments": self.mitre_attack_enrichments, - "mitre_attack_tactics": list(self.mitre_attack_tactics) if self.mitre_attack_tactics is not None else None, - "datamodels": list(self.datamodels) if self.datamodels is not None else None, - "kill_chain_phases": list(self.kill_chain_phases) if self.kill_chain_phases is not None else None - } - - - \ No newline at end of file + model_config = ConfigDict(extra="forbid") + category: List[StoryCategory] = Field(..., min_length=1) + product: List[SecurityContentProductName] = Field(..., min_length=1) + usecase: StoryUseCase = Field(...) + + # enrichment + mitre_attack_enrichments: Optional[List[MitreAttackEnrichment]] = None + mitre_attack_tactics: Optional[Set[MITRE_ATTACK_ID_TYPE]] = None + datamodels: Optional[Set[DataModel]] = None + kill_chain_phases: Optional[Set[KillChainPhase]] = None + cve: List[CVE_TYPE] = [] + group: List[str] = Field( + [], + description="A list of groups who leverage the techniques list in this Analytic Story.", + ) + + def getCategory_conf(self) -> str: + # if len(self.category) > 1: + # print("Story with more than 1 category. We can only have 1 category, fix it!") + return list(self.category)[0] + + @model_serializer + def serialize_model(self): + # no super to call + return { + "category": list(self.category), + "product": list(self.product), + "usecase": self.usecase, + "mitre_attack_enrichments": self.mitre_attack_enrichments, + "mitre_attack_tactics": list(self.mitre_attack_tactics) + if self.mitre_attack_tactics is not None + else None, + "datamodels": list(self.datamodels) + if self.datamodels is not None + else None, + "kill_chain_phases": list(self.kill_chain_phases) + if self.kill_chain_phases is not None + else None, + } diff --git a/contentctl/objects/test_group.py b/contentctl/objects/test_group.py index 6fb76c7d..cd651f54 100644 --- a/contentctl/objects/test_group.py +++ b/contentctl/objects/test_group.py @@ -15,13 +15,16 @@ class TestGroup(BaseModel): :param integration_test: an IntegrationTest :param attack_data: the attack data associated with tests in the TestGroup """ + name: str unit_test: UnitTest integration_test: IntegrationTest attack_data: list[TestAttackData] @classmethod - def derive_from_unit_test(cls, unit_test: UnitTest, name_prefix: str) -> "TestGroup": + def derive_from_unit_test( + cls, unit_test: UnitTest, name_prefix: str + ) -> "TestGroup": """ Given a UnitTest and a prefix, construct a TestGroup, with in IntegrationTest corresponding to the UnitTest :param unit_test: the UnitTest @@ -36,7 +39,7 @@ def derive_from_unit_test(cls, unit_test: UnitTest, name_prefix: str) -> "TestGr name=f"{name_prefix}:{unit_test.name}", unit_test=unit_test, integration_test=integration_test, - attack_data=unit_test.attack_data + attack_data=unit_test.attack_data, ) def unit_test_skipped(self) -> bool: diff --git a/contentctl/objects/threat_object.py b/contentctl/objects/threat_object.py index 020c4b7a..458fe272 100644 --- a/contentctl/objects/threat_object.py +++ b/contentctl/objects/threat_object.py @@ -11,5 +11,6 @@ class ThreatObject(BaseModel): :param field: the name of the field from which the risk object will get it's name :param type_: the type of the risk object (e.g. "system") """ + field: str type: str diff --git a/contentctl/objects/throttling.py b/contentctl/objects/throttling.py index 04998ac6..de6f9cd9 100644 --- a/contentctl/objects/throttling.py +++ b/contentctl/objects/throttling.py @@ -2,25 +2,34 @@ from typing import Annotated -# Alert Suppression/Throttling settings have been taken from +# Alert Suppression/Throttling settings have been taken from # https://docs.splunk.com/Documentation/Splunk/9.2.2/Admin/Savedsearchesconf class Throttling(BaseModel): - fields: list[str] = Field(..., description="The list of fields to throttle on. These fields MUST occur in the search.", min_length=1) - period: Annotated[str,Field(pattern="^[0-9]+[smh]$")] = Field(..., description="How often the alert should be triggered. " - "This may be specified in seconds, minutes, or hours. " - "For example, if an alert should be triggered once a day," - " it may be specified in seconds (86400s), minutes (1440m), or hours import (24h).") - + fields: list[str] = Field( + ..., + description="The list of fields to throttle on. These fields MUST occur in the search.", + min_length=1, + ) + period: Annotated[str, Field(pattern="^[0-9]+[smh]$")] = Field( + ..., + description="How often the alert should be triggered. " + "This may be specified in seconds, minutes, or hours. " + "For example, if an alert should be triggered once a day," + " it may be specified in seconds (86400s), minutes (1440m), or hours import (24h).", + ) + @field_validator("fields") - def no_spaces_in_fields(cls, v:list[str])->list[str]: + def no_spaces_in_fields(cls, v: list[str]) -> list[str]: for field in v: - if ' ' in field: - raise ValueError("Spaces are not presently supported in 'alert.suppress.fields' / throttling fields in conf files. " - "The field '{field}' has a space in it. If this is a blocker, please raise this as an issue on the Project.") + if " " in field: + raise ValueError( + "Spaces are not presently supported in 'alert.suppress.fields' / throttling fields in conf files. " + "The field '{field}' has a space in it. If this is a blocker, please raise this as an issue on the Project." + ) return v - def conf_formatted_fields(self)->str: - ''' + def conf_formatted_fields(self) -> str: + """ TODO: The field alert.suppress.fields is defined as follows: alert.suppress.fields = @@ -28,19 +37,19 @@ def conf_formatted_fields(self)->str: be specified if the digest mode is disabled and suppression is enabled. In order to support fields with spaces in them, we may need to wrap each - field in "". + field in "". This function returns a properly formatted value, where each field - is wrapped in "" and separated with a comma. For example, the fields + is wrapped in "" and separated with a comma. For example, the fields ["field1", "field 2", "field3"] would be returned as the string "field1","field 2","field3 However, for now, we will error on fields with spaces and simply separate with commas - ''' - + """ + return ",".join(self.fields) # The following may be used once we determine proper support # for fields with spaces - #return ",".join([f'"{field}"' for field in self.fields]) \ No newline at end of file + # return ",".join([f'"{field}"' for field in self.fields]) diff --git a/contentctl/objects/unit_test.py b/contentctl/objects/unit_test.py index a07b532b..1bfc6308 100644 --- a/contentctl/objects/unit_test.py +++ b/contentctl/objects/unit_test.py @@ -12,6 +12,7 @@ class UnitTest(BaseTest): """ A unit test for a detection """ + # contentType: SecurityContentType = SecurityContentType.unit_tests # The test type (unit) @@ -28,7 +29,6 @@ def skip(self, message: str) -> None: Skip the test by setting its result status :param message: the reason for skipping """ - self.result = UnitTestResult( # type: ignore - message=message, - status=TestResultStatus.SKIP + self.result = UnitTestResult( # type: ignore + message=message, status=TestResultStatus.SKIP ) diff --git a/contentctl/objects/unit_test_baseline.py b/contentctl/objects/unit_test_baseline.py index 66a60594..ca7e2a3e 100644 --- a/contentctl/objects/unit_test_baseline.py +++ b/contentctl/objects/unit_test_baseline.py @@ -1,12 +1,11 @@ - - -from pydantic import BaseModel,ConfigDict +from pydantic import BaseModel, ConfigDict from typing import Union + class UnitTestBaseline(BaseModel): model_config = ConfigDict(extra="forbid") name: str file: str pass_condition: str - earliest_time: Union[str,None] = None - latest_time: Union[str,None] = None \ No newline at end of file + earliest_time: Union[str, None] = None + latest_time: Union[str, None] = None diff --git a/contentctl/objects/unit_test_result.py b/contentctl/objects/unit_test_result.py index 2a9f230d..56aa863a 100644 --- a/contentctl/objects/unit_test_result.py +++ b/contentctl/objects/unit_test_result.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Union,TYPE_CHECKING +from typing import Union, TYPE_CHECKING from splunklib.data import Record from contentctl.objects.base_test_result import BaseTestResult, TestResultStatus @@ -15,7 +15,7 @@ class UnitTestResult(BaseTestResult): missing_observables: list[str] = [] - + def set_job_content( self, content: Union[Record, None], @@ -40,7 +40,7 @@ def set_job_content( self.exception = exception self.status = status self.job_content = content - + # Set the job content, if given if content is not None: if self.status == TestResultStatus.PASS: @@ -50,7 +50,7 @@ def set_job_content( elif self.status == TestResultStatus.ERROR: self.message = "TEST ERROR" elif self.status == TestResultStatus.SKIP: - #A test that was SKIPPED should not have job content since it should not have been run. + # A test that was SKIPPED should not have job content since it should not have been run. self.message = "TEST SKIPPED" if not config.instance_address.startswith("http://"): @@ -64,7 +64,7 @@ def set_job_content( ) elif self.status == TestResultStatus.SKIP: - self.message = "TEST SKIPPED" + self.message = "TEST SKIPPED" pass elif content is None: diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index 87760373..80c66b23 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING + if TYPE_CHECKING: from contentctl.objects.detection import Detection from contentctl.objects.lookup import Lookup @@ -15,12 +16,11 @@ from contentctl.output.json_writer import JsonWriter - class ApiJsonOutput: output_path: pathlib.Path app_label: str - def __init__(self, output_path:pathlib.Path, app_label: str): + def __init__(self, output_path: pathlib.Path, app_label: str): self.output_path = output_path self.app_label = app_label @@ -53,7 +53,7 @@ def writeDetections( ) for detection in objects ] - #Only a subset of macro fields are required: + # Only a subset of macro fields are required: # for detection in detections: # new_macros = [] # for macro in detection.get("macros",[]): @@ -62,16 +62,15 @@ def writeDetections( # new_macro_fields["definition"] = macro.get("definition") # new_macro_fields["description"] = macro.get("description") # if len(macro.get("arguments", [])) > 0: - # new_macro_fields["arguments"] = macro.get("arguments") + # new_macro_fields["arguments"] = macro.get("arguments") # new_macros.append(new_macro_fields) # detection["macros"] = new_macros # del() - - + JsonWriter.writeJsonObject( os.path.join(self.output_path, "detections.json"), "detections", detections ) - + def writeMacros( self, objects: list[Macro], @@ -81,13 +80,13 @@ def writeMacros( for macro in objects ] for macro in macros: - for k in ["author", "date","version","id","references"]: + for k in ["author", "date", "version", "id", "references"]: if k in macro: - del(macro[k]) + del macro[k] JsonWriter.writeJsonObject( os.path.join(self.output_path, "macros.json"), "macros", macros ) - + def writeStories( self, objects: list[Story], @@ -126,8 +125,9 @@ def writeStories( } for detection in story["detections"] ] - story["detection_names"] = [f"{self.app_label} - {name} - Rule" for name in story["detection_names"]] - + story["detection_names"] = [ + f"{self.app_label} - {name} - Rule" for name in story["detection_names"] + ] JsonWriter.writeJsonObject( os.path.join(self.output_path, "stories.json"), "stories", stories @@ -159,10 +159,10 @@ def writeBaselines( ) for baseline in objects ] - + JsonWriter.writeJsonObject( - os.path.join(self.output_path, "baselines.json"), "baselines", baselines - ) + os.path.join(self.output_path, "baselines.json"), "baselines", baselines + ) def writeInvestigations( self, @@ -221,9 +221,9 @@ def writeLookups( for lookup in objects ] for lookup in lookups: - for k in ["author","date","version","id","references"]: + for k in ["author", "date", "version", "id", "references"]: if k in lookup: - del(lookup[k]) + del lookup[k] JsonWriter.writeJsonObject( os.path.join(self.output_path, "lookups.json"), "lookups", lookups ) @@ -244,16 +244,16 @@ def writeDeployments( "description", "scheduling", "rba", - "tags" - ] + "tags", + ] ) ) for deployment in objects ] - #references are not to be included, but have been deleted in the - #model_serialization logic + # references are not to be included, but have been deleted in the + # model_serialization logic JsonWriter.writeJsonObject( os.path.join(self.output_path, "deployments.json"), "deployments", deployments, - ) \ No newline at end of file + ) diff --git a/contentctl/output/attack_nav_output.py b/contentctl/output/attack_nav_output.py index e94abfde..13076b51 100644 --- a/contentctl/output/attack_nav_output.py +++ b/contentctl/output/attack_nav_output.py @@ -1,24 +1,25 @@ -from typing import List,Union +from typing import List, Union import pathlib from contentctl.objects.detection import Detection from contentctl.output.attack_nav_writer import AttackNavWriter -class AttackNavOutput(): - - def writeObjects(self, detections: List[Detection], output_path: pathlib.Path) -> None: - techniques:dict[str,dict[str,Union[List[str],int]]] = {} +class AttackNavOutput: + def writeObjects( + self, detections: List[Detection], output_path: pathlib.Path + ) -> None: + techniques: dict[str, dict[str, Union[List[str], int]]] = {} for detection in detections: for tactic in detection.tags.mitre_attack_id: if tactic not in techniques: - techniques[tactic] = {'score':0,'file_paths':[]} - + techniques[tactic] = {"score": 0, "file_paths": []} + detection_url = f"https://github.com/splunk/security_content/blob/develop/detections/{detection.source}/{detection.file_path.name}" - techniques[tactic]['score'] += 1 - techniques[tactic]['file_paths'].append(detection_url) - - ''' + techniques[tactic]["score"] += 1 + techniques[tactic]["file_paths"].append(detection_url) + + """ for detection in objects: if detection.tags.mitre_attack_enrichments: for mitre_attack_enrichment in detection.tags.mitre_attack_enrichments: @@ -30,16 +31,16 @@ def writeObjects(self, detections: List[Detection], output_path: pathlib.Path) - else: techniques[mitre_attack_enrichment.mitre_attack_id]['score'] = techniques[mitre_attack_enrichment.mitre_attack_id]['score'] + 1 techniques[mitre_attack_enrichment.mitre_attack_id]['file_paths'].append('https://github.com/splunk/security_content/blob/develop/detections/' + detection.getSource() + '/' + self.convertNameToFileName(detection.name)) - ''' - AttackNavWriter.writeAttackNavFile(techniques, output_path / 'coverage.json') - + """ + AttackNavWriter.writeAttackNavFile(techniques, output_path / "coverage.json") def convertNameToFileName(self, name: str): - file_name = name \ - .replace(' ', '_') \ - .replace('-','_') \ - .replace('.','_') \ - .replace('/','_') \ + file_name = ( + name.replace(" ", "_") + .replace("-", "_") + .replace(".", "_") + .replace("/", "_") .lower() - file_name = file_name + '.yml' + ) + file_name = file_name + ".yml" return file_name diff --git a/contentctl/output/attack_nav_writer.py b/contentctl/output/attack_nav_writer.py index 78e8c514..7d7be94f 100644 --- a/contentctl/output/attack_nav_writer.py +++ b/contentctl/output/attack_nav_writer.py @@ -1,75 +1,67 @@ - import json from typing import Union, List import pathlib + VERSION = "4.3" NAME = "Detection Coverage" DESCRIPTION = "security_content detection coverage" DOMAIN = "mitre-enterprise" -class AttackNavWriter(): - +class AttackNavWriter: @staticmethod - def writeAttackNavFile(mitre_techniques : dict[str,dict[str,Union[List[str],int]]], output_path : pathlib.Path) -> None: + def writeAttackNavFile( + mitre_techniques: dict[str, dict[str, Union[List[str], int]]], + output_path: pathlib.Path, + ) -> None: max_count = 0 for technique_id in mitre_techniques.keys(): - if mitre_techniques[technique_id]['score'] > max_count: - max_count = mitre_techniques[technique_id]['score'] - + if mitre_techniques[technique_id]["score"] > max_count: + max_count = mitre_techniques[technique_id]["score"] + layer_json = { "version": VERSION, "name": NAME, "description": DESCRIPTION, "domain": DOMAIN, - "techniques": [] + "techniques": [], } layer_json["gradient"] = { - "colors": [ - "#ffffff", - "#66b1ff", - "#096ed7" - ], + "colors": ["#ffffff", "#66b1ff", "#096ed7"], "minValue": 0, - "maxValue": max_count + "maxValue": max_count, } layer_json["filters"] = { - "platforms": - ["Windows", - "Linux", - "macOS", - "AWS", - "GCP", - "Azure", - "Office 365", - "SaaS" - ] + "platforms": [ + "Windows", + "Linux", + "macOS", + "AWS", + "GCP", + "Azure", + "Office 365", + "SaaS", + ] } layer_json["legendItems"] = [ - { - "label": "NO available detections", - "color": "#ffffff" - }, - { - "label": "Some detections available", - "color": "#66b1ff" - } + {"label": "NO available detections", "color": "#ffffff"}, + {"label": "Some detections available", "color": "#66b1ff"}, ] - layer_json['showTacticRowBackground'] = True - layer_json['tacticRowBackground'] = "#dddddd" + layer_json["showTacticRowBackground"] = True + layer_json["tacticRowBackground"] = "#dddddd" layer_json["sorting"] = 3 for technique_id in mitre_techniques.keys(): layer_technique = { "techniqueID": technique_id, - "score": mitre_techniques[technique_id]['score'], - "comment": "\n\n".join(mitre_techniques[technique_id]['file_paths']) + "score": mitre_techniques[technique_id]["score"], + "comment": "\n\n".join(mitre_techniques[technique_id]["file_paths"]), } layer_json["techniques"].append(layer_technique) - with open(output_path, 'w') as outfile: + with open(output_path, "w") as outfile: json.dump(layer_json, outfile, ensure_ascii=False, indent=4) diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index a59e9fa5..77640272 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, Callable + if TYPE_CHECKING: from contentctl.objects.detection import Detection from contentctl.objects.lookup import Lookup @@ -18,203 +19,248 @@ from contentctl.output.conf_writer import ConfWriter from contentctl.objects.config import build -class ConfOutput: - config: build +class ConfOutput: + config: build def __init__(self, config: build): self.config = config - #Create the build directory if it does not exist + # Create the build directory if it does not exist config.getPackageDirectoryPath().parent.mkdir(parents=True, exist_ok=True) - - #Remove the app path, if it exists + + # Remove the app path, if it exists shutil.rmtree(config.getPackageDirectoryPath(), ignore_errors=True) - - #Copy all the template files into the app + + # Copy all the template files into the app shutil.copytree(config.getAppTemplatePath(), config.getPackageDirectoryPath()) - def writeHeaders(self) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - for output_app_path in ['default/analyticstories.conf', - 'default/savedsearches.conf', - 'default/collections.conf', - 'default/es_investigations.conf', - 'default/macros.conf', - 'default/transforms.conf', - 'default/workflow_actions.conf', - 'default/app.conf', - 'default/content-version.conf']: - written_files.add(ConfWriter.writeConfFileHeader(pathlib.Path(output_app_path),self.config)) - + written_files: set[pathlib.Path] = set() + for output_app_path in [ + "default/analyticstories.conf", + "default/savedsearches.conf", + "default/collections.conf", + "default/es_investigations.conf", + "default/macros.conf", + "default/transforms.conf", + "default/workflow_actions.conf", + "default/app.conf", + "default/content-version.conf", + ]: + written_files.add( + ConfWriter.writeConfFileHeader( + pathlib.Path(output_app_path), self.config + ) + ) + return written_files - - #The contents of app.manifest are not a conf file, but json. - #DO NOT write a header for this file type, simply create the file - with open(self.config.getPackageDirectoryPath() / pathlib.Path('app.manifest'), 'w'): + # The contents of app.manifest are not a conf file, but json. + # DO NOT write a header for this file type, simply create the file + with open( + self.config.getPackageDirectoryPath() / pathlib.Path("app.manifest"), "w" + ): pass - - - - - def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - - written_files.add(ConfWriter.writeConfFile(pathlib.Path("default/content-version.conf"), - "content-version.j2", - self.config, - [self.config.app])) - - written_files.add(ConfWriter.writeManifestFile(pathlib.Path("app.manifest"), - "app.manifest.j2", - self.config, - [self.config.app])) - + + def writeMiscellaneousAppFiles(self) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path("default/content-version.conf"), + "content-version.j2", + self.config, + [self.config.app], + ) + ) + + written_files.add( + ConfWriter.writeManifestFile( + pathlib.Path("app.manifest"), + "app.manifest.j2", + self.config, + [self.config.app], + ) + ) + written_files.add(ConfWriter.writeServerConf(self.config)) written_files.add(ConfWriter.writeAppConf(self.config)) - return written_files - - def writeDetections(self, objects:list[Detection]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_detections.j2'), - ('default/analyticstories.conf', 'analyticstories_detections.j2')]: - written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, self.config, objects)) + def writeDetections(self, objects: list[Detection]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + for output_app_path, template_name in [ + ("default/savedsearches.conf", "savedsearches_detections.j2"), + ("default/analyticstories.conf", "analyticstories_detections.j2"), + ]: + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path(output_app_path), template_name, self.config, objects + ) + ) + return written_files + + def writeStories(self, objects: list[Story]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path("default/analyticstories.conf"), + "analyticstories_stories.j2", + self.config, + objects, + ) + ) + return written_files + + def writeBaselines(self, objects: list[Baseline]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path("default/savedsearches.conf"), + "savedsearches_baselines.j2", + self.config, + objects, + ) + ) return written_files + def writeInvestigations(self, objects: list[Investigation]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + for output_app_path, template_name in [ + ("default/savedsearches.conf", "savedsearches_investigations.j2"), + ("default/analyticstories.conf", "analyticstories_investigations.j2"), + ]: + ConfWriter.writeConfFile( + pathlib.Path(output_app_path), template_name, self.config, objects + ) + + workbench_panels: list[Investigation] = [] + for investigation in objects: + if investigation.inputs: + response_file_name_xml = ( + investigation.lowercase_name + "___response_task.xml" + ) + workbench_panels.append(investigation) + investigation.search = investigation.search.replace(">", ">") + investigation.search = investigation.search.replace("<", "<") + + ConfWriter.writeXmlFileHeader( + pathlib.Path( + f"default/data/ui/panels/workbench_panel_{response_file_name_xml}" + ), + self.config, + ) + + ConfWriter.writeXmlFile( + pathlib.Path( + f"default/data/ui/panels/workbench_panel_{response_file_name_xml}" + ), + "panel.j2", + self.config, + [investigation.search], + ) + + for output_app_path, template_name in [ + ("default/es_investigations.conf", "es_investigations_investigations.j2"), + ("default/workflow_actions.conf", "workflow_actions.j2"), + ]: + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path(output_app_path), + template_name, + self.config, + workbench_panels, + ) + ) + return written_files + + def writeLookups(self, objects: list[Lookup]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + for output_app_path, template_name in [ + ("default/collections.conf", "collections.j2"), + ("default/transforms.conf", "transforms.j2"), + ]: + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path(output_app_path), template_name, self.config, objects + ) + ) + + # Get the path to the lookups folder + lookup_folder = self.config.getPackageDirectoryPath() / "lookups" + + # Make the new folder for the lookups + # This folder almost certainly already exists because mitre_enrichment.csv has been writtent here from the app template. + lookup_folder.mkdir(exist_ok=True) + + # Copy each lookup into the folder + for lookup in objects: + if isinstance(lookup, FileBackedLookup): + shutil.copy(lookup.filename, lookup_folder / lookup.app_filename.name) + return written_files + + def writeMacros(self, objects: list[Macro]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + written_files.add( + ConfWriter.writeConfFile( + pathlib.Path("default/macros.conf"), "macros.j2", self.config, objects + ) + ) + return written_files + + def writeDashboards(self, objects: list[Dashboard]) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() + written_files.update(ConfWriter.writeDashboardFiles(self.config, objects)) + return written_files - def writeStories(self, objects:list[Story]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/analyticstories.conf'), - 'analyticstories_stories.j2', - self.config, objects)) - return written_files - - - def writeBaselines(self, objects:list[Baseline]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/savedsearches.conf'), - 'savedsearches_baselines.j2', - self.config, objects)) - return written_files - - - def writeInvestigations(self, objects:list[Investigation]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_investigations.j2'), - ('default/analyticstories.conf', 'analyticstories_investigations.j2')]: - ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, - self.config, - objects) - - workbench_panels:list[Investigation] = [] - for investigation in objects: - if investigation.inputs: - response_file_name_xml = investigation.lowercase_name + "___response_task.xml" - workbench_panels.append(investigation) - investigation.search = investigation.search.replace(">",">") - investigation.search = investigation.search.replace("<","<") - - - ConfWriter.writeXmlFileHeader(pathlib.Path(f'default/data/ui/panels/workbench_panel_{response_file_name_xml}'), - self.config) - - ConfWriter.writeXmlFile( pathlib.Path(f'default/data/ui/panels/workbench_panel_{response_file_name_xml}'), - 'panel.j2', - self.config,[investigation.search]) - - for output_app_path, template_name in [ ('default/es_investigations.conf', 'es_investigations_investigations.j2'), - ('default/workflow_actions.conf', 'workflow_actions.j2')]: - written_files.add( ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, - self.config, - workbench_panels)) - return written_files - - - def writeLookups(self, objects:list[Lookup]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - for output_app_path, template_name in [ ('default/collections.conf', 'collections.j2'), - ('default/transforms.conf', 'transforms.j2')]: - written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path), - template_name, - self.config, - objects)) - - #Get the path to the lookups folder - lookup_folder = self.config.getPackageDirectoryPath()/"lookups" - - # Make the new folder for the lookups - # This folder almost certainly already exists because mitre_enrichment.csv has been writtent here from the app template. - lookup_folder.mkdir(exist_ok=True) - - #Copy each lookup into the folder - for lookup in objects: - if isinstance(lookup, FileBackedLookup): - shutil.copy(lookup.filename, lookup_folder/lookup.app_filename.name) - return written_files - - - def writeMacros(self, objects:list[Macro]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/macros.conf'), - 'macros.j2', - self.config, objects)) - return written_files - - - def writeDashboards(self, objects:list[Dashboard]) -> set[pathlib.Path]: - written_files:set[pathlib.Path] = set() - written_files.update(ConfWriter.writeDashboardFiles(self.config, objects)) - return written_files - - def packageAppTar(self) -> None: - - with tarfile.open(self.config.getPackageFilePath(include_version=True), "w:gz") as app_archive: - app_archive.add(self.config.getPackageDirectoryPath(), arcname=self.config.getPackageDirectoryPath().name) - - shutil.copy2(self.config.getPackageFilePath(include_version=True), - self.config.getPackageFilePath(include_version=False), - follow_symlinks=False) - + with tarfile.open( + self.config.getPackageFilePath(include_version=True), "w:gz" + ) as app_archive: + app_archive.add( + self.config.getPackageDirectoryPath(), + arcname=self.config.getPackageDirectoryPath().name, + ) + + shutil.copy2( + self.config.getPackageFilePath(include_version=True), + self.config.getPackageFilePath(include_version=False), + follow_symlinks=False, + ) + def packageAppSlim(self) -> None: - - raise Exception("Packaging with splunk-packaging-toolkit not currently supported as slim only supports Python 3.7. " - "Please raise an issue in the contentctl GitHub if you encounter this exception.") + raise Exception( + "Packaging with splunk-packaging-toolkit not currently supported as slim only supports Python 3.7. " + "Please raise an issue in the contentctl GitHub if you encounter this exception." + ) try: import slim from slim.utils import SlimLogger import logging - #In order to avoid significant output, only emit FATAL log messages + + # In order to avoid significant output, only emit FATAL log messages SlimLogger.set_level(logging.ERROR) try: - slim.package(source=self.config.getPackageDirectoryPath(), output_dir=pathlib.Path(self.config.getBuildDir())) + slim.package( + source=self.config.getPackageDirectoryPath(), + output_dir=pathlib.Path(self.config.getBuildDir()), + ) except SystemExit as e: raise Exception(f"Error building package with slim: {str(e)}") - - + except Exception as e: - print("Failed to import Splunk Packaging Toolkit (slim). slim requires Python<3.10. " - "Packaging app with tar instead. This should still work, but appinspect may catch " - "errors that otherwise would have been flagged by slim.") + print( + "Failed to import Splunk Packaging Toolkit (slim). slim requires Python<3.10. " + "Packaging app with tar instead. This should still work, but appinspect may catch " + "errors that otherwise would have been flagged by slim." + ) raise Exception(f"slim (splunk packaging toolkit) not installed: {str(e)}") - - - - def packageApp(self, method: Callable[[ConfOutput],None]=packageAppTar)->None: + + def packageApp(self, method: Callable[[ConfOutput], None] = packageAppTar) -> None: return method(self) - - - def getElapsedTime(self, startTime:float)->datetime.timedelta: + def getElapsedTime(self, startTime: float) -> datetime.timedelta: return datetime.timedelta(seconds=round(timeit.default_timer() - startTime)) - - \ No newline at end of file diff --git a/contentctl/output/data_source_writer.py b/contentctl/output/data_source_writer.py index 1a6e4f95..8f46a98b 100644 --- a/contentctl/output/data_source_writer.py +++ b/contentctl/output/data_source_writer.py @@ -3,37 +3,50 @@ from typing import List import pathlib -class DataSourceWriter: +class DataSourceWriter: @staticmethod - def writeDataSourceCsv(data_source_objects: List[DataSource], file_path: pathlib.Path): - with open(file_path, mode='w', newline='') as file: + def writeDataSourceCsv( + data_source_objects: List[DataSource], file_path: pathlib.Path + ): + with open(file_path, mode="w", newline="") as file: writer = csv.writer(file) # Write the header - writer.writerow([ - "name", "id", "author", "source", "sourcetype", "separator", - "supported_TA_name", "supported_TA_version", "supported_TA_url", - "description" - ]) + writer.writerow( + [ + "name", + "id", + "author", + "source", + "sourcetype", + "separator", + "supported_TA_name", + "supported_TA_version", + "supported_TA_url", + "description", + ] + ) # Write the data for data_source in data_source_objects: - if len(data_source.supported_TA) > 0: + if len(data_source.supported_TA) > 0: supported_TA_name = data_source.supported_TA[0].name supported_TA_version = data_source.supported_TA[0].version - supported_TA_url = data_source.supported_TA[0].url or '' + supported_TA_url = data_source.supported_TA[0].url or "" else: - supported_TA_name = '' - supported_TA_version = '' - supported_TA_url = '' - writer.writerow([ - data_source.name, - data_source.id, - data_source.author, - data_source.source, - data_source.sourcetype, - data_source.separator, - supported_TA_name, - supported_TA_version, - supported_TA_url, - data_source.description, - ]) + supported_TA_name = "" + supported_TA_version = "" + supported_TA_url = "" + writer.writerow( + [ + data_source.name, + data_source.id, + data_source.author, + data_source.source, + data_source.sourcetype, + data_source.separator, + supported_TA_name, + supported_TA_version, + supported_TA_url, + data_source.description, + ] + ) diff --git a/contentctl/output/doc_md_output.py b/contentctl/output/doc_md_output.py index ab3cec66..9d128742 100644 --- a/contentctl/output/doc_md_output.py +++ b/contentctl/output/doc_md_output.py @@ -6,16 +6,20 @@ from contentctl.output.jinja_writer import JinjaWriter -class DocMdOutput(): +class DocMdOutput: index = 0 files_to_write = 0 - + def writeObjects(self, objects: list, output_path: str) -> None: self.files_to_write = sum([len(obj) for obj in objects]) self.index = 0 - progress_percent = ((self.index+1)/self.files_to_write) * 100 - if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): - print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + progress_percent = ((self.index + 1) / self.files_to_write) * 100 + if sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty(): + print( + f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", + end="", + flush=True, + ) attack_tactics = set() datamodels = set() @@ -31,24 +35,41 @@ def writeObjects(self, objects: list, output_path: str) -> None: if detection.datamodel: datamodels.update(detection.datamodel) - - Path(os.path.join(output_path, 'overview')).mkdir(parents=True, exist_ok=True) - Path(os.path.join(output_path, 'detections')).mkdir(parents=True, exist_ok=True) - Path(os.path.join(output_path, 'stories')).mkdir(parents=True, exist_ok=True) - Path(os.path.join(output_path, 'playbooks')).mkdir(parents=True, exist_ok=True) - JinjaWriter.writeObjectsList('doc_story_page.j2', os.path.join(output_path, 'overview/stories.md'), sorted(objects[0], key=lambda x: x.name)) - self.writeObjectsMd(objects[0], os.path.join(output_path, 'stories'), 'doc_stories.j2') + Path(os.path.join(output_path, "overview")).mkdir(parents=True, exist_ok=True) + Path(os.path.join(output_path, "detections")).mkdir(parents=True, exist_ok=True) + Path(os.path.join(output_path, "stories")).mkdir(parents=True, exist_ok=True) + Path(os.path.join(output_path, "playbooks")).mkdir(parents=True, exist_ok=True) + + JinjaWriter.writeObjectsList( + "doc_story_page.j2", + os.path.join(output_path, "overview/stories.md"), + sorted(objects[0], key=lambda x: x.name), + ) + self.writeObjectsMd( + objects[0], os.path.join(output_path, "stories"), "doc_stories.j2" + ) + + JinjaWriter.writeObjectsList( + "doc_detection_page.j2", + os.path.join(output_path, "overview/detections.md"), + sorted(objects[1], key=lambda x: x.name), + ) + self.writeObjectsMd( + objects[1], os.path.join(output_path, "detections"), "doc_detections.j2" + ) - JinjaWriter.writeObjectsList('doc_detection_page.j2', os.path.join(output_path, 'overview/detections.md'), sorted(objects[1], key=lambda x: x.name)) - self.writeObjectsMd(objects[1], os.path.join(output_path, 'detections'), 'doc_detections.j2') + JinjaWriter.writeObjectsList( + "doc_playbooks_page.j2", + os.path.join(output_path, "overview/paybooks.md"), + sorted(objects[2], key=lambda x: x.name), + ) + self.writeObjectsMd( + objects[2], os.path.join(output_path, "playbooks"), "doc_playbooks.j2" + ) - JinjaWriter.writeObjectsList('doc_playbooks_page.j2', os.path.join(output_path, 'overview/paybooks.md'), sorted(objects[2], key=lambda x: x.name)) - self.writeObjectsMd(objects[2], os.path.join(output_path, 'playbooks'), 'doc_playbooks.j2') - print("Done!") - - + # def writeNavigationPageObjects(self, objects: list, output_path: str) -> None: # for obj in objects: # JinjaWriter.writeObject('doc_navigation_pages.j2', os.path.join(output_path, '_pages', obj.lower().replace(' ', '_') + '.md'), @@ -59,10 +80,17 @@ def writeObjects(self, objects: list, output_path: str) -> None: def writeObjectsMd(self, objects, output_path: str, template_name: str) -> None: for obj in objects: - progress_percent = ((self.index+1)/self.files_to_write) * 100 - self.index+=1 - if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): - print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) - - JinjaWriter.writeObject(template_name, os.path.join(output_path, obj.name.lower().replace(' ', '_') + '.md'), obj) + progress_percent = ((self.index + 1) / self.files_to_write) * 100 + self.index += 1 + if sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty(): + print( + f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", + end="", + flush=True, + ) + JinjaWriter.writeObject( + template_name, + os.path.join(output_path, obj.name.lower().replace(" ", "_") + ".md"), + obj, + ) diff --git a/contentctl/output/jinja_writer.py b/contentctl/output/jinja_writer.py index 05690ea8..0cd0a45a 100644 --- a/contentctl/output/jinja_writer.py +++ b/contentctl/output/jinja_writer.py @@ -4,30 +4,34 @@ class JinjaWriter: - @staticmethod - def writeObjectsList(template_name : str, output_path : str, objects : list) -> None: - + def writeObjectsList(template_name: str, output_path: str, objects: list) -> None: j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), - trim_blocks=False) + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), + trim_blocks=False, + ) template = j2_env.get_template(template_name) output = template.render(objects=objects) - with open(output_path, 'w') as f: - output = output.encode('ascii', 'ignore').decode('ascii') + with open(output_path, "w") as f: + output = output.encode("ascii", "ignore").decode("ascii") f.write(output) - @staticmethod - def writeObject(template_name : str, output_path : str, object: dict[str,Any]) -> None: - + def writeObject( + template_name: str, output_path: str, object: dict[str, Any] + ) -> None: j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), - trim_blocks=False) + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), + trim_blocks=False, + ) template = j2_env.get_template(template_name) output = template.render(object=object) - with open(output_path, 'w') as f: - output = output.encode('ascii', 'ignore').decode('ascii') - f.write(output) \ No newline at end of file + with open(output_path, "w") as f: + output = output.encode("ascii", "ignore").decode("ascii") + f.write(output) diff --git a/contentctl/output/json_writer.py b/contentctl/output/json_writer.py index ee272255..c17ac787 100644 --- a/contentctl/output/json_writer.py +++ b/contentctl/output/json_writer.py @@ -1,19 +1,31 @@ import json from typing import Any -class JsonWriter(): + +class JsonWriter: @staticmethod - def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=True) -> None: + def writeJsonObject( + file_path: str, + object_name: str, + objs: list[dict[str, Any]], + readable_output: bool = True, + ) -> None: try: - with open(file_path, 'w') as outfile: + with open(file_path, "w") as outfile: if readable_output: # At the cost of slightly larger filesize, improve the redability significantly # by sorting and indenting keys/values - sorted_objs = sorted(objs, key=lambda o: o['name']) - json.dump({object_name:sorted_objs}, outfile, ensure_ascii=False, indent=2) + sorted_objs = sorted(objs, key=lambda o: o["name"]) + json.dump( + {object_name: sorted_objs}, + outfile, + ensure_ascii=False, + indent=2, + ) else: - json.dump({object_name:objs}, outfile, ensure_ascii=False) + json.dump({object_name: objs}, outfile, ensure_ascii=False) except Exception as e: - raise Exception(f"Error serializing object to Json File '{file_path}': {str(e)}") - \ No newline at end of file + raise Exception( + f"Error serializing object to Json File '{file_path}': {str(e)}" + ) diff --git a/contentctl/output/svg_output.py b/contentctl/output/svg_output.py index 055b3128..175abad5 100644 --- a/contentctl/output/svg_output.py +++ b/contentctl/output/svg_output.py @@ -5,50 +5,69 @@ from contentctl.output.jinja_writer import JinjaWriter from contentctl.objects.enums import DetectionStatus from contentctl.objects.detection import Detection -class SvgOutput(): - - def get_badge_dict(self, name:str, total_detections:List[Detection], these_detections:List[Detection])->dict[str,Any]: - obj:dict[str,Any] = {} - obj['name'] = name + +class SvgOutput: + def get_badge_dict( + self, + name: str, + total_detections: List[Detection], + these_detections: List[Detection], + ) -> dict[str, Any]: + obj: dict[str, Any] = {} + obj["name"] = name if name == "Production": - obj['color'] = "Green" + obj["color"] = "Green" elif name == "Detections": - obj['color'] = "Green" + obj["color"] = "Green" elif name == "Experimental": - obj['color'] = "Yellow" + obj["color"] = "Yellow" elif name == "Deprecated": - obj['color'] = "Red" + obj["color"] = "Red" - obj['count'] = len(total_detections) - if obj['count'] == 0: - obj['coverage'] = "NaN" + obj["count"] = len(total_detections) + if obj["count"] == 0: + obj["coverage"] = "NaN" else: - obj['coverage'] = len(these_detections) / obj['count'] - obj['coverage'] = "{:.0%}".format(obj['coverage']) + obj["coverage"] = len(these_detections) / obj["count"] + obj["coverage"] = "{:.0%}".format(obj["coverage"]) return obj - - def writeObjects(self, detections: List[Detection], output_path: pathlib.Path, type: SecurityContentType = None) -> None: - - - - total_dict:dict[str,Any] = self.get_badge_dict("Detections", detections, detections) - production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production]) - #deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated]) - #experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental]) - - - - - #Total number of detections - JinjaWriter.writeObject('detection_count.j2', output_path /'detection_count.svg', total_dict) - #JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'production_count.svg'), production_dict) - #JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'deprecated_count.svg'), deprecated_dict) - #JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'experimental_count.svg'), experimental_dict) - - #Percentage of detections that are production - JinjaWriter.writeObject('detection_coverage.j2', output_path/'detection_coverage.svg', production_dict) - #JinjaWriter.writeObject('detection_coverage.j2', os.path.join(output_path, 'detection_coverage.svg'), deprecated_dict) - #JinjaWriter.writeObject('detection_coverage.j2', os.path.join(output_path, 'detection_coverage.svg'), experimental_dict) + def writeObjects( + self, + detections: List[Detection], + output_path: pathlib.Path, + type: SecurityContentType = None, + ) -> None: + total_dict: dict[str, Any] = self.get_badge_dict( + "Detections", detections, detections + ) + production_dict: dict[str, Any] = self.get_badge_dict( + "% Production", + detections, + [ + detection + for detection in detections + if detection.status == DetectionStatus.production + ], + ) + # deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated]) + # experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental]) + + # Total number of detections + JinjaWriter.writeObject( + "detection_count.j2", output_path / "detection_count.svg", total_dict + ) + # JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'production_count.svg'), production_dict) + # JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'deprecated_count.svg'), deprecated_dict) + # JinjaWriter.writeObject('detection_count.j2', os.path.join(output_path, 'experimental_count.svg'), experimental_dict) + + # Percentage of detections that are production + JinjaWriter.writeObject( + "detection_coverage.j2", + output_path / "detection_coverage.svg", + production_dict, + ) + # JinjaWriter.writeObject('detection_coverage.j2', os.path.join(output_path, 'detection_coverage.svg'), deprecated_dict) + # JinjaWriter.writeObject('detection_coverage.j2', os.path.join(output_path, 'detection_coverage.svg'), experimental_dict) diff --git a/contentctl/output/yml_writer.py b/contentctl/output/yml_writer.py index 2e408c83..a7d0381a 100644 --- a/contentctl/output/yml_writer.py +++ b/contentctl/output/yml_writer.py @@ -1,4 +1,3 @@ - import yaml from typing import Any from enum import StrEnum, IntEnum @@ -8,25 +7,22 @@ # to write to files: # yaml.representer.RepresenterError: ('cannot represent an object',..... yaml.SafeDumper.add_multi_representer( - StrEnum, - yaml.representer.SafeRepresenter.represent_str + StrEnum, yaml.representer.SafeRepresenter.represent_str ) yaml.SafeDumper.add_multi_representer( - IntEnum, - yaml.representer.SafeRepresenter.represent_int + IntEnum, yaml.representer.SafeRepresenter.represent_int ) -class YmlWriter: +class YmlWriter: @staticmethod - def writeYmlFile(file_path : str, obj : dict[Any,Any]) -> None: - - with open(file_path, 'w') as outfile: + def writeYmlFile(file_path: str, obj: dict[Any, Any]) -> None: + with open(file_path, "w") as outfile: yaml.safe_dump(obj, outfile, default_flow_style=False, sort_keys=False) @staticmethod - def writeDetection(file_path: str, obj: dict[Any,Any]) -> None: + def writeDetection(file_path: str, obj: dict[Any, Any]) -> None: output = dict() output["name"] = obj["name"] output["id"] = obj["id"] @@ -35,7 +31,7 @@ def writeDetection(file_path: str, obj: dict[Any,Any]) -> None: output["author"] = obj["author"] output["type"] = obj["type"] output["status"] = obj["status"] - output["data_source"] = obj['data_sources'] + output["data_source"] = obj["data_sources"] output["description"] = obj["description"] output["search"] = obj["search"] output["how_to_implement"] = obj["how_to_implement"] @@ -45,20 +41,18 @@ def writeDetection(file_path: str, obj: dict[Any,Any]) -> None: output["tests"] = obj["tags"] YmlWriter.writeYmlFile(file_path=file_path, obj=output) - + @staticmethod - def writeStory(file_path: str, obj: dict[Any,Any]) -> None: + def writeStory(file_path: str, obj: dict[Any, Any]) -> None: output = dict() - output['name'] = obj['name'] - output['id'] = obj['id'] - output['version'] = obj['version'] - output['date'] = obj['date'] - output['author'] = obj['author'] - output['description'] = obj['description'] - output['narrative'] = obj['narrative'] - output['references'] = obj['references'] - output['tags'] = obj['tags'] + output["name"] = obj["name"] + output["id"] = obj["id"] + output["version"] = obj["version"] + output["date"] = obj["date"] + output["author"] = obj["author"] + output["description"] = obj["description"] + output["narrative"] = obj["narrative"] + output["references"] = obj["references"] + output["tags"] = obj["tags"] YmlWriter.writeYmlFile(file_path=file_path, obj=output) - - diff --git a/tests/test_splunk_contentctl.py b/tests/test_splunk_contentctl.py index f560f701..f7cdda74 100644 --- a/tests/test_splunk_contentctl.py +++ b/tests/test_splunk_contentctl.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == '0.1.0' + assert __version__ == "0.1.0" From a67c1f227de8f4af490ffadb5a6c5858997e8f27 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 16:21:51 -0600 Subject: [PATCH 6/7] Ignore reformat in blame --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..d02fa121 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1 @@ +ff87bcaf1741e8ecf15cb8d401438592dfef3ba7 # Mass reformat with adoption of ruff From fd940015bff04338f887a10b640d5aed989ec0b2 Mon Sep 17 00:00:00 2001 From: ljstella Date: Tue, 21 Jan 2025 16:22:21 -0600 Subject: [PATCH 7/7] Test against develop on security_content again --- .github/workflows/test_against_escu.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index f29e6a6f..ef07d445 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -35,7 +35,6 @@ jobs: with: path: security_content repository: splunk/security_content - ref: rba_migration #Install the given version of Python we will test against - name: Install Required Python Version