Skip to content

Commit

Permalink
work in progress for idaholab#331, improvements to extracted_files_ht…
Browse files Browse the repository at this point in the history 
…tp_server.py and the setting/creation of ACL rules on hedgehog
  • Loading branch information
@mmguero
mmguero committed Apr 8, 2024
1 parent a6bf5f5 commit a02badb
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 73 deletions.
Binary file added docs/images/hedgehog/images/file_server_zip.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 4 additions & 2 deletions docs/malcolm-hedgehog-e2e-iso-install.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -477,12 +477,14 @@ Users will prompted to specify which engine(s) to use to analyze extracted files

Files flagged as potentially malicious will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [OpenSearch Dashboards]({{ site.github.repository_url }}#DashboardsVisualizations) when forwarded to Malcolm.

![Extracted file server configuration](./images/hedgehog/images/file_server_zip.png)

Hedgehog Linux provides an extracted files directory listing to browse and download Zeek-extracted files. As this interface is primarily intended to be accessed through the Malcolm user interface, this service is accessible only by IP addresses [included in the ACL for artifact reachback from Malcolm](#HedgehogACL) over port '8006/tcp'. Please read the Malcolm documentation for [**Automatic file extraction and scanning - User interface**](file-scanning.md#ZeekFileExtractionUI) for more information on how to access preserved files.

![File quarantine](./images/hedgehog/images/file_quarantine.png)

Finally, users will be presented with the list of configuration variables that will be used for capture, including the values which have been selected up to this point in this section. Upon choosing **OK** these values will be written back out to the sensor configuration file located at `/opt/sensor/sensor_ctl/control_vars.conf`. Editing this file manually is not recommended. After confirming these values, users will be presented with a confirmation that these settings have been written to the configuration file then returned to the welcome screen.

See the Malcolm documentation for [**Automatic file extraction and scanning - User interface**](file-scanning.md#ZeekFileExtractionUI) for more information on how to access preserved files.

## <a name="HedgehogConfigForwarding"></a> Configure Forwarding

Select **Configure Forwarding** to set up forwarding logs and statistics from the sensor to an aggregator server, such as [Malcolm]({{ site.github.repository_url }}).
Expand Down
2 changes: 1 addition & 1 deletion hedgehog-iso/interface/sensor_ctl/control_vars.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export EXTRACTED_FILE_HTTP_SERVER_ZIP=false
# Specifies the password for encrypted Zeek-extracted files served over HTTP
# If EXTRACTED_FILE_HTTP_SERVER_ZIP is true this is the password for the Zip file,
# otherwise it is the AES-256-CBC decryption password
EXTRACTED_FILE_HTTP_SERVER_KEY=infected
export EXTRACTED_FILE_HTTP_SERVER_KEY=infected
# Whether or not to use libmagic to show MIME types for Zeek-extracted files served
export EXTRACTED_FILE_HTTP_SERVER_MAGIC=false
# HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved")
Expand Down
134 changes: 64 additions & 70 deletions shared/bin/configure-capture.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,13 @@ class Constants:
)
MSG_CONFIG_ICS_BEST_GUESS = 'Should the sensor use "best guess" to identify potential OT/ICS traffic with Zeek?'
MSG_CONFIG_ZEEK_CARVED_SCANNERS = 'Specify scanners for Zeek-carved files'
MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ZIP = 'ZIP preserved files when downloaded via web interface?'
MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ZIP_KEY = (
'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)'
)
MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ENC_KEY = (
'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)'
)
MSG_CONFIG_ZEEK_CARVING = 'Specify Zeek file carving mode'
MSG_CONFIG_ZEEK_CARVING_MIMES = 'Specify file types to carve'
MSG_CONFIG_CARVED_FILE_PRESERVATION = 'Specify which carved files to preserve'
Expand Down Expand Up @@ -191,10 +198,10 @@ class Constants:


###################################################################################################
def rewrite_dict_to_file(vals_dict, config_file_name, backup='.bak'):
def rewrite_dict_to_file(vals_dict, config_file_name, inplace=True, backup='.bak'):
if vals_dict and os.path.isfile(config_file_name):
values_re = re.compile(r"\b(" + '|'.join(list(vals_dict.keys())) + r")\s*=\s*.*?$")
with fileinput.FileInput(config_file_name, inplace=True, backup=backup) as file:
with fileinput.FileInput(config_file_name, inplace=inplace, backup=backup) as file:
for line in file:
line = line.rstrip("\n")
key_match = values_re.search(line)
Expand Down Expand Up @@ -456,15 +463,10 @@ def main():
)
if code == Dialog.OK:
# modify specified values in-place in SENSOR_CAPTURE_CONFIG file
autostart_re = re.compile(r"(\bAUTOSTART_\w+)\s*=\s*.+?$")
with fileinput.FileInput(Constants.SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as file:
for line in file:
line = line.rstrip("\n")
autostart_match = autostart_re.search(line)
if autostart_match is not None:
print(autostart_re.sub(r"\1=%s" % capture_config_dict[autostart_match.group(1)], line))
else:
print(line)
rewrite_dict_to_file(
{k: v for (k, v) in capture_config_dict.items() if k.startswith('AUTOSTART_')},
Constants.SENSOR_CAPTURE_CONFIG,
)

# hooray
code = d.msgbox(text=Constants.MSG_CONFIG_AUTOSTART_SUCCESS)
Expand All @@ -477,7 +479,9 @@ def main():
# previously used capture interfaces
preselected_ifaces = set([x.strip() for x in capture_config_dict["CAPTURE_INTERFACE"].split(',')])

while (len(available_adapters) > 0) and (d.yesno(Constants.MSG_IDENTIFY_NICS) == Dialog.OK):
while (len(available_adapters) > 0) and (
d.yesno(Constants.MSG_IDENTIFY_NICS, yes_label="No", no_label="Yes") != Dialog.OK
):
code, blinky_iface = d.radiolist(
Constants.MSG_SELECT_BLINK_INTERFACE,
choices=[(adapter.name, adapter.description, False) for adapter in available_adapters],
Expand Down Expand Up @@ -545,19 +549,6 @@ def main():
)
prev_capture_filter = capture_filter

# regular expressions for selected name=value pairs to update in configuration file
capture_interface_re = re.compile(r"(\bCAPTURE_INTERFACE)\s*=\s*.+?$")
capture_filter_re = re.compile(r"(\bCAPTURE_FILTER)\s*=\s*.*?$")
pcap_path_re = re.compile(r"(\bPCAP_PATH)\s*=\s*.+?$")
zeek_path_re = re.compile(r"(\bZEEK_LOG_PATH)\s*=\s*.+?$")
zeek_carve_re = re.compile(r"(\bZEEK_EXTRACTOR_MODE)\s*=\s*.+?$")
zeek_file_preservation_re = re.compile(r"(\bEXTRACTED_FILE_PRESERVATION)\s*=\s*.+?$")
zeek_carve_override_re = re.compile(r"(\bZEEK_EXTRACTOR_OVERRIDE_FILE)\s*=\s*.*?$")
zeek_file_watch_re = re.compile(r"(\bZEEK_FILE_WATCH)\s*=\s*.+?$")
zeek_file_scanner_re = re.compile(r"(\bZEEK_FILE_SCAN_\w+)\s*=\s*.+?$")
disable_ics_all_re = re.compile(r"(\bZEEK_DISABLE_ICS_ALL)\s*=\s*.+?$")
ics_best_guess_re = re.compile(r"(\bZEEK_DISABLE_BEST_GUESS_ICS)\s*=\s*.+?$")

# get paths for captured PCAP and Zeek files
while True:
code, path_values = d.form(
Expand All @@ -584,8 +575,10 @@ def main():
code = d.msgbox(text=Constants.MSG_ERROR_DIR_NOT_FOUND)

# enable/disable ICs
ics_network = d.yesno(Constants.MSG_CONFIG_ICS_ANALYZERS) == Dialog.OK
ics_best_guess = ics_network and (d.yesno(Constants.MSG_CONFIG_ICS_BEST_GUESS) == Dialog.OK)
ics_network = d.yesno(Constants.MSG_CONFIG_ICS_ANALYZERS, yes_label="No", no_label="Yes") != Dialog.OK
ics_best_guess = ics_network and (
d.yesno(Constants.MSG_CONFIG_ICS_BEST_GUESS, yes_label="No", no_label="Yes") != Dialog.OK
)

# configure file carving
code, zeek_carve_mode = d.radiolist(
Expand Down Expand Up @@ -645,6 +638,8 @@ def main():
mime_tags = []
capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"] = ""
zeek_carved_file_preservation = PRESERVE_NONE
zeek_carved_file_http_server_zip = False
zeek_carved_file_http_serve_encrypt_key = ''

if zeek_carve_mode.startswith(Constants.ZEEK_FILE_CARVING_CUSTOM):
# get all known mime-to-extension mappings into a dictionary
Expand Down Expand Up @@ -745,13 +740,30 @@ def main():
]:
capture_config_dict[key] = "false"

if zeek_carved_file_preservation != PRESERVE_NONE:
zeek_carved_file_http_server_zip = (
d.yesno(Constants.MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ZIP) == Dialog.OK
)
code, zeek_carved_file_http_serve_encrypt_key = d.inputbox(
(
Constants.MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ZIP_KEY
if zeek_carved_file_http_server_zip
else Constants.MSG_CONFIG_ZEEK_CARVED_HTTP_SERVER_ENC_KEY
),
init=capture_config_dict.get("EXTRACTED_FILE_HTTP_SERVER_KEY", 'infected'),
)

# reconstitute dictionary with user-specified values
capture_config_dict["CAPTURE_INTERFACE"] = ",".join(selected_ifaces)
capture_config_dict["CAPTURE_FILTER"] = capture_filter
capture_config_dict["PCAP_PATH"] = path_values[0]
capture_config_dict["ZEEK_LOG_PATH"] = path_values[1]
capture_config_dict["ZEEK_EXTRACTOR_MODE"] = zeek_carve_mode
capture_config_dict["EXTRACTED_FILE_PRESERVATION"] = zeek_carved_file_preservation
capture_config_dict["EXTRACTED_FILE_HTTP_SERVER_ZIP"] = (
'true' if zeek_carved_file_http_server_zip else 'false'
)
capture_config_dict["EXTRACTED_FILE_HTTP_SERVER_KEY"] = zeek_carved_file_http_serve_encrypt_key
capture_config_dict["ZEEK_DISABLE_ICS_ALL"] = '' if ics_network else 'true'
capture_config_dict["ZEEK_DISABLE_BEST_GUESS_ICS"] = '' if ics_best_guess else 'true'

Expand All @@ -773,49 +785,31 @@ def main():
)
if code == Dialog.OK:
# modify specified values in-place in SENSOR_CAPTURE_CONFIG file
with fileinput.FileInput(Constants.SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as file:
for line in file:
line = line.rstrip("\n")
if capture_interface_re.search(line) is not None:
print(capture_interface_re.sub(r"\1=%s" % ",".join(selected_ifaces), line))
elif zeek_carve_override_re.search(line) is not None:
print(
zeek_carve_override_re.sub(
r'\1="%s"' % capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"], line
)
)
elif zeek_carve_re.search(line) is not None:
print(zeek_carve_re.sub(r"\1=%s" % zeek_carve_mode, line))
elif zeek_file_preservation_re.search(line) is not None:
print(zeek_file_preservation_re.sub(r"\1=%s" % zeek_carved_file_preservation, line))
elif capture_filter_re.search(line) is not None:
print(capture_filter_re.sub(r'\1="%s"' % capture_filter, line))
elif pcap_path_re.search(line) is not None:
print(pcap_path_re.sub(r'\1="%s"' % capture_config_dict["PCAP_PATH"], line))
elif zeek_path_re.search(line) is not None:
print(zeek_path_re.sub(r'\1="%s"' % capture_config_dict["ZEEK_LOG_PATH"], line))
elif zeek_file_watch_re.search(line) is not None:
print(zeek_file_watch_re.sub(r"\1=%s" % capture_config_dict["ZEEK_FILE_WATCH"], line))
elif disable_ics_all_re.search(line) is not None:
print(
disable_ics_all_re.sub(r'\1=%s' % capture_config_dict["ZEEK_DISABLE_ICS_ALL"], line)
)
elif ics_best_guess_re.search(line) is not None:
print(
ics_best_guess_re.sub(
r'\1=%s' % capture_config_dict["ZEEK_DISABLE_BEST_GUESS_ICS"], line
)
)
else:
zeek_file_scanner_match = zeek_file_scanner_re.search(line)
if zeek_file_scanner_match is not None:
print(
zeek_file_scanner_re.sub(
r"\1=%s" % capture_config_dict[zeek_file_scanner_match.group(1)], line
)
)
else:
print(line)
rewrite_dict_to_file(
{
"CAPTURE_FILTER": '"' + capture_config_dict["CAPTURE_FILTER"] + '"',
"CAPTURE_INTERFACE": capture_config_dict["CAPTURE_INTERFACE"],
"EXTRACTED_FILE_HTTP_SERVER_KEY": '"'
+ capture_config_dict["EXTRACTED_FILE_HTTP_SERVER_KEY"]
+ '"',
"EXTRACTED_FILE_HTTP_SERVER_ZIP": capture_config_dict["EXTRACTED_FILE_HTTP_SERVER_ZIP"],
"EXTRACTED_FILE_PRESERVATION": capture_config_dict["EXTRACTED_FILE_PRESERVATION"],
"PCAP_PATH": '"' + capture_config_dict["PCAP_PATH"] + '"',
"ZEEK_DISABLE_BEST_GUESS_ICS": capture_config_dict["ZEEK_DISABLE_BEST_GUESS_ICS"],
"ZEEK_DISABLE_ICS_ALL": capture_config_dict["ZEEK_DISABLE_ICS_ALL"],
"ZEEK_EXTRACTOR_MODE": capture_config_dict["ZEEK_EXTRACTOR_MODE"],
"ZEEK_LOG_PATH": '"' + capture_config_dict["ZEEK_LOG_PATH"] + '"',
"ZEEK_EXTRACTOR_OVERRIDE_FILE": '"'
+ capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]
+ '"',
"ZEEK_FILE_SCAN_CAPA": capture_config_dict["ZEEK_FILE_SCAN_CAPA"],
"ZEEK_FILE_SCAN_CLAMAV": capture_config_dict["ZEEK_FILE_SCAN_CLAMAV"],
"ZEEK_FILE_SCAN_VTOT": capture_config_dict["ZEEK_FILE_SCAN_VTOT"],
"ZEEK_FILE_SCAN_YARA": capture_config_dict["ZEEK_FILE_SCAN_YARA"],
"ZEEK_FILE_WATCH": capture_config_dict["ZEEK_FILE_WATCH"],
},
Constants.SENSOR_CAPTURE_CONFIG,
)

# write out file carving overrides if specified
if (len(mime_tags) > 0) and (len(capture_config_dict["ZEEK_EXTRACTOR_OVERRIDE_FILE"]) > 0):
Expand Down

0 comments on commit a02badb

Please sign in to comment.