diff --git a/conf/default/web.conf.default b/conf/default/web.conf.default index 476e9164ee1..5d9538a18e4 100644 --- a/conf/default/web.conf.default +++ b/conf/default/web.conf.default @@ -78,8 +78,10 @@ enabled = no #enable linux fields on webgui [linux] -#For advanced users only, can be buggy, linux analysis is work in progress for fun +# For advanced users only, can be buggy, linux analysis is work in progress for fun enabled = no +# independent of enabled or not. To not show linux options, but process statically those files +static_only = no [malscore] enabled = no diff --git a/lib/cuckoo/common/demux.py b/lib/cuckoo/common/demux.py index d1287553b8e..2960d19fe30 100644 --- a/lib/cuckoo/common/demux.py +++ b/lib/cuckoo/common/demux.py @@ -32,7 +32,7 @@ cuckoo_conf = Config() web_cfg = Config("web") tmp_path = cuckoo_conf.cuckoo.get("tmppath", "/tmp") -linux_enabled = web_cfg.linux.get("enabled", False) +linux_enabled = web_cfg.linux.get("enabled", False) or web_cfg.linux.get("static_only", False) demux_extensions_list = { b".accdr", @@ -162,7 +162,8 @@ def is_valid_package(package: str) -> bool: return any(ptype in package for ptype in VALID_PACKAGES) -def _sf_children(child: sfFile) -> bytes: +# ToDo fix return type +def _sf_children(child: sfFile): # -> bytes: path_to_extract = "" _, ext = os.path.splitext(child.filename) ext = ext.lower() @@ -184,15 +185,17 @@ def _sf_children(child: sfFile) -> bytes: _ = path_write_file(path_to_extract, child.contents) except Exception as e: log.error(e, exc_info=True) - return path_to_extract.encode() + return (path_to_extract.encode(), child.platform, child.get_type(), child.get_size()) -def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> List[bytes]: +# ToDo fix typing need to add str as error msg +def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True): # -> List[bytes]: retlist = [] # do not extract from .bin (downloaded from us) if os.path.splitext(filename)[1] == b".bin": - return retlist + return retlist, "" + # ToDo need to introduce error msgs here try: password = options2passwd(options) or "infected" try: @@ -201,9 +204,13 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> unpacked = unpack(filename, check_shellcode=check_shellcode) if unpacked.package in whitelist_extensions: - return [filename] + file = File(filename) + magic_type = file.get_type() + platform = file.get_platform() + file_size = file.get_size() + return [filename, platform, magic_type, file_size], "" if unpacked.package in blacklist_extensions: - return [filename] + return [], "blacklisted package" for sf_child in unpacked.children: if sf_child.to_dict().get("children"): retlist.extend(_sf_children(ch) for ch in sf_child.children) @@ -214,7 +221,7 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> retlist.append(_sf_children(sf_child)) except Exception as e: log.error(e, exc_info=True) - return list(filter(None, retlist)) + return list(filter(None, retlist)), "" def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = True, platform: str = ""): # -> tuple[bytes, str]: @@ -227,21 +234,29 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = if isinstance(filename, str) and use_sflock: filename = filename.encode() + error_list = [] retlist = [] # if a package was specified, trim if allowed and required if package: - if package in ("msix",): retlist.append((filename, "windows")) else: if File(filename).get_size() <= web_cfg.general.max_sample_size or ( web_cfg.general.allow_ignore_size and "ignore_size_check" in options ): - retlist.append((filename, platform)) + retlist.append((filename, platform, "")) else: if web_cfg.general.enable_trim and trim_file(filename): retlist.append((trimmed_path(filename), platform)) - return retlist + else: + error_list.append( + { + os.path.basename( + filename + ): "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option" + } + ) + return retlist, error_list # handle quarantine files tmp_path = unquarantine(filename) @@ -259,9 +274,16 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = if use_sflock: if HAS_SFLOCK: retlist = demux_office(filename, password, platform) - return retlist + return retlist, error_list else: log.error("Detected password protected office file, but no sflock is installed: poetry install") + error_list.append( + { + os.path.basename( + filename + ): "Detected password protected office file, but no sflock is installed or correct password provided" + } + ) # don't try to extract from Java archives or executables if ( @@ -279,7 +301,14 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = else: if web_cfg.general.enable_trim and trim_file(filename): retlist.append((trimmed_path(filename), platform)) - return retlist + else: + error_list.append( + { + os.path.basename(filename), + "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option", + } + ) + return retlist, error_list new_retlist = [] @@ -288,26 +317,34 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = check_shellcode = False # all in one unarchiver - retlist = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else [] + retlist, error_msg = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else [] # if it isn't a ZIP or an email, or we aren't able to obtain anything interesting from either, then just submit the # original file if not retlist: + if error_msg: + error_list.append({os.path.basename(filename), error_msg}) new_retlist.append((filename, platform)) else: - for filename in retlist: + for filename, platform, magic_type, file_size in retlist: # verify not Windows binaries here: - file = File(filename) - magic_type = file.get_type() - platform = file.get_platform() if platform == "linux" and not linux_enabled and "Python" not in magic_type: + error_list.append({os.path.basename(filename): "Linux processing is disabled"}) continue - if file.get_size() > web_cfg.general.max_sample_size and not ( + if file_size > web_cfg.general.max_sample_size and not ( web_cfg.general.allow_ignore_size and "ignore_size_check" in options ): if web_cfg.general.enable_trim: # maybe identify here if trim_file(filename): filename = trimmed_path(filename) + else: + error_list.append( + { + os.path.basename(filename), + "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option", + } + ) new_retlist.append((filename, platform)) - return new_retlist[:10] + + return new_retlist[:10], error_list diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index 25555f87359..f582aac1bd0 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -820,6 +820,9 @@ def SevenZip_unpack(file: str, *, filetype: str, data_dictionary: dict, options: ): return + if all([pattern in file_data for pattern in (b"AndroidManifest.xml", b"classes.dex")]): + return + password = "" # Only for real 7zip, breaks others password = options.get("password", "infected") diff --git a/lib/cuckoo/common/web_utils.py b/lib/cuckoo/common/web_utils.py index 4b0de852f48..f47fa29f80a 100644 --- a/lib/cuckoo/common/web_utils.py +++ b/lib/cuckoo/common/web_utils.py @@ -766,7 +766,7 @@ def download_file(**kwargs): if not onesuccess: return "error", {"error": f"Provided hash not found on {kwargs['service']}"} - return "ok", kwargs["task_ids"] + return "ok", kwargs["task_ids"], extra_details.get("erros", []) def save_script_to_storage(task_ids, kwargs): diff --git a/lib/cuckoo/core/database.py b/lib/cuckoo/core/database.py index eb643b56d75..a549b1b870f 100644 --- a/lib/cuckoo/core/database.py +++ b/lib/cuckoo/core/database.py @@ -119,6 +119,7 @@ distconf = Config("distributed") web_conf = Config("web") LINUX_ENABLED = web_conf.linux.enabled +LINUX_STATIC = web_conf.linux.static_only DYNAMIC_ARCH_DETERMINATION = web_conf.general.dynamic_arch_determination if repconf.mongodb.enabled: @@ -1538,7 +1539,7 @@ def demux_sample_and_add_to_db( package, _ = self._identify_aux_func(file_path, package, check_shellcode=check_shellcode) # extract files from the (potential) archive - extracted_files = demux_sample(file_path, package, options, platform=platform) + extracted_files, demux_error_msgs = demux_sample(file_path, package, options, platform=platform) # check if len is 1 and the same file, if diff register file, and set parent if extracted_files and (file_path, platform) not in extracted_files: sample_parent_id = self.register_sample(File(file_path), source_url=source_url) @@ -1547,6 +1548,18 @@ def demux_sample_and_add_to_db( # create tasks for each file in the archive for file, platform in extracted_files: + # ToDo we lose package here and send APKs to windows + if platform in ("linux", "darwin") and LINUX_STATIC: + task_ids += self.add_static( + file_path=file_path, + priority=priority, + tlp=tlp, + user_id=user_id, + username=username, + options=options, + package=package, + ) + continue if static: # On huge loads this just become a bottleneck config = False @@ -1621,6 +1634,8 @@ def demux_sample_and_add_to_db( if config and isinstance(config, dict): details = {"config": config.get("cape_config", {})} + if demux_error_msgs: + details["errors"] = demux_error_msgs # this is aim to return custom data, think of this as kwargs return task_ids, details @@ -1694,7 +1709,7 @@ def add_static( user_id=0, username=False, ): - extracted_files = demux_sample(file_path, package, options) + extracted_files, demux_error_msgs = demux_sample(file_path, package, options) sample_parent_id = None # check if len is 1 and the same file, if diff register file, and set parent if not isinstance(file_path, bytes): diff --git a/modules/processing/CAPE.py b/modules/processing/CAPE.py index f257444236b..a9bc97cd57f 100644 --- a/modules/processing/CAPE.py +++ b/modules/processing/CAPE.py @@ -113,7 +113,7 @@ def _cape_type_string(self, type_strings, file_info, append_file): elif type_strings[0] == "MS-DOS": file_info["cape_type"] = "DOS MZ image: executable" else: - file_info["cape_type"] = file_info["cape_type"] or "PE image" + file_info["cape_type"] = file_info["cape_type"] or "unknown" return append_file def _metadata_processing(self, metadata, file_info, append_file): diff --git a/tests/test_demux.py b/tests/test_demux.py index 605974ee71e..e3f4aa3e946 100644 --- a/tests/test_demux.py +++ b/tests/test_demux.py @@ -85,8 +85,9 @@ def test_demux_sample_pe32(self, grab_sample): def test_demux_package(self): empty_file = tempfile.NamedTemporaryFile() - assert demux.demux_sample(filename=empty_file.name, package="Emotet", options="foo", use_sflock=False) == [ - (empty_file.name, "") + demuxed, _ = demux.demux_sample(filename=empty_file.name, package="Emotet", options="foo", use_sflock=False) + demuxed == [ + (empty_file.name, "", "") ] empty_file.close() diff --git a/tests/test_objects.py b/tests/test_objects.py index ed0966aab7e..45af7ea7b06 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -223,7 +223,9 @@ def test_get_type(self, test_files): [ ("temp_pe32", "PE32 executable (GUI) Intel 80386, for MS Windows", True), # emulated magic type ("temp_pe64", "PE32+ executable (GUI) x86-64, for MS Windows", True), # emulated magic type - ("temp_pe_aarch64", "MS-DOS executable PE32 executable Aarch64, for MS Windows", True), + # Broken we remove "MS-DOS executable" + # ("temp_pe_aarch64", "MS-DOS executable PE32 executable Aarch64, for MS Windows", True), + ("temp_pe_aarch64", "PE32 executable Aarch64, for MS Windows", True), ("temp_elf32", "ELF 32-bit LSB", False), ("temp_elf64", "ELF 64-bit LSB", False), ("temp_macho_arm64", "Mach-O 64-bit arm64 executable", False), diff --git a/utils/submit.py b/utils/submit.py index c66378f8ce3..8785b58f5c8 100644 --- a/utils/submit.py +++ b/utils/submit.py @@ -345,6 +345,7 @@ def main(): try: tmp_path = store_temp_file(open(file_path, "rb").read(), sanitize_filename(os.path.basename(file_path))) with db.session.begin(): + # ToDo expose extra_details["errors"] task_ids, extra_details = db.demux_sample_and_add_to_db( file_path=tmp_path, package=args.package, diff --git a/web/apiv2/views.py b/web/apiv2/views.py index 0fb0babfa49..293c968d090 100644 --- a/web/apiv2/views.py +++ b/web/apiv2/views.py @@ -187,7 +187,7 @@ def tasks_create_static(request): options = request.data.get("options", "") priority = force_int(request.data.get("priority")) - resp["error"] = False + resp["error"] = [] files = request.FILES.getlist("file") extra_details = {} task_ids = [] @@ -203,6 +203,8 @@ def tasks_create_static(request): user_id=request.user.id or 0, ) task_ids.extend(task_id) + if extra_details.get("erros"): + resp["errors"].extend(extra_details["errors"]) except CuckooDemuxError as e: resp = {"error": True, "error_value": e} return Response(resp) @@ -226,7 +228,6 @@ def tasks_create_static(request): resp["url"].append("{0}/submit/status/{1}".format(apiconf.api.get("url"), tid)) else: resp = {"error": True, "error_value": "Error adding task to database"} - return Response(resp) @@ -341,12 +342,22 @@ def tasks_create_file(request): if tmp_path: details["path"] = tmp_path details["content"] = content - status, task_ids_tmp = download_file(**details) + demux_error_msgs = [] + + result = download_file(**details) + if len(result) == 2: + status, task_ids_tmp = result + elif len(result) == 3: + status, task_ids_tmp, demux_error_msgs = result + if status == "error": details["errors"].append({os.path.basename(tmp_path).decode(): task_ids_tmp}) else: details["task_ids"] = task_ids_tmp + if demux_error_msgs: + details["errors"].extend(demux_error_msgs) + if details["task_ids"]: tasks_count = len(details["task_ids"]) else: @@ -565,12 +576,20 @@ def tasks_create_dlnexec(request): "user_id": request.user.id or 0, } - status, task_ids_tmp = download_file(**details) + result = download_file(**details) + if len(result) == 2: + status, task_ids_tmp = result + elif len(result) == 3: + status, task_ids_tmp, demux_error_msgs = result + if status == "error": details["errors"].append({os.path.basename(path).decode(): task_ids_tmp}) else: details["task_ids"] = task_ids_tmp + if demux_error_msgs: + details["errors"].extend(demux_error_msgs) + if details["task_ids"]: tasks_count = len(details["task_ids"]) else: diff --git a/web/submission/views.py b/web/submission/views.py index dcf6d42d411..0efc0db2ba0 100644 --- a/web/submission/views.py +++ b/web/submission/views.py @@ -508,7 +508,13 @@ def index(request, task_id=None, resubmit_hash=None): details["path"] = path details["content"] = content - status, task_ids_tmp = download_file(**details) + result = download_file(**details) + if len(result) == 2: + status, task_ids_tmp = result + elif len(result) == 3: + status, task_ids_tmp, demux_error_msg = result + if demux_error_msg: + details["errors"].extend(demux_error_msg) if status == "error": details["errors"].append({os.path.basename(filename): task_ids_tmp}) else: @@ -537,7 +543,13 @@ def index(request, task_id=None, resubmit_hash=None): details["path"] = path details["content"] = content - status, task_ids_tmp = download_file(**details) + result = download_file(**details) + if len(result) == 2: + status, task_ids_tmp = result + elif len(result) == 3: + status, task_ids_tmp, demux_error_msg = result + if demux_error_msg: + details["errors"].extend(demux_error_msg) if status == "error": details["errors"].append({os.path.basename(path): task_ids_tmp}) else: @@ -619,7 +631,14 @@ def index(request, task_id=None, resubmit_hash=None): details["content"] = content details["service"] = "DLnExec" details["source_url"] = samples - status, task_ids_tmp = download_file(**details) + result = download_file(**details) + if len(result) == 2: + status, task_ids_tmp = result + elif len(result) == 3: + status, task_ids_tmp, demux_error_msg = result + if demux_error_msg: + details["errors"].extend(demux_error_msg) + if status == "error": details["errors"].append({os.path.basename(path): task_ids_tmp}) else: