Partial rewrite of the ClamAV signature translation script to add sup…

…port

for the LDB database. Also fixed a bug causing signatures targeting any
filetype to be ignored.
The Yara module now receives information regarding the version_info
location.
Wrote a simple executable to be used in unit tests.

.gitignore

@@ -24,12 +24,16 @@
 *.out
 *.app
 bin/manalyze
+bin/manalyze-tests
 
 # Compiled Yara rules + ClamAV rules and databases
 *.yarac
 bin/yara_rules/clamav.*
 bin/yara_rules/*.cvd
 
+# Compiled Python scripts
+bin/yara_rules/*.pyc
+
 # CMake artifacts
 CMakeFiles
 CMakeCache.txt

.travis.yml

@@ -23,6 +23,7 @@ before_script:
 script:
   - make
   - bin/manalyze --version
+  - python bin/yara_rules/update_clamav_signatures.py
   - bin/manalyze-tests
 after_success:
   - coveralls --exclude external/yara --exclude test --gcov gcov-4.8

bin/yara_rules/update_clamav_signatures.py

@@ -22,9 +22,9 @@
 import shutil
 import tarfile
 import zlib
-import subprocess
 import urllib2
 import argparse
+from parse_clamav import parse_ndb, parse_ldb
 
 
 URL_MAIN = "http://database.clamav.net/main.cvd"
@@ -59,7 +59,7 @@ def download_file(url):
         file_size_dl += len(buffer)
         outfile.write(buffer)
         status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
-        status = status + chr(8) * (len(status) + 1)
+        status += chr(8) * (len(status) + 1)
         print status,
 
     outfile.close()
@@ -114,14 +114,24 @@ def update_signatures(url, download):
     f.close()
     g.close()
 
+    # We need to keep a set of all the signatures seen, because there are duplicates in the ClamAV database and
+    # Yara doesn't like that.
+    RULES = set()
+
     # Excract the signatures
     zlib_decompress("%s.tar.gz" % file_basename, "%s.tar" % file_basename)
     tar = tarfile.open("%s.tar" % file_basename)
     tar.extract("%s.ndb" % file_basename)
     os.chmod("%s.ndb" % file_basename, 0644)
+    if "%s.ldb" % file_basename in tar.getnames():
+        tar.extract("%s.ldb" % file_basename)
+        os.chmod("%s.ldb" % file_basename, 0644)
     tar.close()
+    parse_ndb("%s.ndb" % file_basename, "clamav.yara", file_basename != "main")
+    if os.path.exists("%s.ldb" % file_basename):
+        parse_ldb("%s.ldb" % file_basename, "clamav.yara", file_basename != "main")
+        os.remove("%s.ldb" % file_basename)
     os.remove("%s.tar" % file_basename)
-    subprocess.call([sys.executable, "./parse_clamav.py", "-i", "%s.ndb" % file_basename, "-o", "clamav.yara"])
     os.remove("%s.ndb" % file_basename)
 
 # Work in the script's directory

include/manape/resources.h

@@ -45,7 +45,7 @@ class Resource
 			 const std::string&		language,
 			 boost::uint32_t		codepage,
 			 boost::uint32_t		size,
-			 unsigned int			offset_in_file,
+			 boost::uint32_t			offset_in_file,
 			 const std::string&		path_to_pe)
 		: _type(type),
 		  _name(name),
@@ -62,7 +62,7 @@ class Resource
 			const std::string&		language,
 			boost::uint32_t			codepage,
 			boost::uint32_t			size,
-			unsigned int			offset_in_file,
+			boost::uint32_t			offset_in_file,
 			const std::string&		path_to_pe)
 		: _type(type),
 		  _name(""),
@@ -81,6 +81,7 @@ class Resource
 	DECLSPEC boost::uint32_t	get_codepage()	const { return _codepage; }
 	DECLSPEC boost::uint32_t	get_size()		const { return _size; }
 	DECLSPEC boost::uint32_t	get_id()		const { return _id; }
+	DECLSPEC boost::uint32_t	get_offset()	const { return _offset_in_file; }
 	DECLSPEC double				get_entropy()	const {
 		return utils::shannon_entropy(*get_raw_data());
 	}

manape/pe.cpp

@@ -783,20 +783,31 @@ boost::shared_ptr<manape_data> PE::create_manape_module_data() const
 	boost::shared_ptr<manape_data> res(new manape_data, delete_manape_module_data);
 	res->entrypoint = _ioh->AddressOfEntryPoint;
 	res->number_of_sections = _sections.size();
-	res->sections = (manape_section*) malloc(res->number_of_sections * sizeof(manape_section));
+	res->sections = (manape_file_portion*) malloc(res->number_of_sections * sizeof(manape_file_portion));
 	if (res->sections != nullptr)
 	{
 		for (boost::uint32_t i = 0 ; i < res->number_of_sections ; ++i)
 		{
-			res->sections[i].section_start = _sections[i]->get_pointer_to_raw_data();
-			res->sections[i].section_size = _sections[i]->get_size_of_raw_data();
+			res->sections[i].start = _sections[i]->get_pointer_to_raw_data();
+			res->sections[i].size = _sections[i]->get_size_of_raw_data();
 		}
 	}
 	else
 	{
 		PRINT_WARNING << "Not enough memory to allocate data for the MANAPE module!" << std::endl;
 		res->number_of_sections = 0;
 	}
+
+	// Add VERSION_INFO location for some ClamAV signatures
+	for (auto it = _resource_table.begin() ; it != _resource_table.end() ; ++it)
+	{
+		if (*(*it)->get_type() == "RT_VERSION")
+		{
+			res->version_info.start = (*it)->get_offset();
+			res->version_info.size = (*it)->get_size();
+			break;
+		}
+	}
 	return res;
 }
 

plugins/plugin_imports.cpp

@@ -30,7 +30,7 @@ std::string anti_debug =
 	"OutputDebugString|SwitchToThread|NtQueryInformationProcess"	// Standard anti-debug API calls
 	"QueryPerformanceCounter";	// Techniques based on timing. GetTickCount ignored (too many false positives)
 
-std::string vanilla_injection = "VirtualAlloc(.*)|WriteProcessMemory|CreateRemoteThread|OpenProcess";
+std::string vanilla_injection = "VirtualAlloc(.*)|WriteProcessMemory|CreateRemoteThread(Ex)?|OpenProcess";
 
 std::string keylogger_api = "SetWindowsHook(Ex)?|GetAsyncKeyState|GetForegroundWindow|AttachThreadInput|CallNextHook(Ex)?|MapVirtualKey";
 

plugins/plugin_resources.cpp

@@ -58,7 +58,7 @@ class ResourcesPlugin : public IPlugin
 			yara::const_matches matches = y.scan_bytes(*(*it)->get_raw_data());
 			if (matches->size() > 0)
 			{
-				for (int i = 0 ; i < matches->size() ; ++i)
+				for (size_t i = 0 ; i < matches->size() ; ++i)
 				{
 					std::string ext = matches->at(i)->operator[]("extension");
 					if (ext == ".exe" || ext == ".sys" || ext == ".cab")

test/pe.cpp

@@ -47,8 +47,8 @@ class SetWorkingDirectory
 
 BOOST_FIXTURE_TEST_CASE(parse_calc, SetWorkingDirectory)
 {
-    mana::PE pe("testfiles/calc.exe");
-	BOOST_CHECK_EQUAL(pe.get_filesize(), 115200);
+    mana::PE pe("testfiles/manatest.exe");
+	BOOST_CHECK_EQUAL(pe.get_filesize(), 16360);
 
     // DOS Header
     boost::optional<mana::dos_header> pdos = pe.get_dos_header();

test/testfiles/manatest.cpp

@@ -0,0 +1,16 @@
+#include <Windows.h>
+#include <iostream>
+
+int main(int argc, char** argv)
+{	
+	// Vanilla injection imports
+	OpenProcess(0, 0, 0);
+	CreateRemoteThread(0, 0, 0, 0, 0, 0, 0);
+	WriteProcessMemory(0, 0, 0, 0, 0);
+
+	std::cout << "cmd.exe" << std::endl;
+	// EICAR antivirus signature. ClamAV's rules only detect the standard signature at offset 0, but somehow has a rule
+	// for the base64-encoded string.
+	std::cout << "WDVPIVAlQEFQWzRcUFpYNTQoUF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNU\nLUZJTEUhJEgrSCo=\n"; 
+	std::cout << std::hex;
+}