Changed the Yara interface so the offset of matched strings could be …

…obtained.

[plugin suspicious_strings] Excluded authenticode signature and RT_MANIFEST from the domain name search.
[plugin suspicious_strings] Added a rule to detect embedded PE executables in binaries.

bin/yara_rules/domains.yara

@@ -0,0 +1,37 @@
+/*
+    This file is part of Manalyze.
+
+    Manalyze is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Manalyze is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Manalyze.  If not, see <http://www.gnu.org/licenses/>.
+*/
+import "manape"
+
+rule Domains_URLs
+{
+    meta:
+        description = "Contains domain names"
+        author = "Sergey Mineev"
+    strings:
+        $domain1 = /www\.[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9]\.[a-zA-Z]{2,}$/
+        $domain2 = /[a-zA-Z0-9\-\.]{5,}\.(com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us)/ nocase fullword
+        $domain3 = /(https?|ftp):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-]*[\w\-])?/
+        $domain4 = /(ht|f)tps?\:\/\/[a-zA-Z0-9\-\._]+(\.[a-zA-Z0-9\-\._]+){2,}(\/?)([a-zA-Z0-9\-\.\?\,\'\/\\\+&%\$#_]*)/
+        $domain5 = /https?\:\/\/www.[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}/ fullword
+        $domain6 = /[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]{5,}\.((com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us))/ fullword nocase
+        $domain7 = /[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]{5,}\.(com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us)/ fullword nocase
+    condition:
+		// Calling C++ code in Manalyze takes care of filtering results in the authenticode signature or RT_MANIFEST resource.
+		// This is needed because Yara reports all matching strings if the condition evaluates to "true", even if some of the strings
+		// are located in a zone excluded in the condition.
+		any of them
+}

bin/yara_rules/suspicious_strings.yara

@@ -603,16 +603,12 @@ rule VMWare_Detection : AntiVM
         // VMware MAC addresses
         $vmware_mac_1a = "00-05-69" wide ascii
         $vmware_mac_1b = "00:05:69" wide ascii
-        $vmware_mac_1c = "000569" wide ascii
         $vmware_mac_2a = "00-50-56" wide ascii
         $vmware_mac_2b = "00:50:56" wide ascii
-        $vmware_mac_2c = "005056" wide ascii
         $vmware_mac_3a = "00-0C-29" nocase wide ascii
         $vmware_mac_3b = "00:0C:29" nocase wide ascii
-        $vmware_mac_3c = "000C29" nocase wide ascii
         $vmware_mac_4a = "00-1C-14" nocase wide ascii
         $vmware_mac_4b = "00:1C:14" nocase wide ascii
-        $vmware_mac_4c = "001C14" nocase wide ascii
 
         // PCI Vendor IDs, from Hacking Team's leak
         $virtualbox_vid_1 = "VEN_15ad" nocase wide ascii
@@ -1559,6 +1555,17 @@ rule Base64d_PE
         any of them
 }
 
+rule Embedded_PE
+{
+	meta:
+		description = "Contains another PE executable"
+		author = "Ivan Kwiatkowski (@JusticeRage)"
+	strings:
+		$a = "This program cannot be run in DOS mode."
+	condition:
+		for any i in (1..#a): ( @a[i] > 0x4E  and uint16(@a[i]-0x4E) == 0x5A4D )
+}
+
 rule Misc_Suspicious_Strings
 {
     meta:
@@ -1575,25 +1582,6 @@ rule Misc_Suspicious_Strings
         any of them
 }
 
-rule Domains_URLs
-{
-    meta:
-        description = "Contains domain names or URLs"
-        author = "Sergey Mineev"
-    strings:
-		// TLD list taken from: https://data.iana.org/TLD/tlds-alpha-by-domain.txt
-        $domain1 = /www\.[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9]\.[a-zA-Z]{2,}$/
-        $domain2 = /[a-zA-Z0-9\-\.]{5,}\.(com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us)/ nocase fullword
-        $domain3 = /(https?|ftp):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-]*[\w\-])?/
-        $domain4 = /(ht|f)tps?\:\/\/[a-zA-Z0-9\-\._]+(\.[a-zA-Z0-9\-\._]+){2,}(\/?)([a-zA-Z0-9\-\.\?\,\'\/\\\+&%\$#_]*)/
-        $domain5 = /https?\:\/\/www.[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}/ fullword
-        $domain6 = /[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]{5,}\.((com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us))/ fullword nocase
-        $domain7 = /[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]+\.[a-zA-Z0-9\-\.]{5,}\.(com|org|net|de|uk|fr|ru|info|top|xyz|tk|cn|br|jp|it|ir|nl|ca|au|es|ch|gov|edu|se|us)/ fullword nocase
-    condition:
-		// Exclude the authenticode signature because it often contains URLs related to the certificate authority.
-        for any of them: ($ in (0..manape.authenticode.start) or $ in (manape.authenticode.start..filesize))
-}
-
 rule BITS_CLSID
 {
     meta:
@@ -1682,6 +1670,6 @@ rule CVE_2020_0601
     strings:
         $oid = { 06 07 2a 86 48 ce 3d 01 01 }
     condition:
-        $oid in (manape.authenticode.start..filesize)        
+        $oid in (manape.authenticode.start..manape.authenticode.end)
 }
 

plugins/plugins_yara.cpp

@@ -43,6 +43,46 @@ void delete_manape_module_data(manape_data* data)
 
 // ----------------------------------------------------------------------------
 
+/**
+ *	@brief	A predicate which excludes strings found in PE regions created by
+ *			the compiler or Visual Studio.
+ *
+ *	Currently, this excludes both the authenticode signature and the 
+ *	RT_MANIFEST resource.
+ *
+ *	@param	pe The PE on which the Yara rule was run.
+ *	@param	m The structure representing the match to evaluate.
+ *
+ *	@return	Whether the match should be kept (true) or discarded (false).
+ */
+bool exclude_microsoft_data(const mana::PE& pe, yara::Match::pSingleMatch m)
+{
+	auto offset = m->get_offset();
+
+	const auto resources = pe.get_resources();
+	if (resources != nullptr)
+	{
+		// Exclude matches located in the authenticode section.
+		auto ioh = pe.get_image_optional_header();
+		if (ioh && 
+			ioh->directories[IMAGE_DIRECTORY_ENTRY_SECURITY].VirtualAddress < offset &&
+			offset < static_cast<boost::uint64_t>(ioh->directories[IMAGE_DIRECTORY_ENTRY_SECURITY].VirtualAddress) + ioh->directories[IMAGE_DIRECTORY_ENTRY_SECURITY].Size) {
+			return false;
+		}
+
+		for (auto& it : *resources)
+		{
+			// Exclude matches located inside the RT_MANIFEST
+			if (*it->get_type() == "RT_MANIFEST" && it->get_offset() < offset && offset < static_cast<boost::uint64_t>(it->get_offset()) + it->get_size()) {
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
+// ----------------------------------------------------------------------------
+
 class YaraPlugin : public IPlugin
 {
 
@@ -67,7 +107,7 @@ class YaraPlugin : public IPlugin
 				 LEVEL level, 
 				 const std::string& meta_field_name,
 				 bool show_strings = false,
-				 bool (*callback)(const std::string&) = nullptr)
+				 bool (*callback)(const mana::PE&, yara::Match::pSingleMatch) = nullptr)
 	{
 		pResult res = create_result();
 		if (!_load_rules()) {
@@ -84,8 +124,13 @@ class YaraPlugin : public IPlugin
 				auto found = it->get_found_strings();
 				if (callback != nullptr)
 				{
-					std::set<std::string> found_filtered;
-					std::copy_if(found.begin(), found.end(), std::inserter(found_filtered, found_filtered.end()), callback);
+					std::vector<yara::Match::pSingleMatch> found_filtered;
+					for (const auto& single_match : found)
+					{
+						if (callback(pe, single_match)) {
+							found_filtered.push_back(single_match);
+						}
+					}
 					found = found_filtered;
 				}
 				if (found.empty()) {
@@ -98,10 +143,16 @@ class YaraPlugin : public IPlugin
 				}
 				else
 				{
+					// Create a set of all the matching strings to delete duplicates.
+					std::set<std::string> found_unique;
+					for (const auto& it2 : found) {
+						found_unique.insert(it2->get_str());
+					}
+
 					io::pNode output = boost::make_shared<io::OutputTreeNode>(it->operator[](meta_field_name),
 					io::OutputTreeNode::STRINGS, io::OutputTreeNode::NEW_LINE);
 
-					for (const auto& it2 : found) {
+					for (const auto& it2 : found_unique) {
 						output->append(it2);
 					}
 					res->add_information(output);
@@ -169,6 +220,7 @@ class YaraPlugin : public IPlugin
                 {
                     res->sections[i].start = sections->at(i)->get_pointer_to_raw_data();
                     res->sections[i].size = sections->at(i)->get_size_of_raw_data();
+					res->sections[i].end = res->sections[i].start + res->sections[i].size;
                 }
             }
             else
@@ -179,7 +231,7 @@ class YaraPlugin : public IPlugin
             }
         }
 
-        // Add VERSION_INFO location for some ClamAV signatures
+        // Add VERSION_INFO and MANIFEST location for some ClamAV signatures and rules
         const auto resources = pe.get_resources();
 		if (resources != nullptr)
 		{
@@ -189,7 +241,13 @@ class YaraPlugin : public IPlugin
 				{
 					res->version_info.start = it->get_offset();
 					res->version_info.size = it->get_size();
-					break;
+					res->version_info.end = res->version_info.start + res->version_info.size;
+				}
+				else if (*it->get_type() == "RT_MANIFEST")
+				{
+					res->manifest.start = it->get_offset();
+					res->manifest.size = it->get_size();
+					res->manifest.end = res->manifest.start + res->manifest.size;
 				}
 			}
 		}
@@ -199,6 +257,7 @@ class YaraPlugin : public IPlugin
 		{
 			res->authenticode.start = ioh->directories[IMAGE_DIRECTORY_ENTRY_SECURITY].VirtualAddress;
 			res->authenticode.size = ioh->directories[IMAGE_DIRECTORY_ENTRY_SECURITY].Size;
+			res->authenticode.end = res->authenticode.start + res->authenticode.size;
 		}
 
         return res;
@@ -290,8 +349,26 @@ class SuspiciousStringsPlugin : public YaraPlugin
 public:
 	SuspiciousStringsPlugin() : YaraPlugin("yara_rules/suspicious_strings.yara") {}
 
-	pResult analyze(const mana::PE& pe) override {
-		return scan(pe, "Strings found in the binary may indicate undesirable behavior:", SUSPICIOUS, "description", true);
+	pResult analyze(const mana::PE& pe) override 
+	{
+		auto res = scan(pe, "Strings found in the binary may indicate undesirable behavior:", SUSPICIOUS, "description", true);
+		// Scan for domain names separately as results need to be filtered out.
+		_rule_file = "yara_rules/domains.yara";
+		// Search for domain names in the PE body, but exclude irrelevant regions.
+		auto domains = scan(pe, "Interesting strings found in the binary:", NO_OPINION, "description", true, exclude_microsoft_data);
+
+		// If one of the rules didn't return anything, return the output of the other one (which may be empty too).
+		if (!res || !res->get_output()) {
+			return domains;
+		}
+		else if (!domains || !domains->get_output()) {
+			return res;
+		}
+
+		// Otherwise, merge the results.
+		res->merge(*domains);
+
+		return res;
 	}
 
 	boost::shared_ptr<std::string> get_id() const override {
@@ -354,9 +431,11 @@ class CryptoCurrencyAddress : public YaraPlugin
 
 	pResult analyze(const mana::PE& pe) override
 	{
-		auto btc = scan(pe, "This program may be a ransomware.", MALICIOUS, "description", true, hash::test_btc_address);
+		auto btc = scan(pe, "This program may be a ransomware.", MALICIOUS, "description", true, 
+			[] (const mana::PE&, yara::Match::pSingleMatch m) { return hash::test_btc_address(m->get_str()); });
 		_rule_file = "yara_rules/monero.yara";
-		auto monero = scan(pe, "This program may be a miner.", MALICIOUS, "description", true, hash::test_xmr_address);
+		auto monero = scan(pe, "This program may be a miner.", MALICIOUS, "description", true, 
+			[] (const mana::PE&, yara::Match::pSingleMatch m) { return hash::test_xmr_address(m->get_str()); });
 
 		// If one of the plugins didn't return anything, return the output of the other one (which may be empty too).
 		if (!btc || !btc->get_output()) {

src/main.cpp

@@ -169,7 +169,7 @@ bool validate_args(po::variables_map& vm, po::options_description& desc, char**
 				continue;
 			}
 
-			auto found = std::find_if(plugins.begin(), plugins.end(), boost::bind(&plugin::name_matches, *it, _1));
+			auto found = std::find_if(plugins.begin(), plugins.end(), boost::bind(&plugin::name_matches, *it, boost::placeholders::_1));
 			if (found == plugins.end())
 			{
 				print_help(desc, argv[0]);