Merge pull request #10 from AbuseIO/master

Merge from upstream
AbuseIO · Apr 3, 2015 · 2f6f9ee · 2f6f9ee
2 parents bf2215c + 7a65434
commit 2f6f9ee
Show file tree

Hide file tree

Showing 43 changed files with 214 additions and 117 deletions.
diff --git a/INSTALL.md b/INSTALL.md
@@ -144,6 +144,14 @@ AbuseIO will process all (new) mails from this mailbox. If parsing succeeds fetc
 read. If the mail cannot be parsed by AbuseIO, fetchmail will not touch the email. If you want to re-process 
 an abuse report, simply mark the abuse email as (new) and run fetchmail again.
 
+### Using ASH information texts
+
+We include a default set of information texts per class in APP/www/ash/infotext/defaults/ split up in multiple
+languages. These are shown in combination with a report of that named class.
+
+If you want your own text with a class you can create the class html in APP/www/ash/infotext/ with the same name.
+files in this folder are preferred over the defaults. You for conviniance we included a little template.html
+file to get you started.
 
 ## Note on Patches/Pull Requests
 

diff --git a/bin/housekeeper b/bin/housekeeper
@@ -32,7 +32,19 @@ if(!reportHousekeeping()) {
     logger(LOG_ERR, "Housekeeper failed to do reports");
 }
 
-if (null !== COLLECTOR_SNDS_ENABLED && COLLECTOR_SNDS_ENABLED == true) {
+if (defined('REMOVE_EVIDENCE')) {
+    logger(LOG_DEBUG, "Housekeeper - starting old evidence removal (older then ".REMOVE_EVIDENCE.")");
+    $timestamp = strtotime(REMOVE_EVIDENCE . " ago");
+    if(date('Y', $timestamp) < 2013 || strtotime(date('d-m-Y H:i:s',$timestamp)) !== (int)$timestamp) {
+        logger(LOG_WARNING, " Evidence cleanup was called with an incorrect timestamp (${timestamp})");
+    } else {
+        if(!evidenceCleanup($timestamp)) {
+            logger(LOG_ERR, "Housekeeper - cleanup of old evidence failed");
+        }
+    }
+}
+
+if (defined('COLLECTOR_SNDS_ENABLED') && COLLECTOR_SNDS_ENABLED == true) {
     logger(LOG_DEBUG, "Housekeeper - starting SNDS collector");
 
     $config = array();
@@ -41,21 +53,23 @@ if (null !== COLLECTOR_SNDS_ENABLED && COLLECTOR_SNDS_ENABLED == true) {
     }
 }
 
-if (null !== COLLECTOR_OSINT_ENABLED && COLLECTOR_OSINT_ENABLED == true) {
+if (defined('COLLECTOR_OSINT_ENABLED') && COLLECTOR_OSINT_ENABLED == true) {
     logger(LOG_DEBUG, "Housekeeper - starting OSINT collector");
     $config = array();
     if(!collect_osint($config)) {
         logger(LOG_ERR, "Housekeeper failed to run OSINT collector");
     }
 }
 
-if (null !== COLLECTOR_RBLSCAN_ENABLED && COLLECTOR_RBLSCAN_ENABLED == true) {
+if (defined('COLLECTOR_RBLSCAN_ENABLED') && COLLECTOR_RBLSCAN_ENABLED == true) {
     logger(LOG_DEBUG, "Housekeeper - starting RBLSCAN collector");
     $config = array();
     if(!collect_rblscan($config)) {
         logger(LOG_ERR, "Housekeeper failed to run RBLSCAN collector");
     }
 }
 
+
+
 logger(LOG_DEBUG, "Housekeeper completed");
 ?>
diff --git a/bin/rblscanner b/bin/rblscanner
@@ -26,7 +26,7 @@
 
 include (realpath(dirname(__FILE__))."/../lib/core/loader.php");
 
-$usage  = 'Usage: --asn 666 --range 1.1.1.1/24 --range 2.2.2.2/8 --rbl rblname:/var/rbldnsd/data/rbl1.bla.tld --rbl rblname:rbl2.bla.tld --file file.list' . PHP_EOL;
+$usage  = 'Usage: --asn 666 --range 1.1.1.1/24 --range 2.2.2.2/8 --rbl source:zone:/var/rbldnsd/data/rbl1.bla.tld --rbl source:rbl2.bla.tld --file file.list' . PHP_EOL;
 $usage .= '       example --range 1.1.1.0/24 --rbl Spamhaus:zen.spamhaus.org' . PHP_EOL;
 
 $shortopts = '';
@@ -35,6 +35,7 @@ $longopts  = array(
                     'rbl:',
                     'range:',
                     'file:',
+                    'debug:',
                  );
 $options = getopt($shortopts, $longopts);
 
@@ -60,19 +61,33 @@ if (!empty($options['asn'])) {
     die($usage.PHP_EOL);
 }
 
+
+// build a list with RBL's and their zone(file)
 if (!empty($options['rbl'])) {
-    if(is_array($options['rbl'])) {
-        foreach($options['rbl'] as $rblpart) {
-            $parts = explode(":", $rblpart);
-            $rbllist[$parts[0]] = $parts[1];
+    if(!is_array($options['rbl'])) {
+        $tmp = $options['rbl'];
+	$options['rbl'] = array(''=>$tmp);
+    }
+
+    foreach($options['rbl'] as $rblpart) {
+        $parts = explode(":", $rblpart);
+
+        if(count($parts) == 2) {
+            $rbllist[$parts[0]] = array(
+                                        'name' => $parts[1],
+                                        'zone' => $parts[1],
+                                       );
+        } elseif(count($parts) == 3) {
+            $rbllist[$parts[0]] = array(
+                                        'name' => $parts[1],
+                                        'file' => $parts[2],
+                                       );
+        } else {
+            die("ERROR: Arguments wrong" .PHP_EOL);
         }
-    } else {
-        $parts = explode(":", $options['rbl']);
-        $rbllist[$parts[0]] = $parts[1];
     }
 }
 
-
 if(empty($iplist) || !is_array($iplist)) {
     die($usage.PHP_EOL);
 }
@@ -98,93 +113,91 @@ foreach($iplist as $netblock) {
 }
 unset($iplist);
 
+foreach($rbllist as $source => $rbl) {
+    logger(LOG_DEBUG, "RBLSCANNER CLI starting with RBL: ${rbl['name']}");
 
-foreach($netblocks as $netblock => $info) {
-    logger(LOG_DEBUG, "RBLSCANNER CLI scanning $netblock");
-
-    foreach($rbllist as $rblname => $rblhost) {
+    if(!empty($rbl['file']) && is_file($rbl['file'])) {
+        // Walk the RBL file and match each IP to known local IP's
+        logger(LOG_DEBUG, "RBLSCANNER CLI using zonefile");
 
-        if(is_file($rblhost)) {
-            logger(LOG_DEBUG, "RBLSCANNER CLI start loading rbldnsd zonefile $rblhost");
-            $rbldata = parse_zonefile($rblhost);
-            logger(LOG_DEBUG, "RBLSCANNER CLI completed loading rbldnsd zonefile $rblhost");
-        }
-
-        for($pos = $info['begin']; $pos <= $info['end']; $pos++) {
-            $ip = long2ip($pos);
-            $ip_reversed = implode('.',array_reverse(preg_split('/\./',$ip)));
+	// Build a cache of local IP's for quick in_array matching
+	foreach($netblocks as $netblock => $info) {
+            for($pos = $info['begin']; $pos <= $info['end']; $pos++) {
+                $ipcache[long2ip($pos)] = '';
+            }
+	}
 
-            $lookup = $ip_reversed.'.'.$rblhost;
+        $handle = @popen("cat ${rbl['file']}", "r");
+        if ($handle) {
+            while (!feof($handle)) {
+                $line = str_replace("\n","",fgets($handle, 4096));
+                $ip = filter_var($line, FILTER_VALIDATE_IP);
 
-            if(is_file($rblhost)) {
-                // Use zone file
-                if(in_array($ip, $rbldata)) {
+                if (strpos($line, ":127.0.0") !== false) {
+                    preg_match('/:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*/', $line, $match);
+                    $code = $match[1];
+                }
 
+                if(isset($ipcache[$ip])) {
                     $outReport = array(
-                        'source'=>$rblname,
-                        'ip'=>$ip,
-                        'class'=>'RBL Listed',
-                        'type'=>'INFO',
-                        'timestamp'=>time(),
-                        'information'=> array(
-                                           'Return code' => '127.0.0.1',
-                                             )
+                                'source'=>$source,
+                                'ip'=>$ip,
+                                'class'=>'RBL Listed',
+                                'type'=>'INFO',
+                                'timestamp'=>time(),
+                                'information'=> array(
+                                                        'List' => $rbl['name'],
+                                                        'Return code' => $code,
+                                                     )
                     );
 
                     if (!reportAdd($outReport)) {
+                        logger(LOG_DEBUG, "RBLSCANNER CLI died with: " . implode(",", $outReport));
                         die('error adding report'.PHP_EOL);
                     }
-
 		}
+            }
+        }
+        pclose($handle);
 
-            } elseif(dns_get_record($rblhost, DNS_SOA)) {
-                // Use DNS queries
-                if ($result = gethostbyname($lookup)) {
-                    if($result != $lookup) {
-
-                        $outReport = array(
-                            'source'=>$rblname,
-                            'ip'=>$ip,
-                            'class'=>'RBL Listed',
-                            'type'=>'INFO',
-                            'timestamp'=>time(),
-                            'information'=> array(
-                                           'Return code' => $result,
-                                             )
-                        );
-
-                        if (!reportAdd($outReport)) {
-                            die('error adding report'.PHP_EOL);
+    } else {
+        // Resolve each IP by DNS Queries
+        logger(LOG_DEBUG, "RBLSCANNER CLI using DNS queries");
+
+        foreach($netblocks as $netblock => $info) {
+            logger(LOG_DEBUG, "RBLSCANNER CLI scanning $netblock");
+
+            for($pos = $info['begin']; $pos <= $info['end']; $pos++) {
+                $ip = long2ip($pos);
+                $ip_reversed = implode('.',array_reverse(preg_split('/\./',$ip)));
+                $lookup = $ip_reversed.'.'.$rbl['zone'];
+
+                if(dns_get_record($rbl['zone'], DNS_SOA)) {
+                    if ($result = gethostbyname($lookup)) {
+                        if($result != $lookup) {
+
+                            $outReport = array(
+                                'source'=>$source,
+                                'ip'=>$ip,
+                                'class'=>'RBL Listed',
+                                'type'=>'INFO',
+                                'timestamp'=>time(),
+                                'information'=> array(
+							'List' => $rbl['name'],
+                                                      	'Return code' => $result,
+                                                     )
+                            );
+
+                            if (!reportAdd($outReport)) {
+                                logger(LOG_DEBUG, "RBLSCANNER CLI died with: " . implode(",", $outReport));
+                                die('error adding report'.PHP_EOL);
+                            }
                         }
                     }
                 }
-            } else {
-                logger(LOG_DEBUG, "RBLSCANNER CLI cant detect RBL type");
             }
         }
     }
 }
 logger(LOG_DEBUG, "RBLSCANNER CLI completed");
-
-
-function parse_zonefile($file) {
-    if(!is_file($file)) {
-        return false;
-    }
-
-    $rbllist = array();
-    $regexp = "";
-    $handle = @popen("cat ${file}", "r");
-    if ($handle) {
-        while (!feof($handle)) {
-            $line = str_replace("\n","",fgets($handle, 4096));
-            $ip = filter_var($line, FILTER_VALIDATE_IP);
-            $rbllist[] = $ip;
-        }
-    }
-    pclose($handle);
-
-    return $rbllist;
-}
-
 ?>
diff --git a/etc/settings.conf.example b/etc/settings.conf.example
@@ -2,6 +2,7 @@ FALLBACK_MAIL = bounce@host.tld                 ; this is required to bounce inc
 TIME_ZONE = Europe/Amsterdam
 KEEP_MAILS = true                               ; this will store all incoming mail in a file in APP/archive/ for future use (e.g. reseeding database)
 KEEP_EVIDENCE = true                            ; this will store all incoming mail into a evidence table and links each related ticket
+REMOVE_EVIDENCE = 500 days                       ; this will remove all evidence from the database after 'x days, x months' etc
 SQL_HOST = localhost
 SQL_USER = abuseio
 SQL_PASS = 

diff --git a/lib/collectors/osint.php b/lib/collectors/osint.php
@@ -86,7 +86,6 @@ function collect_osint($config) {
                 $outReport['information']   = array();
                 $outReport['timestamp']     = '';
 
-                print_r($fields);
             }
         }
     }

diff --git a/lib/collectors/snds.php b/lib/collectors/snds.php
@@ -36,7 +36,7 @@ function collect_snds($config) {
         logger(LOG_DEBUG,'No SNDS key specified in config, skipping SNDS reporting');
         return false;
 
-    } else if ($data = file_get_contents('https://postmaster.live.com/snds/ipStatus.aspx?key='.COLLECTOR_SNDS_KEY)) {
+    } else if ($data = @file_get_contents('https://postmaster.live.com/snds/ipStatus.aspx?key='.COLLECTOR_SNDS_KEY)) {
         $sndsMap = array(
             'E-mail address harvesting'=>array(
                 'class'=>'Harvesting',
@@ -53,7 +53,11 @@ function collect_snds($config) {
                 'information'=>array(
                     'delisting_url'=>'https://www.spamhaus.org/lookup/',
                 ),
-            )
+            ),
+            'Blocked due to user complaints or other evidence of spamming'=>array(
+                'class'=>'SPAM',
+                'information'=>array(),
+            ),
         );
         preg_match_all('/([^,]+),([^,]+),([^,]+),([^\r\n]+)\r?\n/',$data, $regs);
         $first_ip = $regs[1];

diff --git a/lib/core/evidence.php b/lib/core/evidence.php
@@ -93,6 +93,39 @@ function evidenceList($ticket) {
 }
 
 
+/*
+** Function: evidenceCleanup
+** Parameters:
+**  (timestamp): Cleans up all old evidence from SQL (NOT! the archive)
+** Returns:
+**  (boolean): SQL result
+*/
+function evidenceCleanup($timestamp) {
+    if(!is_numeric($timestamp)) {
+        return false;
+    }
+
+    $reports = array();
+
+    $query = "SELECT ID, LastModified FROM Evidence WHERE 1 AND LastModified < FROM_UNIXTIME('${timestamp}');";
+
+    $evidences = _mysqli_fetch($query);
+
+    foreach($evidences as $evidence) {
+        if(_mysqli_query("DELETE FROM EvidenceLinks WHERE EvidenceID = '${evidence['ID']}';")) {
+            if(!_mysqli_query("DELETE FROM Evidence WHERE ID = '${evidence['ID']}';")) {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+
+
 /*
 ** Function: evidenceGet
 ** Parameters: