From b717db9b6bf9444116f57e35d1cd3c4fe97bcf70 Mon Sep 17 00:00:00 2001
From: computermacgyver <computermacgyver@gmail.com>
Date: Sun, 4 Nov 2012 10:10:34 +0000
Subject: [PATCH] add whitelistURL option

---
 LIB_simple_spider.php | 18 ++++++++++++++++++
 example_CONFIG_db.php |  9 +++++++++
 2 files changed, 27 insertions(+)

diff --git a/LIB_simple_spider.php b/LIB_simple_spider.php
index 8185414..54dcb47 100644
--- a/LIB_simple_spider.php
+++ b/LIB_simple_spider.php
@@ -45,6 +45,12 @@
 	$whitelistdomainlist_part=":$whitelistdomainlist:";
 }
 
+global $whitelisturl,$whitelisturllist,$whitelisturllist_arr;
+if ($whitelisturl) {
+	$whitelisturl_arr=explode(":",$whitelisturllist);
+}
+
+
 
 
 
@@ -339,6 +345,18 @@ function exclude_link($link)
 				$exclude=true;
 			}
 	     }
+	   }
+	   
+	   global $whitelisturl,$$whitelisturllist_arr
+	   if ($whitelisturl) {
+      	 	$found=false;
+      	 	for ($x=0;$x<count($whitelisturllist_arr);$x++) {
+      	 		if (strpos($link,$whitelisturllist_arr[$x])!==false) {
+      	 			$found=true;
+      	 			break;
+      	 		}
+      	 	}
+      	 	if ($found===false) $exclude=true;
 	   }      
 
     return $exclude;
diff --git a/example_CONFIG_db.php b/example_CONFIG_db.php
index b524345..135539a 100644
--- a/example_CONFIG_db.php
+++ b/example_CONFIG_db.php
@@ -28,6 +28,15 @@
 $whitelistdomainlevel=2; 
 //list of domains separated with : (no starting / ending :)
 $whitelistdomainlist="gov.uk:org.uk";
+
+//Restrict crawling to a URL whitelist? true | false
+$whitelisturl=false;	
+//list of phrases that must appear in url separated with : (no starting / ending :)
+$whitelisturllist="example.com/this-part-only:anothersite.com:thirdsite.com/with-sub-section";
+
+
+
+
 //Fetch only first part of each page, to avoid huge files? (Experimental!)
 $fetchrangeonly=false;