add whitelistURL option

LIB_simple_spider.php

@@ -45,6 +45,12 @@
 	$whitelistdomainlist_part=":$whitelistdomainlist:";
 }
 
+global $whitelisturl,$whitelisturllist,$whitelisturllist_arr;
+if ($whitelisturl) {
+	$whitelisturl_arr=explode(":",$whitelisturllist);
+}
+
+
 
 
 
@@ -339,6 +345,18 @@ function exclude_link($link)
 				$exclude=true;
 			}
 	     }
+	   }
+
+	   global $whitelisturl,$$whitelisturllist_arr
+	   if ($whitelisturl) {
+      	 	$found=false;
+      	 	for ($x=0;$x<count($whitelisturllist_arr);$x++) {
+      	 		if (strpos($link,$whitelisturllist_arr[$x])!==false) {
+      	 			$found=true;
+      	 			break;
+      	 		}
+      	 	}
+      	 	if ($found===false) $exclude=true;
 	   }      
 
     return $exclude;

example_CONFIG_db.php

@@ -28,6 +28,15 @@
 $whitelistdomainlevel=2; 
 //list of domains separated with : (no starting / ending :)
 $whitelistdomainlist="gov.uk:org.uk";
+
+//Restrict crawling to a URL whitelist? true | false
+$whitelisturl=false;	
+//list of phrases that must appear in url separated with : (no starting / ending :)
+$whitelisturllist="example.com/this-part-only:anothersite.com:thirdsite.com/with-sub-section";
+
+
+
+
 //Fetch only first part of each page, to avoid huge files? (Experimental!)
 $fetchrangeonly=false;