Added ability to crawl only allowed domains in config file

rtrevinnoc · Mar 7, 2021 · 0835a05 · 0835a05
1 parent 1d8810b
commit 0835a05
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -38,6 +38,8 @@ HOST_NAME = "my_public_future_instance"         # THE NAMES 'private' and 'weare
 with open("tranco_JKGY.csv") as tranco:
         SEED_URLS = [x.strip() for x in tranco.readlines()]
 PEER_PORT = 3000
+LIMIT_DOMAINS = None
+ALLOWED_DOMAINS = ["*"]
 CONCURRENT_REQUESTS = 10
 CONCURRENT_REQUESTS_PER_DOMAIN = 2.0
 CONCURRENT_ITEMS = 100

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -47,6 +47,8 @@ It is suggested to start with this configuration template, which is essentially
         with open("tranco_JKGY.csv") as tranco:
                 SEED_URLS = [x.strip() for x in tranco.readlines()]
         PEER_PORT = 3000
+        LIMIT_DOMAINS = None
+        ALLOWED_DOMAINS = ["*"]
         CONCURRENT_REQUESTS = 10
         CONCURRENT_REQUESTS_PER_DOMAIN = 2.0
         CONCURRENT_ITEMS = 100

diff --git a/linkreaper.py b/linkreaper.py
@@ -27,7 +27,7 @@
 from scrapy.crawler import CrawlerProcess
 from nltk.tokenize import word_tokenize
 from gensim.models import KeyedVectors
-from config import SEED_URLS, CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN, CONCURRENT_ITEMS, REACTOR_THREADPOOL_MAXSIZE, DOWNLOAD_MAXSIZE, LOG_LEVEL, AUTOTHROTTLE, DEPTH_PRIORITY, TARGET_CONCURRENCY, MAX_DELAY, START_DELAY
+from config import SEED_URLS, CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN, CONCURRENT_ITEMS, REACTOR_THREADPOOL_MAXSIZE, DOWNLOAD_MAXSIZE, LOG_LEVEL, AUTOTHROTTLE, DEPTH_PRIORITY, TARGET_CONCURRENCY, MAX_DELAY, START_DELAY, LIMIT_DOMAINS, ALLOWED_DOMAINS
 from Monad import *
 import numpy as np
 
@@ -123,7 +123,7 @@ def returnDataFromImageTags(url: str, someIterable: list) -> list:
 
 class Indexer(scrapy.Spider):
     name = "indexer"
-    allowed_urls = ["*"]
+    allowed_urls = ALLOWED_DOMAINS
     custom_settings = {
         "CONCURRENT_REQUESTS": CONCURRENT_REQUESTS,
         "CONCURRENT_REQUESTS_PER_DOMAIN": CONCURRENT_REQUESTS_PER_DOMAIN,
@@ -151,7 +151,10 @@ class Indexer(scrapy.Spider):
         "DEPTH_PRIORITY": DEPTH_PRIORITY,
         "SCHEDULER_DISK_QUEUE": 'scrapy.squeues.PickleFifoDiskQueue',
         "SCHEDULER_MEMORY_QUEUE": 'scrapy.squeues.FifoMemoryQueue',
-        "AJAXCRAWL_ENABLED": True
+        "AJAXCRAWL_ENABLED": True,
+        "SPIDER_MIDDLEWARES": {
+            'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': LIMIT_DOMAINS
+        }
     }
 
     start_urls = SEED_URLS