URLs cleaned up

hesussavas · Jul 20, 2017 · 8001758 · 8001758
1 parent 5c23489
commit 8001758
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/corners442/corners442/spiders/four_four_two.py b/corners442/corners442/spiders/four_four_two.py
@@ -23,8 +23,8 @@ def start_requests(self):
 
         for league in leagues:
             for year in range(2010, 2015):
-                url = 'https://www.fourfourtwo.com/statszone/results/{}-{}'.format(
-                    league.league_id, year)
+                url = urljoin(BASE_URL, 'statszone/results/{}-{}'.format(
+                    league.league_id, year))
                 request = Request(url, self.parse)
                 request.meta['league_id'] = league.league_id
                 request.meta['year'] = year

diff --git a/corners442/corners442/spiders/leagues.py b/corners442/corners442/spiders/leagues.py
@@ -1,3 +1,5 @@
+from urllib.parse import urljoin
+
 from scrapy.http.request import Request
 from scrapy.spiders import Spider
 
@@ -15,7 +17,7 @@ class MySpider(Spider):
 
     def start_requests(self):
 
-        yield Request(BASE_URL, self.parse)
+        yield Request(urljoin(BASE_URL, 'statszone'), self.parse)
 
     def parse(self, response):
         # get competitions name and id: