Skip to content
Browse files

[IMP] website: test_crawl dont follow external link

In case a controller with a relative url return an absolute link
the current crawler follow the redirect and all external link of this external url too.

Now we don't follow redirection if it is on an other netloc that the current one
  • Loading branch information
bvr-odoo committed Dec 2, 2019
1 parent be83965 commit 517783949b7052539c8b1921dfd512a5000b049e
Showing with 14 additions and 4 deletions.
  1. +11 −1 addons/website/tests/
  2. +3 −3 odoo/tests/
@@ -53,7 +53,17 @@ def crawl(self, url, seen=None, msg=''):
seen.add(url_slug)"%s %s", msg, url)
r = self.url_open(url)
r = self.url_open(url, allow_redirects=False)
if r.status_code in (301, 302):
# check local redirect to avoid fetch externals pages
new_url = r.headers.get('Location')
current_url = r.url
if urls.url_parse(new_url).netloc == urls.url_parse(current_url).netloc:
r = self.url_open(new_url)
_logger.warning('>>>>>>>> oups, %s %s', current_url, new_url)
return seen

code = r.status_code
self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))

@@ -1077,13 +1077,13 @@ def setUp(self):
self.opener = requests.Session()
self.opener.cookies['session_id'] = self.session_id

def url_open(self, url, data=None, files=None, timeout=10, headers=None):
def url_open(self, url, data=None, files=None, timeout=10, headers=None, allow_redirects=True):
if url.startswith('/'):
url = "http://%s:%s%s" % (HOST, PORT, url)
if data or files:
return, data=data, files=files, timeout=timeout, headers=headers)
return self.opener.get(url, timeout=timeout, headers=headers)
return, data=data, files=files, timeout=timeout, headers=headers, allow_redirects=allow_redirects)
return self.opener.get(url, timeout=timeout, headers=headers, allow_redirects=allow_redirects)

def _wait_remaining_requests(self, timeout=10):

0 comments on commit 5177839

Please sign in to comment.
You can’t perform that action at this time.