Skip to content
Permalink
Browse files

[IMP] website: test_crawl dont follow external link

In case a controller with a relative url return an absolute link
the current crawler follow the redirect and all external link of this external url too.

Now we don't follow redirection if it is on an other netloc that the current one
  • Loading branch information
bvr-odoo authored and qsm-odoo committed Dec 2, 2019
1 parent 2e00024 commit 8aeb2766f1ce0953ea2a5a7d4ebe9e0cbbcf3743
Showing with 14 additions and 4 deletions.
  1. +11 −1 addons/website/tests/test_crawl.py
  2. +3 −3 odoo/tests/common.py
@@ -53,7 +53,17 @@ def crawl(self, url, seen=None, msg=''):
seen.add(url_slug)

_logger.info("%s %s", msg, url)
r = self.url_open(url)
r = self.url_open(url, allow_redirects=False)
if r.status_code in (301, 302):
# check local redirect to avoid fetch externals pages
new_url = r.headers.get('Location')
current_url = r.url
if urls.url_parse(new_url).netloc == urls.url_parse(current_url).netloc:
r = self.url_open(new_url)
else:
_logger.warning('>>>>>>>> oups, %s %s', current_url, new_url)
return seen

code = r.status_code
self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))

@@ -1077,13 +1077,13 @@ def setUp(self):
self.opener = requests.Session()
self.opener.cookies['session_id'] = self.session_id

def url_open(self, url, data=None, files=None, timeout=10, headers=None):
def url_open(self, url, data=None, files=None, timeout=10, headers=None, allow_redirects=True):
self.env['base'].flush()
if url.startswith('/'):
url = "http://%s:%s%s" % (HOST, PORT, url)
if data or files:
return self.opener.post(url, data=data, files=files, timeout=timeout, headers=headers)
return self.opener.get(url, timeout=timeout, headers=headers)
return self.opener.post(url, data=data, files=files, timeout=timeout, headers=headers, allow_redirects=allow_redirects)
return self.opener.get(url, timeout=timeout, headers=headers, allow_redirects=allow_redirects)

def _wait_remaining_requests(self, timeout=10):

0 comments on commit 8aeb276

Please sign in to comment.
You can’t perform that action at this time.