Mimic web browser to validate user help URLs

planemo/lint.py

@@ -11,6 +11,7 @@
     URLError,
 )
 from six.moves.urllib.request import (
+    Request,
     urlopen,
 )
 
@@ -126,12 +127,19 @@ def lint_xsd(lint_ctx, schema_path, path):
 
 def lint_urls(root, lint_ctx):
     """Find referenced URLs and verify they are valid."""
-    urls = find_urls_for_xml(root)
+    urls, docs = find_urls_for_xml(root)
 
-    def validate_url(url, lint_ctx):
+    # This is from Google Chome 53.0.2785.143, current at time of writing:
+    BROWSER_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"
+
+    def validate_url(url, lint_ctx, user_agent=None):
         is_valid = True
+        if user_agent:
+            req = Request(url, headers={"User-Agent": user_agent})
+        else:
+            req = url
         try:
-            handle = urlopen(url)
+            handle = urlopen(req)
             handle.read(100)
         except HTTPError as e:
             if e.code == 429:
@@ -148,6 +156,8 @@ def validate_url(url, lint_ctx):
 
     for url in urls:
         validate_url(url, lint_ctx)
+    for url in docs:
+        validate_url(url, lint_ctx, BROWSER_USER_AGENT)
 
 
 __all__ = [

planemo/shed/__init__.py

@@ -223,6 +223,11 @@ def process_repo(realized_repository):
 
 
 def find_urls_for_xml(root):
+    """Returns two lists: explicit package URLs, and help text URLs.
+
+    For validating the user-facing URLs is it sensible to mimic
+    a web browser user agent.
+    """
     urls = []
     for packages in root.findall("package"):
         install_els = packages.findall("install")
@@ -240,11 +245,12 @@ def find_urls_for_xml(root):
                 if hasattr(subaction, 'packages'):
                     urls.extend(subaction.packages)
 
+    docs = []
     for help_text in root.findall("help"):
         for url in _find_urls_in_text(help_text.text):
-            urls.append(url[0])
+            docs.append(url[0])
 
-    return urls
+    return urls, docs
 
 
 def handle_force_create(realized_repository, ctx, shed_context, **kwds):