Improve checklinks script and its Pipfile

Pipfile

@@ -11,7 +11,7 @@ requests = "*"
 colorama = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3"
 
 [scripts]
 checklinks = "./checklinks.py"

checklinks.py

@@ -8,6 +8,44 @@
 import requests
 
 
+def checklink(url, headers=None):
+    try:
+        response = requests.head(url, allow_redirects=True, headers=headers)
+
+        if response.status_code == 200:
+            ok = True
+        # 403 = forbidden
+        # Some shitty WAFs seem to dislike Python as user agent
+        # -> fake user agent
+        elif response.status_code == 403:
+            user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
+            return checklink(url, headers={'User-Agent': user_agent}) 
+        # 405 = method not allowed
+        # -> retry with GET
+        elif response.status_code == 405:
+            response = requests.get(url, headers=headers)
+
+            if response.status_code == 200:
+                ok = True
+            else:
+                ok = False
+        else:
+            ok = False
+    except requests.exceptions.SSLError:
+        ok = False
+        print('SSL error for {}'.format(url), file=sys.stderr)
+    except requests.ConnectionError:
+        ok = False
+        print('Can\'t connect to {}'.format(url), file=sys.stderr)
+
+    if ok:
+        print(Fore.GREEN + 'OK' + Style.RESET_ALL)
+        return True
+
+    print(Fore.RED + 'FAIL' + Style.RESET_ALL)
+    return False
+
+
 def main():
     with open('htdocs/index.html') as f:
         html_doc = f.read()
@@ -19,30 +57,7 @@ def main():
 
         if url.startswith('http'):
             print('Checking URL {}'.format(url))
-
-            try:
-                response = requests.head(url, allow_redirects=True)
-
-                if response.status_code == 200:
-                    ok = True
-                # 405 = method not allowed -> retry with GET
-                elif response.status_code == 405:
-                    response = requests.get(url)
-
-                    if response.status_code == 200:
-                        ok = True
-                    else:
-                        ok = False
-                else:
-                    ok = False
-            except requests.ConnectionError:
-                ok = False
-                print('Can\'t connect to {}'.format(url), file=sys.stderr)
-
-            if ok:
-                print(Fore.GREEN + 'OK' + Style.RESET_ALL)
-            else:
-                print(Fore.RED + 'FAIL' + Style.RESET_ALL)
+            checklink(url)
 
 
 if __name__ == '__main__':