Merge pull request #986 from Andries-Smit/dev

Add redirect scripts

utils/deleteAllRedirects.py

@@ -0,0 +1,58 @@
+import requests
+import json
+import time
+import sys
+import re
+
+PROJECT = 'androidaps'
+URL = 'https://readthedocs.org/api/v3/projects/' + PROJECT + '/redirects/'
+TOKEN = len(sys.argv) == 2 and sys.argv[1]
+HEADERS = {'Authorization': f'token {TOKEN}'}
+
+def delItem (url) :
+    delResponse = requests.delete(url, headers=HEADERS)
+    if delResponse.status_code == 204 :
+        print('removed ' + url)
+    elif delResponse.status_code == 429:
+        detail = delResponse.json()['detail']
+        wait = int(re.search(r'\d+', detail).group())
+        print('Throttled, wait for ' + str(wait + 1) + ' seconds')
+        time.sleep(wait + 1)
+        delItem(url)
+    else : 
+        results = delResponse.json()
+        print(results)
+
+def deleteList() :
+    response = requests.get(URL, headers=HEADERS)
+    listResult = response.json()
+
+    if response.status_code == 200:
+        redirects = listResult['results']
+        for redirect in redirects :
+            url = redirect['_links']['_self']
+            delItem(url)
+        return listResult['count']
+    elif response.status_code == 429:
+        detail = response.json()['detail']
+        wait = int(re.search(r'\d+', detail).group())
+        print('Throttled, wait for ' + str(wait + 1) + ' seconds')
+        time.sleep(wait + 1)
+        deleteList()
+    else :
+        print('ERROR code:', response.status_code, listResult)
+        return 0
+
+def main() :
+    while True:
+        count = deleteList()
+        print('Removed, still counting: ' + str(count))
+        if count == 0 :
+            break
+    print('done')
+
+if not TOKEN :
+    print('Please provide a API token as parameter')
+    print('useage: $ python deleteAllRedirects.py <apikey>')
+else :
+    main()

utils/generateRedirects.py

@@ -0,0 +1,38 @@
+import os
+import json
+
+FILE = 'redirect.json'
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+language_path = os.path.join(dir_path, '..\\', 'docs', 'CROWDIN') + '\\'
+print('Generate redirect for al .md and .rst files in path ' + language_path)
+
+redirects = []
+
+class Object:
+    def toJSON(self):
+       return json.dumps(self, default=lambda o: o.__dict__, 
+            sort_keys=True, indent=4)
+
+for path, subdirs, files in os.walk(language_path):
+    for filename in files:
+        if filename.endswith(('.md', '.rst')):
+            file = os.path.splitext(os.path.join(path, filename))[0] + '.html'
+            relative_file = file[len(language_path):]
+            original = '\\' + os.path.join('en', 'latest', 'CROWDIN', relative_file)
+            to = '\\' + os.path.join(relative_file[:2], 'latest', relative_file[3:])
+            r = Object()
+            r.from_url = original.replace('\\', '/')
+            r.to_url = to.replace('\\', '/')
+            r.type = 'exact'
+            redirects.append(r)
+
+def obj_dict(obj):
+    return obj.__dict__
+
+json_object = json.dumps(redirects, indent=4, default=obj_dict) 
+
+with open(FILE, 'w') as outfile: 
+    outfile.write(json_object) 
+
+print('Done, ' + str(len(redirects)) + ' results are stored in ' + FILE)

utils/importRedirects.py

@@ -0,0 +1,46 @@
+import requests
+import json
+import time
+import sys
+import re
+
+PROJECT = 'androidaps'
+URL = 'https://readthedocs.org/api/v3/projects/' + PROJECT + '/redirects/'
+TOKEN = len(sys.argv) == 2 and sys.argv[1]
+HEADERS = {'Authorization': f'token {TOKEN}'}
+FILE = 'redirect.json'
+
+def create(redirect, index) :
+    response = requests.post(
+            URL,
+            json=redirect,
+            headers=HEADERS,
+    ) 
+    if response.status_code == 201 :
+        print ('create redirect (' + str(index) + ') ' + redirect['from_url'])
+    elif response.status_code == 429:
+        detail = response.json()['detail']
+        wait = int(re.search(r'\d+', detail).group())
+        print('Throttled, wait for ' + str(wait + 1) + ' seconds ')
+        time.sleep(wait + 1)
+        create(redirect, index)
+    else :
+        print(response.status_code , response.json())
+
+def main():
+    try:
+        with open(FILE) as json_file:
+            redirects = json.load(json_file)
+            print('Creating ' + str(len(redirects)) + ' redirects ')
+            for index, redirect in enumerate(redirects):
+                create(redirect, index)
+        print ('done')
+    except IOError:
+        print('File ' + FILE + ' is not accessible, please make sure you run the "generateRedirect" script')
+
+
+if not TOKEN :
+    print('Please provide a API token as parameter')
+    print('useage: $ python importRedirects.py <apikey>')
+else :
+    main()

utils/readme.md

@@ -0,0 +1,48 @@
+# Redirect 
+These redirect scripts are created to facilitate language split of the projects.
+
+The standard user defined redirects are not capable to redirect into another language or project.
+https://docs.readthedocs.io/en/stable/user-defined-redirects.html
+
+We can use the public API to create the redirects.
+
+## Redirects
+
+`/en/latest/CROWDIN/<lang>/*` will be redirected `/cs/latest/*`
+
+Example:
+
+`/en/latest/CROWDIN/cs/index.html` => `/cs/latest/index.html`
+
+## Scripts
+
+### Generate Redirects
+
+Generate a list of redirects. For each page in the CROWDIN folder a redirect is generated and stored in a `redirect.json` file.
+
+``` console
+$ python generateRedirects.py
+```
+
+### Import Redirects
+
+This script loads the `redirect.json` and calls for each record the readthedocs API
+https://docs.readthedocs.io/en/stable/api/v3.html#redirects
+
+Note that the API is limited in throughput and will be throttled, and the script will take some time to complete.
+
+The API token can be generated in the GUI and should be passed as a parameter.
+https://docs.readthedocs.io/en/stable/api/v3.html#token
+
+``` console
+$ python importRedirects.py <APIKEY>
+```
+
+### Import Redirects
+
+This script removes all redirect from the project. The script could be used for testing or maintenance.
+**NOTE: this script will remove all redirects, including the ones that are manual added trough the GUI.
+
+``` console
+$ python deleteAllRedirects.py <APIKEY>
+```