Parameters files versioning enabler

.gitignore

@@ -35,4 +35,10 @@
 /copter/source/docs/parameters.rst
 /plane/source/docs/parameters.rst
 /rover/source/docs/parameters.rst
-/antennatracker/source/docs/parameters.rst
+/antennatracker/source/docs/parameters.rst
+
+# Versioned parameters files
+/copter/source/docs/parameters-Copter*.rst
+/plane/source/docs/parameters-Plane*.rst
+/rover/source/docs/parameters-Rover*.rst
+/antennatracker/source/docs/parameters-AntennaTracker*.rst

antennatracker/source/conf.py

@@ -15,6 +15,9 @@
 import sys
 import os
 
+# Parameter versioning need 3000+ ; Set to 5000 to make room for next versions.
+sys.setrecursionlimit(5000)
+
 #Import common configuration information as "common_conf"
 sys.path.insert(0,'../..')
 import common_conf

build_parameters.py

@@ -318,14 +318,15 @@ def replace_anchors(source_file, dest_file, version_tag):
         version = str(commits_to_checkout_and_parse[i][1])
         commit_id = str(commits_to_checkout_and_parse[i][2])
 
+        # Not elegant workaround:
         # These versions present errors when parsing using param_parser.py. Needs more investigation?
         if  ( 
             "3.2.1" in version or # last stable APM Copte
             "3.4.0" in version or # last stable APM Plane 
-            "3.4.6" in version or # Copter.
-            "2.42" in version or # last stable APM Rover?
-            "2.51" in version or # last beta APM Rover?
-            "0.7.2" in version  # Antennatracker
+            "3.4.6" in version or # Copter
+            "2.42" in version or  # last stable APM Rover?
+            "2.51" in version or  # last beta APM Rover?
+            "0.7.2" in version    # Antennatracker
             ):
             debug("Ignoring APM version:\t" + vehicle + "\t" + version)
             continue
@@ -444,6 +445,8 @@ def print_versions(commits_to_checkout_and_parse):
     debug("")
 
 
+
+
 #################################
 
 # Step 1 - Select the versions for generate parameters

copter/source/_static/parameters_versioning_script.inc

@@ -4,7 +4,7 @@
 <select class="selectpicker" onChange="window.document.location.href=this.options[this.selectedIndex].value;"></select>
 
 <script type="text/javascript">
-jQuery.getJSON("parameters-Copter.json", {}, function(json) {
+jQuery.getJSON("../_static/parameters-Copter.json", {}, function(json) {
     $.each(json, function(key, value) {
         $('.selectpicker').append('<option value="' + json[key] + '">' + key + '</option>');
     });

copter/source/conf.py

@@ -15,6 +15,9 @@
 import sys
 import os
 
+# Parameter versioning needs 3000+ ; Set to 5000 to make room for next versions.
+sys.setrecursionlimit(5000)
+
 #Import common configuration information as "common_conf"
 sys.path.insert(0,'../..')
 import common_conf

plane/source/_static/parameters_versioning_script.inc

@@ -4,7 +4,7 @@
 <select class="selectpicker" onChange="window.document.location.href=this.options[this.selectedIndex].value;"></select>
 
 <script type="text/javascript">
-jQuery.getJSON("parameters-Planer.json", {}, function(json) {
+jQuery.getJSON("../_static/parameters-Plane.json", {}, function(json) {
     $.each(json, function(key, value) {
         $('.selectpicker').append('<option value="' + json[key] + '">' + key + '</option>');
     });

plane/source/conf.py

@@ -15,6 +15,9 @@
 import sys
 import os
 
+# Parameter versioning needs 3000+ ; Set to 5000 to make room for next versions.
+sys.setrecursionlimit(5000)
+
 #Import common configuration information as "common_conf"
 sys.path.insert(0,'../..')
 import common_conf

rover/source/_static/parameters_versioning_script.inc

@@ -4,7 +4,7 @@
 <select class="selectpicker" onChange="window.document.location.href=this.options[this.selectedIndex].value;"></select>
 
 <script type="text/javascript">
-jQuery.getJSON("parameters-Rover.json", {}, function(json) {
+jQuery.getJSON("../_static/parameters-Rover.json", {}, function(json) {
     $.each(json, function(key, value) {
         $('.selectpicker').append('<option value="' + json[key] + '">' + key + '</option>');
     });

rover/source/conf.py

@@ -15,6 +15,9 @@
 import sys
 import os
 
+# Parameter versioning needs 3000+ ; Set to 5000 to make room for next versions.
+sys.setrecursionlimit(5000)
+
 #Import common configuration information as "common_conf"
 sys.path.insert(0,'../..')
 import common_conf

update.py

@@ -32,6 +32,7 @@
 import subprocess
 import shutil
 import glob
+import filecmp
 
 
 DEFAULT_COPY_WIKIS =['copter', 'plane', 'rover']
@@ -48,11 +49,25 @@
 parser.add_argument('--cached-parameter-files', default=False, help="Do not re-download parameter files", type=bool)
 parser.add_argument('--destdir', default="/var/sites/wiki/web", help="Destination directory for compiled docs")
 parser.add_argument('--paramversioning', default=False, help="Build multiple parameters pages for each vehicle based on its firmware repo.", type=bool)
+parser.add_argument('--verbose', dest='verbose', action='store_false', default=True, help="show debugging output")
 args = parser.parse_args()
 #print(args.site)
 #print(args.clean)
 
 PARAMETER_SITE={'rover':'APMrover2', 'copter':'ArduCopter','plane':'ArduPlane','antennatracker':'AntennaTracker' }
+error_count = 0
+
+def debug(str_to_print):
+    """Debug output if verbose is set."""
+    if args.verbose:
+        print(str_to_print)
+
+
+def error(str_to_print):
+    """Show and count the errors."""
+    global error_count
+    error_count += 1
+    print(str_to_print)
 
 
 def fetchparameters(site=args.site):
@@ -85,8 +100,6 @@ def fetchparameters(site=args.site):
             subprocess.check_call(["mv", 'Parameters.rst', targetfile])
 
 
-
-
 def sphinx_make(site):
     """
     Calls 'make html' to build each site
@@ -102,7 +115,6 @@ def sphinx_make(site):
         subprocess.check_call(["make", "-C", wiki ,"html"])
 
 
-
 def copy_build(site):
     """
     Copies each site into the target location
@@ -155,6 +167,7 @@ def copy_build(site):
             #print("no delete of olddir")
             pass
 
+
 def generate_copy_dict(start_dir=COMMON_DIR):
     """
     This creates a dict which indexes copy targets for all common docs.
@@ -261,7 +274,6 @@ def fix_site_shortcode(matchobj):
     return newText
 
 
-
 def logmatch_code(matchobj, prefix):
 
     try:
@@ -308,50 +320,167 @@ def logmatch_code(matchobj, prefix):
 
 
 def fetch_versioned_parameters(site=args.site):
+    """
+    It relies on "build_parameters.py" be executed before the "update.py"     
+    
+    Once the generated files are on ../new_params_mversion it tut all parameters and JSON files in their destinations.
+    
+    """
 
     for key, value in PARAMETER_SITE.items():
-
-        # Clean cached parameters.rst otherwise it makes build warning regarding the anchors.
-        target_param_file='./%s/source/docs/parameters.rst' % key
-        target_json_file='./%s/source/_static/parameters-%s.json' % (key,value)
 
         if site==key or site==None:
-
-            #Remove target files if it exists
+            # Remove old param single file
+            single_param_file='./%s/source/docs/parameters.rst' % key
+            debug("Erasing " + single_param_file)
             try: 
-                subprocess.check_call(["rm", target_param_file])
-            except:
+                subprocess.check_call(["rm", single_param_file])
+            except Exception as e:
+                error(e)
+                pass
+
+            # Remove old versioned param files
+            if key is 'antennatracker': # To main the original script approach instead of the build_parameters.py approach.
+                old_parameters_mask = os.getcwd() + '/%s/source/docs/parameters-%s-' % (key,value)
+            else:
+                old_parameters_mask = os.getcwd() + '/%s/source/docs/parameters-%s-' % (key,key.title()) 
+            try:  
+                old_parameters_files = [f for f in glob.glob(old_parameters_mask + "*.rst")]
+                for filename in old_parameters_files:
+                    debug("Erasing " + filename)
+                    os.remove(filename)    
+            except Exception as e:
+                error(e)
                 pass
-            # try: 
-            #     print("apagando: " + target_json_file)
-            #     subprocess.check_call(["rm", target_json_file])
-            # except:
-            #     pass
 
-            # Moves all parameter files to vehicle folder
+            # Remove old json file
+            if key is 'antennatracker': # To main the original script approach instead of the build_parameters.py approach.
+                target_json_file='./%s/source/_static/parameters-%s.json' % (key,value)
+            else:
+                target_json_file='./%s/source/_static/parameters-%s.json' % (value,key.title())
+            debug("Erasing " + target_json_file)
             try:
-                new_parameters_folder = os.getcwd() + '/../new_params_mversion/%s/' % value
-                files = [f for f in glob.glob(new_parameters_folder + "*.rst")]
+                subprocess.check_call(["rm", target_json_file])
+            except Exception as e:
+                error(e)
+                pass
 
-                for filename in files:
-                    new_file = key + "/source/docs/" + filename[str(filename).rfind("/")+1:]
-                    os.rename(filename, new_file)
+            # Moves the updated JSON file  
+            if key is 'antennatracker':  # To main the original script approach instead of the build_parameters.py approach. 
+                vehicle_json_file = os.getcwd() + '/../new_params_mversion/%s/parameters-%s.json' % (key,value)
+            else: 
+                vehicle_json_file = os.getcwd() + '/../new_params_mversion/%s/parameters-%s.json' % (value,key.title())   
+            new_file = key + "/source/_static/" + vehicle_json_file[str(vehicle_json_file).rfind("/")+1:] 
+            try:  
+                debug("Moving " + vehicle_json_file)
+                #os.rename(vehicle_json_file, new_file)
+                shutil.copy2(vehicle_json_file, new_file)
+            except Exception as e:
+                error(e)
+                pass
 
-            except:
+            # Copy all parameter files to vehicle folder IFF it is new
+            try:
+                new_parameters_folder = os.getcwd() + '/../new_params_mversion/%s/' % value
+                new_parameters_files = [f for f in glob.glob(new_parameters_folder + "*.rst")]
+            except Exception as e:
+                error(e)
                 pass
+            for filename in new_parameters_files:
+                # Check possible cached version
+                try:
+                    new_file = key + "/source/docs/" + filename[str(filename).rfind("/")+1:]
+                    if os.path.isfile(filename.replace("new_params_mversion","old_params_mversion")):                   # The cached file exists?
+                        if ("latest" in filename) or (not filecmp.cmp(filename, filename.replace("new_params_mversion","old_params_mversion"))):    # It is different?  OR is this one the latest. | Latest file must be built everytime in order to enable Sphinx create the correct references across the wiki.
+                            debug("Overwriting " + new_file)                               
+                            shutil.copy2(filename, new_file)
+                            create_latest_parameter_redirect(filename[str(filename).rfind("/")+1:], key)               # Piggyback this moment to create a redirect file called "parameters.rst" that pointer to the latest parameter file.  
+                        else:
+                            debug("Ignoring " + new_file)                               
+                    else:                                                                                               # If not cached, build it anyway.
+                        debug("Creating " + new_file)
+                        shutil.copy2(filename, new_file)
+                        create_latest_parameter_redirect(filename[str(filename).rfind("/")+1:], key)
+
+                except Exception as e:
+                    error(e)
+                    pass
+
+
+def create_latest_parameter_redirect(default_param_file, vehicle):
+    """
+    For a given vehicle create a file called parameters.rst that redirects to the latest parameters file.(Create to maintaim retro compatibility.)
+    
+    """
+    out_line = "\n.. raw:: html\n\n"
+    out_line += "   <script>location.replace(\"" + default_param_file[:-3] + "html" + "\")</script>"
+    out_line += "\n\n"
+
+    filename = vehicle + "/source/docs/parameters.rst"
+    with open(filename, "w") as text_file:
+        text_file.write(out_line)
 
-            # Moves JSON file  
+
+def cache_parameters_files(site=args.site):
+    """
+    For each vechile: put new_params_mversion/ content in old_params_mversion/ folders and .html built files as well.
+    
+    """
+    for key, value in PARAMETER_SITE.items():
+        if site==key or site==None:
             try:
-                vehicle_json_file = os.getcwd() + '/../new_params_mversion/%s/parameters-%s.json' % (value,value)
-                new_file = key + "/source/_static/" + vehicle_json_file[str(vehicle_json_file).rfind("/")+1:]
-                os.rename(vehicle_json_file, new_file)
+                debug("Cleaning old_params_mversion/%s/" % value)
+                old_parameters_folder = os.getcwd() + '/../old_params_mversion/%s/' % value
+                old_parameters_files = [f for f in glob.glob(old_parameters_folder + "*.*")]
+                for file in old_parameters_files:
+                    os.remove(file)
 
+                debug("Caching files from new_params_mversion/%s/ " % value)
+                new_parameters_folder = os.getcwd() + '/../new_params_mversion/%s/' % value
+                new_parameters_files = [f for f in glob.glob(new_parameters_folder + "*.rst")]
+                for filename in new_parameters_files:
+                    shutil.copy2(filename, old_parameters_folder)
+
+                debug("Caching built HTML files from /%s/build/html/docs/" % value)
+                built_folder = os.getcwd() + "/" + key + "/build/html/docs/" 
+                built_parameters_files = [f for f in glob.glob(built_folder + "parameters-*.html")]
+                for built in built_parameters_files:
+                    shutil.copy2(built, old_parameters_folder)
+
+            except Exception as e:
+                error(e)
+                pass
+
+
+def put_cached_parameters_files_in_sites(site=args.site):
+    """
+    For each vechile: put built .html files in site folder
+    
+    """
+    for key, value in PARAMETER_SITE.items():
+        if site==key or site==None:
+            try:
+                built_folder = os.getcwd() + '/../old_params_mversion/%s/' % value
+                built_parameters_files = [f for f in glob.glob(built_folder + "parameters-*.html")]
+                vehicle_folder = os.getcwd() + "/" + key + "/build/html/docs/" 
+                debug("Getting previously built files from " + built_folder)
+                for built in built_parameters_files:
+                    if ("latest" not in built):  # latest parameters files must be built every time
+                        debug("Reusing " + built)
+                        shutil.copy(built, vehicle_folder)
             except:
-                pass 
+                pass
+
+
+
+
 
 
 ###############################################################################################################
 
+
+
+
 if args.paramversioning:                
     fetch_versioned_parameters(args.site)   # Parameters for all versions availble on firmware.ardupilot.org    
 else:
@@ -361,5 +490,10 @@ def fetch_versioned_parameters(site=args.site):
 sphinx_make(args.site)
 copy_build(args.site)
 
+if args.paramversioning:                
+    put_cached_parameters_files_in_sites(args.site)
+    cache_parameters_files(args.site)
+
+
 
-# to test final results on the PR locally: run chrome with --allow-file-access-from-files
+# To navigate locally and use the versioning script for parameters is necessary run Chrome as "chrome --allow-file-access-from-files".