Tools: fixed manifest generation

Tools/scripts/check_firmware_version.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+'''
+check firmware-version.txt in binaries directory
+'''
+
+import os, shutil, subprocess
+
+VEHICLES = ['AntennaTracker', 'Copter', 'Plane', 'Rover', 'Sub']
+
+def parse_git_version(gfile):
+    '''parse git-version.txt, producing a firmware-version.txt'''
+    gv = open(gfile).readlines()
+    vline = gv[-1]
+    if not vline.startswith("APMVERSION:"):
+        print("Bad version %s in %s" % (vline, gfile))
+        return None
+    vline = vline[11:]
+    a = vline.split('V')
+    if len(a) != 2:
+        return None
+    vers = a[1].strip()
+    if vers[-1].isdigit():
+        return vers+"-FIRMWARE_VERSION_TYPE_OFFICIAL"
+    print("Bad vers %s in %s" % (vers, gfile))
+    return None
+
+def check_fw_version(version):
+    try:
+        (version_numbers, release_type) = version.split("-")
+        (major, minor, patch) = version_numbers.split(".")
+    except Exception:
+        return False
+    return True
+
+def check_version(vehicle):
+    '''check firmware-version.txt version for a vehicle'''
+    for d in os.listdir(vehicle):
+        if not d.startswith("stable"):
+            continue
+        stable_dir = '%s/%s' % (vehicle, d)
+        for b in sorted(os.listdir(stable_dir)):
+            if not os.path.isdir(os.path.join(stable_dir, b)):
+                continue
+            vfile = os.path.join(stable_dir, b, "firmware-version.txt")
+            if os.path.exists(vfile):
+                v = open(vfile).read()
+                if check_fw_version(v):
+                    continue
+            gfile = os.path.join(stable_dir, b, "git-version.txt")
+            if not os.path.exists(gfile):
+                print("Missing %s" % gfile)
+                continue
+            v = parse_git_version(gfile)
+            if v is not None:
+                open(vfile, "w").write(v)
+                print("Added %s" % vfile)
+                continue
+            print("Failed for %s" % gfile)
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='check_firmware_version.py')
+
+    args = parser.parse_args()
+
+    for v in VEHICLES:
+        check_version(v)

Tools/scripts/generate_manifest.py

@@ -208,26 +208,41 @@ def add_firmware_data_from_dir(self,
             print("Error listing '%s'" % dir)
             return
         for platformdir in dlist:
+            if platformdir.startswith("."):
+                continue
             some_dir = os.path.join(dir, platformdir)
+            if not os.path.isdir(some_dir):
+                continue
+            git_version_txt = os.path.join(some_dir, "git-version.txt")
+            if not os.path.exists(git_version_txt):
+                print("No file %s" % git_version_txt, file=sys.stderr)
+                continue
             try:
-                git_sha = self.git_sha_from_git_version(
-                    os.path.join(some_dir, "git-version.txt"))
-            except Exception:
+                git_sha = self.git_sha_from_git_version(git_version_txt)
+            except Exception as ex:
+                print("Failed to parse %s" % git_version_txt, ex, file=sys.stderr)
                 continue
+
+            # we require a firmware-version.txt. These files have been added to
+            # old builds that didn't have them
             firmware_version_file = os.path.join(some_dir,
                                                  "firmware-version.txt")
+            if not os.path.exists(firmware_version_file):
+                print("Missing %s" % firmware_version_file, file=sys.stderr)
+                continue
+
             try:
                 firmware_version = open(firmware_version_file).read()
                 firmware_version = firmware_version.strip()
                 (version_numbers, release_type) = firmware_version.split("-")
             except ValueError:
-                # print("malformed firmware-version.txt at (%s)" %
-                #     (firmware_version_file,), file=sys.stderr)
-                firmware_version = None
-            except Exception:
+                print("malformed firmware-version.txt at (%s)" % (firmware_version_file,), file=sys.stderr)
+                continue
+            except Exception as ex:
+                print("bad file %s" % firmware_version_file, file=sys.stderr)
                 # this exception is swallowed.... the current archive
                 # is incomplete.
-                firmware_version = None
+                continue
 
             m = platform_frame_regex.match(platformdir)
             if m is not None:
@@ -243,11 +258,7 @@ def add_firmware_data_from_dir(self,
                 platform = platformdir  # e.g. apm2
 
             for file in os.listdir(some_dir):
-                if file == "git-version.txt":
-                    continue
-                if file == "firmware-version.txt":
-                    continue
-                if file == "files.html":
+                if file in ["git-version.txt", "firmware-version.txt", "files.html"]:
                     continue
                 if file.startswith("."):
                     continue
@@ -265,23 +276,10 @@ def add_firmware_data_from_dir(self,
 
                 firmware_format = "".join(file.split(".")[-1:])
 
-                if vehicletype not in firmware_data:
-                    firmware_data[vehicletype] = dict()
-                if file_platform not in firmware_data[vehicletype]:
-                    firmware_data[vehicletype][file_platform] = dict()
-                if git_sha not in firmware_data[vehicletype][file_platform]:
-                    firmware_data[vehicletype][file_platform][git_sha] = dict()
-
-                sha_dict = firmware_data[vehicletype][file_platform][git_sha]
-                if firmware_format not in sha_dict:
-                    sha_dict[firmware_format] = dict()
-                if frame not in sha_dict[firmware_format]:
-                    sha_dict[firmware_format][frame] = Firmware()
-
-                firmware = sha_dict[firmware_format][frame]
+                firmware = Firmware()
 
                 # translate from supplied "release type" into both a
-                # "latest" flag andan actual release type.  Also sort
+                # "latest" flag and an actual release type.  Also sort
                 # out which filepath we should use:
                 firmware["latest"] = 0
                 if releasetype == "dev":
@@ -307,17 +305,7 @@ def add_firmware_data_from_dir(self,
                 firmware["format"] = firmware_format
                 firmware["firmware-version"] = firmware_version
 
-    def xfirmwares_to_firmwares(self, xfirmwares):
-        '''takes hash structure of firmwares, returns list of them'''
-        if isinstance(xfirmwares, dict):
-            ret = []
-            for value in xfirmwares.values():
-                o = self.xfirmwares_to_firmwares(value)
-                for oo in o:
-                    ret.append(oo)
-            return ret
-        else:
-            return [xfirmwares]
+                firmware_data.append(firmware)
 
     def valid_release_type(self, tag):
         '''check for valid release type'''
@@ -336,7 +324,7 @@ def walk_directory(self, basedir):
         structure representing releases in that structure'''
         year_month_regex = re.compile("(?P<year>\d{4})-(?P<month>\d{2})")
 
-        xfirmwares = dict()
+        firmwares = []
 
         # used to listdir basedir here, but since this is also a web
         # document root, there's a lot of other stuff accumulated...
@@ -358,35 +346,22 @@ def walk_directory(self, basedir):
                     # this is a dated directory e.g. binaries/Copter/2016-02
                     # we do not include dated directories in the manifest ATM:
                     continue
-                    year_month_path = os.path.join(basedir,
-                                                   vehicletype,
-                                                   firstlevel)
-                    for fulldate in os.listdir(year_month_path):
-                        if fulldate in ["files.html", ".makehtml"]:
-                            # generated file which should be ignored
-                            continue
-                        self.add_firmware_data_from_dir(
-                            os.path.join(year_month_path, fulldate),
-                            xfirmwares,
-                            vehicletype)
-                else:
-                    # assume this is a release directory such as
-                    # "beta", or the "latest" directory (treated as a
-                    # release and handled specially later)
-                    tag = firstlevel
-                    if not self.valid_release_type(tag):
-                        print("Unknown tag (%s) in directory (%s)" %
-                              (tag, os.path.join(vdir)), file=sys.stderr)
-                        continue
-                    tag_path = os.path.join(basedir, vehicletype, tag)
-                    if not os.path.isdir(tag_path):
-                        continue
-                    self.add_firmware_data_from_dir(tag_path,
-                                                    xfirmwares,
-                                                    vehicletype,
-                                                    releasetype=tag)
-
-        firmwares = self.xfirmwares_to_firmwares(xfirmwares)
+
+                # assume this is a release directory such as
+                # "beta", or the "latest" directory (treated as a
+                # release and handled specially later)
+                tag = firstlevel
+                if not self.valid_release_type(tag):
+                    print("Unknown tag (%s) in directory (%s)" %
+                          (tag, os.path.join(vdir)), file=sys.stderr)
+                    continue
+                tag_path = os.path.join(basedir, vehicletype, tag)
+                if not os.path.isdir(tag_path):
+                    continue
+                self.add_firmware_data_from_dir(tag_path,
+                                                firmwares,
+                                                vehicletype,
+                                                releasetype=tag)
 
         # convert from ardupilot-naming conventions to common JSON format:
         firmware_json = []
@@ -408,8 +383,12 @@ def walk_directory(self, basedir):
                 "format": firmware["format"],
             })
             if firmware["firmware-version"]:
-                (major, minor, patch, release_type) = self.parse_fw_version(
-                    firmware["firmware-version"])
+                try:
+                    (major, minor, patch, release_type) = self.parse_fw_version(
+                        firmware["firmware-version"])
+                except Exception:
+                    print("Badly formed firmware-version.txt %s" % firmware["firmware-version"], file=sys.stderr)
+                    continue
                 some_json["mav-firmware-version"] = ".".join([major,
                                                               minor,
                                                               patch])
@@ -419,6 +398,7 @@ def walk_directory(self, basedir):
 
             self.add_USB_IDs(some_json)
 
+            #print(some_json['url'])
             firmware_json.append(some_json)
 
         ret = {