scripts: add fetch releases.py to automatically generate/update remot…

…es.json with new releases

scripts/fetch_releases.py

@@ -0,0 +1,187 @@
+import requests
+import json
+import re
+import optparse
+
+IGNORE_VERSIONS_BEFORE = '4.3'
+
+def version_number_and_type(git_hash, ap_source_subdir):
+    url = "https://raw.githubusercontent.com/ArduPilot/ardupilot/%s/%s/version.h" % (git_hash, ap_source_subdir)
+    response = requests.get(url=url)
+
+    if response.status_code != 200:
+        print(response.text)
+        print(url)
+        raise Exception("Couldn't fetch version.h from github server. Got status code %d" % response.status_code)
+
+    exp = re.compile('#define FIRMWARE_VERSION (\d+),(\d+),(\d+),FIRMWARE_VERSION_TYPE_(\w+)')
+    matches = re.findall(exp, response.text)
+    major, minor, patch, fw_type = matches[0]
+    fw_type = fw_type.lower()
+
+    if fw_type == 'official':
+        fw_type = 'stable'  # to avoid any confusion, beta is also 'official' ;-)
+    return f'{major}.{minor}.{patch}', fw_type
+
+
+def fetch_tags_from_github():
+    url = 'https://api.github.com/repos/ardupilot/ardupilot/git/refs/tags'
+    headers = {'X-GitHub-Api-Version': '2022-11-28', 'Accept': 'application/vnd.github+json'}
+    response = requests.get(url=url, headers=headers)
+    if response.status_code != 200:
+        print(response.text)
+        print(url)
+        raise Exception("Couldn't fetch tags from github server. Got status code %d" % response.status_code)
+
+    tags_objs = json.loads(response.text)
+    return tags_objs
+
+def remove_duplicate_entries(releases):
+    temp = dict()
+    for release in releases:
+        # if we have already seen a version with similar hash 
+        # and we now see a beta release with same hash, we skip it
+        if temp.get(release['commit_reference']) and release['release_type'] == 'beta':
+            continue
+
+        temp[release['commit_reference']] = release
+
+    return list(temp.values())
+
+def construct_vehicle_versions_list(vehicle, ap_source_subdir, fw_server_vehicle_subdir, tag_filter_exps, tags):
+    ret = []
+    for tag_info in tags:
+        tag = tag_info['ref'].removeprefix('refs/tags/')
+
+        matches = []
+        for exp in tag_filter_exps:
+            # the regexes capture two groups
+            # first group is the matched substring itself
+            # second group is the tag body (e.g., beta, stable, 4.5.1 etc)
+            matches.extend(re.findall(re.compile(exp), tag))
+
+        if matches:
+            matched_string, tag_body = matches[0]
+
+            if len(matched_string) < len(tag):
+                print("Partial match. Ignoring. Matched '%s' in '%s'." % (matched_string, tag))
+                continue
+
+            try:
+                v_num, v_type = version_number_and_type(tag_info['object']['sha'], ap_source_subdir)
+            except Exception as e:
+                print('Cannot determine version number for tag %s' % tag)
+                print(e)
+                continue
+
+            if v_num < IGNORE_VERSIONS_BEFORE:
+                print("Version too old. Ignoring.")
+                continue
+
+            if re.search('\d+\.\d+\.\d+', tag_body):
+                # we do stable version tags in this format
+                # e.g. Rover-4.5.1, Copter-4.5.1, where Rover and Copter are prefixes and 4.5.1 is the tag body
+                # artifacts for these versions are stored in firmware server at stable-x.y.z subdirs
+                afcts_url = f'https://firmware.ardupilot.org/{fw_server_vehicle_subdir}/stable-{tag_body}'
+            else:
+                afcts_url = f'https://firmware.ardupilot.org/{fw_server_vehicle_subdir}/{tag_body}'
+
+            ret.append({
+                'release_type': v_type,
+                'version_number': v_num,
+                'ap_build_atrifacts_url': afcts_url,
+                'commit_reference': tag_info['object']['sha']
+            })
+
+    ret = remove_duplicate_entries(ret)
+
+    # entry for master
+    ret.append({
+        'release_type': 'latest',
+        'version_number': 'NA',
+        'ap_build_atrifacts_url': f'https://firmware.ardupilot.org/{fw_server_vehicle_subdir}/latest',
+        'commit_reference': 'master'
+    })
+
+    return {
+        'name': vehicle,
+        'releases': ret
+    }
+
+
+parser = optparse.OptionParser("fetch_releases.py")
+parser.add_option("", "--out", type="string",
+                  default='./remotes.json',
+                  help="Output file path")
+
+parser.add_option("", "--remotename", type="string",
+                  default='ardupilot',
+                  help="Remote name to write in json file")
+
+cmd_opts, cmd_args = parser.parse_args()
+
+tags = fetch_tags_from_github()
+vehicles = []
+
+vehicles.append(construct_vehicle_versions_list("Copter",
+                                                "ArduCopter",
+                                                "Copter",
+                                                ["(ArduCopter-(beta-4.3|beta|stable))", "(Copter-(\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Plane",
+                                                "ArduPlane",
+                                                "Plane",
+                                                ["(ArduPlane-(beta-4.3|beta|stable))", "(Plane-(\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Rover",
+                                                "Rover",
+                                                "Rover",
+                                                ["(APMrover2-(beta-4.3|beta|stable))", "(Rover-(\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Sub",
+                                                "ArduSub",
+                                                "Sub",
+                                                ["(ArduSub-(beta-4.3|beta|stable))", "(Sub-(\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Tracker",
+                                                "AntennaTracker",
+                                                "AntennaTracker",
+                                                ["(AntennaTracker-(beta-4.3|beta|stable))", "(Tracker-(\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Blimp",
+                                                "Blimp",
+                                                "Blimp",
+                                                ["(Blimp-(beta-4.3|beta|stable|\d+\.\d+\.\d+))"],
+                                                tags))
+
+vehicles.append(construct_vehicle_versions_list("Heli",
+                                                "ArduCopter",
+                                                "Copter",
+                                                ["(ArduCopter-(beta-4.3|beta|stable)-heli)"],
+                                                tags))
+
+remotes_json = {
+    "name": cmd_opts.remotename,
+    "url": "https://github.com/ardupilot/ardupilot.git",
+    "vehicles": vehicles
+}
+
+try:
+    with open(cmd_opts.out, 'r') as f:
+        remotes = json.loads(f.read())
+
+    # remove existing ardupilot remote entry from the list
+    remotes = [remote for remote in remotes if remote['name'] != 'ardupilot']
+except Exception as e:
+    print(e)
+    print("Writing to empty file")
+    remotes = []
+
+with open(cmd_opts.out, 'w') as f:
+    remotes.append(remotes_json)
+    f.write(json.dumps(remotes, indent=2))