Backport JSON info files patch

While unmerged, patch it here openwrt/openwrt#2192 Signed-off-by: Paul Spooren <mail@aparcar.org>
aparcar · Aug 22, 2019 · 9916259 · 9916259
1 parent 20e9145
commit 9916259
Show file tree

Hide file tree

Showing 4 changed files with 393 additions and 0 deletions.
diff --git a/meta b/meta
@@ -205,3 +205,7 @@ make "$1" \
     EXTRA_IMAGE_NAME="$EXTRA_IMAGE_NAME" \
     FILES="$FILES"
 )
+
+[ "$1" = "image" ] && (cd "$IB_DIR" &&
+make "jsonmergeimageinfo" OUTPUT_DIR="$ROOT_DIR/bin/$DISTRO/$VERSION_PATH"
+)
diff --git a/patches/0001-build-create-JSON-files-containing-image-info.patch b/patches/0001-build-create-JSON-files-containing-image-info.patch
@@ -0,0 +1,156 @@
+From f9777b0434a1854e723026347b3e06b19d8a456b Mon Sep 17 00:00:00 2001
+From: Paul Spooren <mail@aparcar.org>
+Date: Sun, 18 Aug 2019 09:56:45 -1000
+Subject: [PATCH 1/2] build: create JSON files containing image info
+
+The JSON info files contain details about the created firmware images
+per device and are stored next to the created images.
+
+The JSON files are stored as "$(IMAGE_PREFIX).json" and contain some
+device/image meta data as well as a list of created firmware images.
+
+An example of openwrt-ath79-generic-tplink_tl-wdr3600-v1.json
+
+    {
+      "id": "tplink_tl-wdr3600-v1",
+      "image_prefix": "openwrt-ath79-generic-tplink_tl-wdr3600-v1",
+      "image_size": "7936k",
+      "images": [
+	{
+	  "name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-sysupgrade.bin",
+	  "sha256": "60ef977447d57ffe406f1f6170860be8043654d961933b73645850b25c6a1990",
+	  "type": "sysupgrade"
+	},
+	{
+	  "name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-factory.bin",
+	  "sha256": "c6fae436b13f512e65ef05c0ae94308dd1cc9e20fd929dd3e0422574fe58d2b5",
+	  "type": "factory"
+	}
+      ],
+      "metadata_version": 1,
+      "model": "TL-WDR3600",
+      "supported_devices": [
+	"tplink,tl-wdr3600-v1",
+	"tl-wdr4300"
+      ],
+      "target": "ath79/generic",
+      "title": [
+	"TP-Link TL-WDR3600 v1"
+      ],
+      "variant": "v1",
+      "vendor": "TP-Link",
+      "version_commit": "r10764-84c103509a",
+      "version_number": "SNAPSHOT"
+    }
+
+Signed-off-by: Paul Spooren <mail@aparcar.org>
+---
+ config/Config-build.in         |  7 +++++
+ include/image.mk               | 25 ++++++++++++++++-
+ scripts/json_add_image_info.py | 51 ++++++++++++++++++++++++++++++++++
+ 3 files changed, 82 insertions(+), 1 deletion(-)
+ create mode 100755 scripts/json_add_image_info.py
+
+diff --git a/include/image.mk b/include/image.mk
+index c6a6ab7993..cfb2e2a90a 100644
+--- a/include/image.mk
++++ b/include/image.mk
+@@ -554,7 +554,28 @@ define Device/Build/image
+
+   $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2))
+ 	cp $$^ $$@
+-
++	$(if $(CONFIG_JSON_ADD_IMAGE_INFO), \
++		DEVICE_ID="$(DEVICE_NAME)" \
++		TOPDIR="$(TOPDIR)" \
++		BIN_DIR="$(BIN_DIR)" \
++		IMAGE_NAME="$(IMAGE_NAME)" \
++		IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
++		IMAGE_SIZE="$(IMAGE_SIZE)" \
++		IMAGE_PREFIX="$(IMAGE_PREFIX)" \
++		DEVICE_TITLE="$(DEVICE_TITLE)" \
++		DEVICE_VENDOR="$(DEVICE_VENDOR)" \
++		DEVICE_MODEL="$(DEVICE_MODEL)" \
++		DEVICE_VARIANT="$(DEVICE_VARIANT)" \
++		DEVICE_ALT0_TITLE="$(DEVICE_ALT0_TITLE)" \
++		DEVICE_ALT1_TITLE="$(DEVICE_ALT1_TITLE)" \
++		DEVICE_ALT2_TITLE="$(DEVICE_ALT2_TITLE)" \
++		TARGET="$(BOARD)" \
++		SUBTARGET="$(SUBTARGET)" \
++		VERSION_NUMBER="$(VERSION_NUMBER)" \
++		VERSION_CODE="$(VERSION_CODE)" \
++		SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
++		$(TOPDIR)/scripts/json_add_image_info.py \
++	)
+ endef
+
+ define Device/Build/artifact
+@@ -572,6 +593,8 @@ define Device/Build/artifact
+ endef
+
+ define Device/Build
++  $(if $(CONFIG_JSON_ADD_IMAGE_INFO), $(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json))
++
+   $(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1)))
+   $(call Device/Build/kernel,$(1))
+
+diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py
+new file mode 100755
+index 0000000000..31b8d1c123
+--- /dev/null
++++ b/scripts/json_add_image_info.py
+@@ -0,0 +1,51 @@
++#!/usr/bin/env python3
++
++import json
++import os
++import hashlib
++
++
++def e(variable):
++    return os.environ.get(variable)
++
++
++json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX"))
++
++with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file:
++    image_hash = hashlib.sha256(image_file.read()).hexdigest()
++
++if not os.path.exists(json_path):
++    device_info = {
++        "id": e("DEVICE_ID"),
++        "image_prefix": e("IMAGE_PREFIX"),
++        "image_size": e("IMAGE_SIZE"),
++        "images": [],
++        "metadata_version": 1,
++        "model": e("DEVICE_MODEL"),
++        "supported_devices": e("SUPPORTED_DEVICES").split(),
++        "target": "{}/{}".format(e("TARGET"), e("SUBTARGET")),
++        "title": list(
++            filter(
++                None,
++                [
++                    e("DEVICE_TITLE"),
++                    e("DEVICE_ALT0_TITLE"),
++                    e("DEVICE_ALT1_TITLE"),
++                    e("DEVICE_ALT2_TITLE"),
++                ],
++            )
++        ),
++        "variant": e("DEVICE_VARIANT"),
++        "vendor": e("DEVICE_VENDOR"),
++        "version_commit": e("VERSION_CODE"),
++        "version_number": e("VERSION_NUMBER"),
++    }
++else:
++    with open(json_path, "r") as json_file:
++        device_info = json.load(json_file)
++
++image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash}
++device_info["images"].append(image_info)
++
++with open(json_path, "w") as json_file:
++    json.dump(device_info, json_file, sort_keys=True, indent="  ")
+-- 
+2.20.1
+
diff --git a/patches/0002-build-add-JSON-info-merge-script.patch b/patches/0002-build-add-JSON-info-merge-script.patch
@@ -0,0 +1,229 @@
+From 1ea12b6d1bb067d255bec10eec6be18c73c24786 Mon Sep 17 00:00:00 2001
+From: Paul Spooren <mail@aparcar.org>
+Date: Sun, 18 Aug 2019 10:47:24 -1000
+Subject: [PATCH 2/2] build: add JSON info merge script
+
+Script creates three different files types with different functions:
+
+* map.json
+
+Stored in each target/subtarget folder containing a mapping between
+$(SUPPORTED_DEVICES) and JSON image info files. With this file a router
+can automatically obtain a sysuprade firmware image.
+
+    {
+      "devices": {
+        "netgear,r7800": {
+          "info": "openwrt-ipq806x-generic-netgear_r7800.json",
+          "sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a"
+        },
+        "r7800": {
+          "info": "openwrt-ipq806x-generic-netgear_r7800.json",
+          "sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a"
+        }
+      },
+      "metadata_version": 1,
+      "target": "ipq806x/generic"
+    }
+
+This mapping style allows device identifiers to change between releases, as it
+happend between ar71xx and ath79.
+
+Sha256sums are stored to verify file integrity, later more on that.
+
+This files is stored in $(OUTPUT_DIR) and contains a list of all
+targets where images exist for.
+
+* targets.json
+
+    {
+      "metadata_version": 1,
+      "targets": {
+        "ar71xx/generic": {
+          "path": "targets/ath79/generic/map.json",
+          "sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7"
+        },
+        "ath79/generic": {
+          "path": "targets/ath79/generic/map.json",
+          "sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7"
+        },
+        "ipq806x/generic": {
+          "path": "targets/ipq806x/generic/map.json",
+          "sha256": "6afba149bc0b0f22e19ebb49e10d7e2a98e80286f84eafcd13b65384f71f401f"
+        },
+        "x86/64": {
+          "path": "targets/x86/64/map.json",
+          "sha256": "0f445ca9807d7f21624fff802f33320b30247eedfca636237317a144729c02e1"
+        }
+      }
+    }
+
+This mapping style allows device targets to change between releases, as it
+happend between ar71xx and ath79. Devices searching for ar71xx are forwarded to
+compatible deivces of ath79.
+
+The workflow would be the following:
+
+* Devices request a (tbd) signed versions.json file from the server
+* Parse the file on device and optain the folder containing targets.json
+* Request targets.json and optain path to target specific map.json
+* Request map.json and optain device specific info file
+* Parse info file images for a sysupgrade
+* Request sysupgrade and flash device
+
+The versions.json would be signed containing sha256sums of targets.json files.
+This allows a chain of trust:
+
+    versions.json -> targes.json -> map.json -> device_id.json -> sysupgrade.bin
+
+The versions.json file could look like the following:
+
+    {
+        "versions": [
+            {
+                "name": "19.07-SNAPSHOT",
+                "path": "releases/19.07-SNAPSHOTS",
+                "sha256": "e659ccba1cd70124138726efabd9dd7a60ee220bf8471a91082c7e05488cac19"
+            },
+            {
+                "name": "Snapshot",
+                "path": "snapshots",
+                "sha256": "7c84e6140f35c70d6fe71cd1ca8e5d98d9f88a887a4550adf137d4fd2a1f0ea6"
+            }
+        ],
+        "metadata_version": 1
+    }
+
+* overview.json
+
+Contains a mapping between device title(s) and device info files. Having
+this file greatly simpifies firmware image retreiveal. A (web) client can
+search trhough the mapping and show the results to an end user.
+
+    {
+      "devices": {
+        " TL-WR840N v5": "targets/ramips/mt76x8/openwrt-ramips-mt76x8-tplink_tl-wr840n-v5.json",
+        "7Links PX-4885 4M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-4m.json",
+        "7Links PX-4885 8M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-8m.json",
+        "8devices Carambola": "targets/ramips/rt305x/openwrt-ramips-rt305x-8devices_carambola.json",
+        "8devices Carambola2": "targets/ath79/generic/openwrt-ath79-generic-8dev_carambola2.json",
+        ...
+      },
+      "metadata_version": 1
+    }
+
+Signed-off-by: Paul Spooren <mail@aparcar.org>
+---
+ Makefile                           |  5 ++
+ config/Config-build.in             | 12 +++++
+ scripts/json_merge_image_info.py   | 75 ++++++++++++++++++++++++++++++
+ target/imagebuilder/files/Makefile |  4 ++
+ 4 files changed, 96 insertions(+)
+ create mode 100755 scripts/json_merge_image_info.py
+
+diff --git a/scripts/json_merge_image_info.py b/scripts/json_merge_image_info.py
+new file mode 100755
+index 0000000000..bc61424dd6
+--- /dev/null
++++ b/scripts/json_merge_image_info.py
+@@ -0,0 +1,75 @@
++#!/usr/bin/env python3
++
++import json
++import glob
++import os
++import hashlib
++
++output_dir = os.path.join(os.environ.get("OUTPUT_DIR", "./bin"))
++
++dev_overview = {"metadata_version": 1, "devices": {}}
++targets = {"metadata_version": 1, "targets": {}}
++
++# find all json files in ./bin/targets and create sha256 checksums
++targets_dir = os.path.join(output_dir, "targets")
++
++for root, dirs, files in os.walk(targets_dir):
++    current_dir = root[len(targets_dir) + 1 :]
++    # check if root contains one slash aka target/subtarget
++    if current_dir.count("/") == 1:
++        targets["targets"][current_dir] = {
++            "path": "targets/{}/map.json".format(current_dir)
++        }
++        # initialize maps.json for target/subtarget
++        dev_map = {"metadata_version": 1, "target": current_dir, "devices": {}}
++        for file in files:
++            # ignore existing map.json files
++            if file.endswith(".json") and file != "map.json":
++                # load device info
++                with open(os.path.join(root, file), "r") as dev_file:
++                    dev_info = json.load(dev_file)
++
++                # generate sha256sum
++                with open(os.path.join(root, file), "rb") as dev_file_b:
++                    dev_sha256 = hashlib.sha256(dev_file_b.read()).hexdigest()
++
++                # generate map entry for each supported device
++                for supported in dev_info["supported_devices"]:
++                    dev_map["devices"][supported] = {}
++                    dev_map["devices"][supported]["info"] = file
++                    dev_map["devices"][supported]["sha256"] = dev_sha256
++
++                # path from overview.json to device info files
++                dev_path = "{}/{}".format(dev_info["target"], file)
++
++                # add title(s) to overview
++                for title in dev_info.get("title"):
++                    if title in dev_overview["devices"]:
++                        print(
++                            "WARNING: '{}' pointing to '{}' already exists in overview and will be overwritten with '{}/{}'".format(
++                                title,
++                                dev_overview["devices"][title],
++                                dev_info["target"],
++                                file,
++                            )
++                        )
++                    dev_overview["devices"][title] = dev_path
++
++        # write map.json to target/subtarget
++        with open(os.path.join(root, "map.json"), "w") as dev_map_file:
++            json.dump(dev_map, dev_map_file, sort_keys=True, indent="  ")
++
++for target, data in targets["targets"].items():
++    # generate sha256sum of target/subtarget/maps.json
++    with open(os.path.join(output_dir, data["path"]), "rb") as dev_file_b:
++        targets["targets"][target]["sha256"] = hashlib.sha256(
++            dev_file_b.read()
++        ).hexdigest()
++
++# write targets.json to ./bin/
++with open(os.path.join(output_dir, "targets.json"), "w") as targets_file:
++    json.dump(targets, targets_file, sort_keys=True, indent="  ")
++
++# write overview.json to ./bin/
++with open(os.path.join(output_dir, "overview.json"), "w") as dev_overview_file:
++    json.dump(dev_overview, dev_overview_file, sort_keys=True, indent="  ")
+diff --git a/Makefile b/Makefile
+index 22b2731358..018abc9a23 100644
+--- a/Makefile
++++ b/Makefile
+@@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build
+ 	$(MAKE) -s prepare_rootfs
+ 	$(MAKE) -s build_image
+ 	$(MAKE) -s checksum
++	$(if $(CONFIG_JSON_MERGE_IMAGE_INFO),$(MAKE) -s jsonmergeimageinfo)
+
+ _call_manifest: FORCE
+ 	rm -rf $(TARGET_DIR)
+@@ -201,6 +202,9 @@ image:
+ 		$(if $(BIN_DIR),BIN_DIR="$(BIN_DIR)") \
+ 		$(if $(DISABLED_SERVICES),DISABLED_SERVICES="$(DISABLED_SERVICES)"))
+
++jsonmergeimageinfo: FORCE
++	$(SCRIPT_DIR)/json_merge_image_info.py
++
+ manifest: FORCE
+ 	$(MAKE) -s _check_profile
+ 	(unset PROFILE FILES PACKAGES MAKEFLAGS; \
+-- 
+2.20.1
+
diff --git a/scripts/activate-json-info-files.sh b/scripts/activate-json-info-files.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+chmod +x ./scripts/json_*.py
+echo "CONFIG_JSON_ADD_IMAGE_INFO=y" >> .config