From 2bcd45521b15455e190f8e741d9a65d1de1d19c8 Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Thu, 8 Aug 2019 11:33:09 -1000 Subject: [PATCH] Backport JSON info files patch While unmerged, patch it here https://github.com/openwrt/openwrt/pull/2192 Signed-off-by: Paul Spooren --- meta | 4 + ...ate-JSON-files-containing-image-info.patch | 156 ++++++++++++ ...002-build-add-JSON-info-merge-script.patch | 229 ++++++++++++++++++ scripts/activate-json-info-files.sh | 4 + 4 files changed, 393 insertions(+) create mode 100644 patches/0001-build-create-JSON-files-containing-image-info.patch create mode 100644 patches/0002-build-add-JSON-info-merge-script.patch create mode 100644 scripts/activate-json-info-files.sh diff --git a/meta b/meta index 55fbf8e..54ebe5f 100755 --- a/meta +++ b/meta @@ -204,3 +204,7 @@ make "$1" \ EXTRA_IMAGE_NAME="$EXTRA_IMAGE_NAME" \ FILES="$FILES" ) + +[ "$1" = "image" ] && (cd "$IB_DIR" && +make "jsonmergeimageinfo" OUTPUT_DIR="$ROOT_DIR/bin/$DISTRO/$VERSION" +) diff --git a/patches/0001-build-create-JSON-files-containing-image-info.patch b/patches/0001-build-create-JSON-files-containing-image-info.patch new file mode 100644 index 0000000..d119bd6 --- /dev/null +++ b/patches/0001-build-create-JSON-files-containing-image-info.patch @@ -0,0 +1,156 @@ +From f9777b0434a1854e723026347b3e06b19d8a456b Mon Sep 17 00:00:00 2001 +From: Paul Spooren +Date: Sun, 18 Aug 2019 09:56:45 -1000 +Subject: [PATCH 1/2] build: create JSON files containing image info + +The JSON info files contain details about the created firmware images +per device and are stored next to the created images. + +The JSON files are stored as "$(IMAGE_PREFIX).json" and contain some +device/image meta data as well as a list of created firmware images. + +An example of openwrt-ath79-generic-tplink_tl-wdr3600-v1.json + + { + "id": "tplink_tl-wdr3600-v1", + "image_prefix": "openwrt-ath79-generic-tplink_tl-wdr3600-v1", + "image_size": "7936k", + "images": [ + { + "name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-sysupgrade.bin", + "sha256": "60ef977447d57ffe406f1f6170860be8043654d961933b73645850b25c6a1990", + "type": "sysupgrade" + }, + { + "name": "openwrt-ath79-generic-tplink_tl-wdr3600-v1-squashfs-factory.bin", + "sha256": "c6fae436b13f512e65ef05c0ae94308dd1cc9e20fd929dd3e0422574fe58d2b5", + "type": "factory" + } + ], + "metadata_version": 1, + "model": "TL-WDR3600", + "supported_devices": [ + "tplink,tl-wdr3600-v1", + "tl-wdr4300" + ], + "target": "ath79/generic", + "title": [ + "TP-Link TL-WDR3600 v1" + ], + "variant": "v1", + "vendor": "TP-Link", + "version_commit": "r10764-84c103509a", + "version_number": "SNAPSHOT" + } + +Signed-off-by: Paul Spooren +--- + config/Config-build.in | 7 +++++ + include/image.mk | 25 ++++++++++++++++- + scripts/json_add_image_info.py | 51 ++++++++++++++++++++++++++++++++++ + 3 files changed, 82 insertions(+), 1 deletion(-) + create mode 100755 scripts/json_add_image_info.py + +diff --git a/include/image.mk b/include/image.mk +index c6a6ab7993..cfb2e2a90a 100644 +--- a/include/image.mk ++++ b/include/image.mk +@@ -554,7 +554,28 @@ define Device/Build/image + + $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2)) + cp $$^ $$@ +- ++ $(if $(CONFIG_JSON_ADD_IMAGE_INFO), \ ++ DEVICE_ID="$(DEVICE_NAME)" \ ++ TOPDIR="$(TOPDIR)" \ ++ BIN_DIR="$(BIN_DIR)" \ ++ IMAGE_NAME="$(IMAGE_NAME)" \ ++ IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \ ++ IMAGE_SIZE="$(IMAGE_SIZE)" \ ++ IMAGE_PREFIX="$(IMAGE_PREFIX)" \ ++ DEVICE_TITLE="$(DEVICE_TITLE)" \ ++ DEVICE_VENDOR="$(DEVICE_VENDOR)" \ ++ DEVICE_MODEL="$(DEVICE_MODEL)" \ ++ DEVICE_VARIANT="$(DEVICE_VARIANT)" \ ++ DEVICE_ALT0_TITLE="$(DEVICE_ALT0_TITLE)" \ ++ DEVICE_ALT1_TITLE="$(DEVICE_ALT1_TITLE)" \ ++ DEVICE_ALT2_TITLE="$(DEVICE_ALT2_TITLE)" \ ++ TARGET="$(BOARD)" \ ++ SUBTARGET="$(SUBTARGET)" \ ++ VERSION_NUMBER="$(VERSION_NUMBER)" \ ++ VERSION_CODE="$(VERSION_CODE)" \ ++ SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \ ++ $(TOPDIR)/scripts/json_add_image_info.py \ ++ ) + endef + + define Device/Build/artifact +@@ -572,6 +593,8 @@ define Device/Build/artifact + endef + + define Device/Build ++ $(if $(CONFIG_JSON_ADD_IMAGE_INFO), $(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json)) ++ + $(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1))) + $(call Device/Build/kernel,$(1)) + +diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py +new file mode 100755 +index 0000000000..31b8d1c123 +--- /dev/null ++++ b/scripts/json_add_image_info.py +@@ -0,0 +1,51 @@ ++#!/usr/bin/env python3 ++ ++import json ++import os ++import hashlib ++ ++ ++def e(variable): ++ return os.environ.get(variable) ++ ++ ++json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX")) ++ ++with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file: ++ image_hash = hashlib.sha256(image_file.read()).hexdigest() ++ ++if not os.path.exists(json_path): ++ device_info = { ++ "id": e("DEVICE_ID"), ++ "image_prefix": e("IMAGE_PREFIX"), ++ "image_size": e("IMAGE_SIZE"), ++ "images": [], ++ "metadata_version": 1, ++ "model": e("DEVICE_MODEL"), ++ "supported_devices": e("SUPPORTED_DEVICES").split(), ++ "target": "{}/{}".format(e("TARGET"), e("SUBTARGET")), ++ "title": list( ++ filter( ++ None, ++ [ ++ e("DEVICE_TITLE"), ++ e("DEVICE_ALT0_TITLE"), ++ e("DEVICE_ALT1_TITLE"), ++ e("DEVICE_ALT2_TITLE"), ++ ], ++ ) ++ ), ++ "variant": e("DEVICE_VARIANT"), ++ "vendor": e("DEVICE_VENDOR"), ++ "version_commit": e("VERSION_CODE"), ++ "version_number": e("VERSION_NUMBER"), ++ } ++else: ++ with open(json_path, "r") as json_file: ++ device_info = json.load(json_file) ++ ++image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash} ++device_info["images"].append(image_info) ++ ++with open(json_path, "w") as json_file: ++ json.dump(device_info, json_file, sort_keys=True, indent=" ") +-- +2.20.1 + diff --git a/patches/0002-build-add-JSON-info-merge-script.patch b/patches/0002-build-add-JSON-info-merge-script.patch new file mode 100644 index 0000000..d2fe5b0 --- /dev/null +++ b/patches/0002-build-add-JSON-info-merge-script.patch @@ -0,0 +1,229 @@ +From 1ea12b6d1bb067d255bec10eec6be18c73c24786 Mon Sep 17 00:00:00 2001 +From: Paul Spooren +Date: Sun, 18 Aug 2019 10:47:24 -1000 +Subject: [PATCH 2/2] build: add JSON info merge script + +Script creates three different files types with different functions: + +* map.json + +Stored in each target/subtarget folder containing a mapping between +$(SUPPORTED_DEVICES) and JSON image info files. With this file a router +can automatically obtain a sysuprade firmware image. + + { + "devices": { + "netgear,r7800": { + "info": "openwrt-ipq806x-generic-netgear_r7800.json", + "sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a" + }, + "r7800": { + "info": "openwrt-ipq806x-generic-netgear_r7800.json", + "sha256": "9d24baeba60c4f592838bf7fdf2a1f87b61d4cd1432953d2960315a37149b89a" + } + }, + "metadata_version": 1, + "target": "ipq806x/generic" + } + +This mapping style allows device identifiers to change between releases, as it +happend between ar71xx and ath79. + +Sha256sums are stored to verify file integrity, later more on that. + +This files is stored in $(OUTPUT_DIR) and contains a list of all +targets where images exist for. + +* targets.json + + { + "metadata_version": 1, + "targets": { + "ar71xx/generic": { + "path": "targets/ath79/generic/map.json", + "sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7" + }, + "ath79/generic": { + "path": "targets/ath79/generic/map.json", + "sha256": "434bb3de0e5e0a1240a714d08360667daeb11f92b36cea4fb590046392f2f3a7" + }, + "ipq806x/generic": { + "path": "targets/ipq806x/generic/map.json", + "sha256": "6afba149bc0b0f22e19ebb49e10d7e2a98e80286f84eafcd13b65384f71f401f" + }, + "x86/64": { + "path": "targets/x86/64/map.json", + "sha256": "0f445ca9807d7f21624fff802f33320b30247eedfca636237317a144729c02e1" + } + } + } + +This mapping style allows device targets to change between releases, as it +happend between ar71xx and ath79. Devices searching for ar71xx are forwarded to +compatible deivces of ath79. + +The workflow would be the following: + +* Devices request a (tbd) signed versions.json file from the server +* Parse the file on device and optain the folder containing targets.json +* Request targets.json and optain path to target specific map.json +* Request map.json and optain device specific info file +* Parse info file images for a sysupgrade +* Request sysupgrade and flash device + +The versions.json would be signed containing sha256sums of targets.json files. +This allows a chain of trust: + + versions.json -> targes.json -> map.json -> device_id.json -> sysupgrade.bin + +The versions.json file could look like the following: + + { + "versions": [ + { + "name": "19.07-SNAPSHOT", + "path": "releases/19.07-SNAPSHOTS", + "sha256": "e659ccba1cd70124138726efabd9dd7a60ee220bf8471a91082c7e05488cac19" + }, + { + "name": "Snapshot", + "path": "snapshots", + "sha256": "7c84e6140f35c70d6fe71cd1ca8e5d98d9f88a887a4550adf137d4fd2a1f0ea6" + } + ], + "metadata_version": 1 + } + +* overview.json + +Contains a mapping between device title(s) and device info files. Having +this file greatly simpifies firmware image retreiveal. A (web) client can +search trhough the mapping and show the results to an end user. + + { + "devices": { + " TL-WR840N v5": "targets/ramips/mt76x8/openwrt-ramips-mt76x8-tplink_tl-wr840n-v5.json", + "7Links PX-4885 4M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-4m.json", + "7Links PX-4885 8M": "targets/ramips/rt305x/openwrt-ramips-rt305x-7links_px-4885-8m.json", + "8devices Carambola": "targets/ramips/rt305x/openwrt-ramips-rt305x-8devices_carambola.json", + "8devices Carambola2": "targets/ath79/generic/openwrt-ath79-generic-8dev_carambola2.json", + ... + }, + "metadata_version": 1 + } + +Signed-off-by: Paul Spooren +--- + Makefile | 5 ++ + config/Config-build.in | 12 +++++ + scripts/json_merge_image_info.py | 75 ++++++++++++++++++++++++++++++ + target/imagebuilder/files/Makefile | 4 ++ + 4 files changed, 96 insertions(+) + create mode 100755 scripts/json_merge_image_info.py + +diff --git a/scripts/json_merge_image_info.py b/scripts/json_merge_image_info.py +new file mode 100755 +index 0000000000..bc61424dd6 +--- /dev/null ++++ b/scripts/json_merge_image_info.py +@@ -0,0 +1,75 @@ ++#!/usr/bin/env python3 ++ ++import json ++import glob ++import os ++import hashlib ++ ++output_dir = os.path.join(os.environ.get("OUTPUT_DIR", "./bin")) ++ ++dev_overview = {"metadata_version": 1, "devices": {}} ++targets = {"metadata_version": 1, "targets": {}} ++ ++# find all json files in ./bin/targets and create sha256 checksums ++targets_dir = os.path.join(output_dir, "targets") ++ ++for root, dirs, files in os.walk(targets_dir): ++ current_dir = root[len(targets_dir) + 1 :] ++ # check if root contains one slash aka target/subtarget ++ if current_dir.count("/") == 1: ++ targets["targets"][current_dir] = { ++ "path": "targets/{}/map.json".format(current_dir) ++ } ++ # initialize maps.json for target/subtarget ++ dev_map = {"metadata_version": 1, "target": current_dir, "devices": {}} ++ for file in files: ++ # ignore existing map.json files ++ if file.endswith(".json") and file != "map.json": ++ # load device info ++ with open(os.path.join(root, file), "r") as dev_file: ++ dev_info = json.load(dev_file) ++ ++ # generate sha256sum ++ with open(os.path.join(root, file), "rb") as dev_file_b: ++ dev_sha256 = hashlib.sha256(dev_file_b.read()).hexdigest() ++ ++ # generate map entry for each supported device ++ for supported in dev_info["supported_devices"]: ++ dev_map["devices"][supported] = {} ++ dev_map["devices"][supported]["info"] = file ++ dev_map["devices"][supported]["sha256"] = dev_sha256 ++ ++ # path from overview.json to device info files ++ dev_path = "{}/{}".format(dev_info["target"], file) ++ ++ # add title(s) to overview ++ for title in dev_info.get("title"): ++ if title in dev_overview["devices"]: ++ print( ++ "WARNING: '{}' pointing to '{}' already exists in overview and will be overwritten with '{}/{}'".format( ++ title, ++ dev_overview["devices"][title], ++ dev_info["target"], ++ file, ++ ) ++ ) ++ dev_overview["devices"][title] = dev_path ++ ++ # write map.json to target/subtarget ++ with open(os.path.join(root, "map.json"), "w") as dev_map_file: ++ json.dump(dev_map, dev_map_file, sort_keys=True, indent=" ") ++ ++for target, data in targets["targets"].items(): ++ # generate sha256sum of target/subtarget/maps.json ++ with open(os.path.join(output_dir, data["path"]), "rb") as dev_file_b: ++ targets["targets"][target]["sha256"] = hashlib.sha256( ++ dev_file_b.read() ++ ).hexdigest() ++ ++# write targets.json to ./bin/ ++with open(os.path.join(output_dir, "targets.json"), "w") as targets_file: ++ json.dump(targets, targets_file, sort_keys=True, indent=" ") ++ ++# write overview.json to ./bin/ ++with open(os.path.join(output_dir, "overview.json"), "w") as dev_overview_file: ++ json.dump(dev_overview, dev_overview_file, sort_keys=True, indent=" ") +diff --git a/Makefile b/Makefile +index 22b2731358..018abc9a23 100644 +--- a/Makefile ++++ b/Makefile +@@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build + $(MAKE) -s prepare_rootfs + $(MAKE) -s build_image + $(MAKE) -s checksum ++ $(if $(CONFIG_JSON_MERGE_IMAGE_INFO),$(MAKE) -s jsonmergeimageinfo) + + _call_manifest: FORCE + rm -rf $(TARGET_DIR) +@@ -201,6 +202,9 @@ image: + $(if $(BIN_DIR),BIN_DIR="$(BIN_DIR)") \ + $(if $(DISABLED_SERVICES),DISABLED_SERVICES="$(DISABLED_SERVICES)")) + ++jsonmergeimageinfo: FORCE ++ $(SCRIPT_DIR)/json_merge_image_info.py ++ + manifest: FORCE + $(MAKE) -s _check_profile + (unset PROFILE FILES PACKAGES MAKEFLAGS; \ +-- +2.20.1 + diff --git a/scripts/activate-json-info-files.sh b/scripts/activate-json-info-files.sh new file mode 100644 index 0000000..6a5a475 --- /dev/null +++ b/scripts/activate-json-info-files.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +chmod +x ./scripts/json_*.py +echo "CONFIG_JSON_ADD_IMAGE_INFO=y" >> .config