Proper Tool Metadata Abstraction

agent/tool-scripts/meta.json

@@ -0,0 +1,48 @@
+{
+	"transient":{
+		"blktrace": null,
+		"bpftrace": null,
+		"cpuacct": null,
+		"disk": null,
+		"dm-cache": null,
+		"docker": null,
+		"docker-info": null,
+		"external-data-source": null,
+		"haproxy-ocp": null,
+		"iostat": null,
+		"jmap": null,
+		"jstack": null,
+		"kvm-spinlock": null,
+		"kvmstat": null,
+		"kvmtrace": null,
+		"lockstat": null,
+		"mpstat": null,
+		"numastat": null,
+		"oc": null,
+		"openvswitch": null,
+		"pcp": null,
+		"perf": null,
+		"pidstat": null,
+		"pprof": null,
+		"proc-interrupts": null,
+		"proc-sched_debug": null,
+		"proc-vmstat": null,
+		"prometheus-metrics": null,
+		"qemu-migrate": null,
+		"rabbit": null,
+		"sar": null,
+		"strace": null,
+		"sysfs": null,
+		"systemtap": null,
+		"tcpdump": null,
+		"turbostat": null,
+		"user-tool": null,
+		"virsh-migrate": null,
+		"vmstat": null
+	},
+
+	"persistent":{
+		"node-exporter": {"collector": "prometheus", "port": "9100"},
+		"dcgm": {"collector": "prometheus", "port": "8000"}
+	}
+}

agent/util-scripts/gold/pbench-register-tool/test-44.txt

@@ -19,7 +19,6 @@ Available tools:
 	blktrace
 	bpftrace
 	cpuacct
-	dcgm
 	disk
 	dm-cache
 	docker
@@ -34,7 +33,6 @@ Available tools:
 	kvmtrace
 	lockstat
 	mpstat
-	node-exporter
 	numastat
 	oc
 	openvswitch
@@ -57,6 +55,8 @@ Available tools:
 	user-tool
 	virsh-migrate
 	vmstat
+	node-exporter
+	dcgm
 
 For a list of tool specific options, run:
 	/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/<tool-name> --help

agent/util-scripts/gold/pbench-register-tool/test-46.txt

@@ -19,7 +19,6 @@ Available tools:
 	blktrace
 	bpftrace
 	cpuacct
-	dcgm
 	disk
 	dm-cache
 	docker
@@ -34,7 +33,6 @@ Available tools:
 	kvmtrace
 	lockstat
 	mpstat
-	node-exporter
 	numastat
 	oc
 	openvswitch
@@ -57,6 +55,8 @@ Available tools:
 	user-tool
 	virsh-migrate
 	vmstat
+	node-exporter
+	dcgm
 
 For a list of tool specific options, run:
 	/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/<tool-name> --help

agent/util-scripts/gold/pbench-register-tool/test-47.txt

@@ -19,7 +19,6 @@ Available tools:
 	blktrace
 	bpftrace
 	cpuacct
-	dcgm
 	disk
 	dm-cache
 	docker
@@ -34,7 +33,6 @@ Available tools:
 	kvmtrace
 	lockstat
 	mpstat
-	node-exporter
 	numastat
 	oc
 	openvswitch
@@ -57,6 +55,8 @@ Available tools:
 	user-tool
 	virsh-migrate
 	vmstat
+	node-exporter
+	dcgm
 
 For a list of tool specific options, run:
 	/var/tmp/pbench-test-utils/opt/pbench-agent/tool-scripts/<tool-name> --help

agent/util-scripts/pbench-register-tool

@@ -116,9 +116,7 @@ function usage() {
 	printf -- "\tdenoted by a leading hash, or pound (\"#\"), character.\n"
 	printf -- "\nAvailable tools:\n"
 	local tool=""
-	for tool in $(find ${pbench_bin}/tool-scripts -maxdepth 1 ! -type d ! -name '*README*' ! -name base-tool ! -name unittests -printf "%P\n" 2> /dev/null | sort); do
-		printf -- "\t${tool}\n"
-	done
+	python3 -c "import sys, json; meta = json.load(open(sys.argv[1])); [print(f'\t{tool}') for tool in (*meta['transient'].keys(), *meta['persistent'].keys()) ]" ${pbench_bin}/tool-scripts/meta.json
 	#                     1         2         3         4         5         6         7         8
 	# (no tab)   12345678901234567890123456789012345678901234567890123456789012345678901234567890
 	printf -- "\nFor a list of tool specific options, run:\n"

agent/util-scripts/pbench-tool-meister-start

@@ -32,6 +32,8 @@ import redis
 
 from pbench.agent.tool_data_sink import main as tds_main
 from pbench.agent.tool_meister import main as tm_main
+from pbench.agent import PbenchAgentConfig
+from pbench.agent.modules import metaclass
 
 
 # Port number is "One Tool" in hex 0x17001
@@ -369,6 +371,28 @@ def main(argv):
         ), f"bad channel: {resp!r}"
         assert resp["data"] == 1, f"bad data: {resp!r}"
 
+    # 2.5. Add tool metadata json to redis
+    try:
+        inst_dir = PbenchAgentConfig(os.environ["_PBENCH_AGENT_CONFIG"]).pbench_install_dir
+    except BadConfig as exc:
+        logger.error("%s", exc)
+        return 1
+    except Exception:
+        logger.error("Unexpected error encountered logging pbench agent configuration: '%s'", exc)
+        return 1
+
+    try:
+        tm_start_path = Path(inst_dir).resolve(strict=True)
+    except FileNotFoundError:
+        logger.error("Unable to determine proper installation directory, '%s' not found", inst_dir)
+        return 1
+    except Exception as exc:
+        logger.exception("Unexpected error encountered resolving installation directory: '%s'", exc)
+        return 1
+    tool_metadata = metaclass.ToolMetadata("json", tm_start_path, logger)
+    tool_metadata.loadIntoRedis(redis_server)
+
+
     # 3. Start the tool-data-sink process
     #   - leave a PID file for the tool data sink process
     tds_param_key = "tds-{}".format(group)

lib/pbench/agent/modules/metaclass.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+import json
+import os
+
+
+class ToolMetadataExc(Exception):
+    pass
+
+
+class ToolMetadata:
+    def __init__(self, mode, context, logger):
+        self.logger = logger
+        assert mode in (
+            "redis",
+            "json",
+        ), f"Logic bomb! Unexpected mode, {mode}, encountered constructing tool meta data"
+        assert (
+            context
+        ), "Logic bomb! No context given on ToolMetadata object construction"
+        self.mode = mode
+        if mode == "redis":
+            self.redis_server = context
+            self.json_file = None
+        else:
+            self.redis_server = None
+            json_path = Path(context, "tool-scripts", "meta.json")
+            try:
+                self.json = json_path.resolve(strict=True)
+            except FileNotFoundError:
+                raise ToolMetadataExc(f"missing {json_path}")
+            except Exception:
+                raise
+        self.data = self.__getInitialData()
+
+    def __getInitialData(self):
+        if self.mode == "json":
+            if not os.path.isfile(self.json):
+                self.logger.error(
+                    "There is no tool-scripts/meta.json in given install dir"
+                )
+                return None
+            with self.json.open("r") as json_file:
+                metadata = json.load(json_file)
+        elif self.mode == "redis":
+            try:
+                meta_raw = self.redis_server.get("tool-metadata")
+            except Exception:
+                self.logger.exception(
+                    "Failure to fetch tool metadata from the Redis server"
+                )
+                raise
+            else:
+                if meta_raw is None:
+                    self.logger.error("Metadata has not been loaded into redis yet")
+                    return None
+            try:
+                metadata = json.loads(meta_raw.decode("utf-8"))
+            except Exception as exc:
+                self.logger.error(
+                    "Bad metadata loaded into Redis server, '%s', json=%r",
+                    exc,
+                    meta_raw,
+                )
+                return None
+        return metadata
+
+    def __dataCheck(self):
+        if not self.data:
+            self.data == self.__getInitialData()
+            if not self.data:
+                self.logger.error(f"Unable to access data through {self.mode}")
+                return 0
+        return 1
+
+    def getFullData(self):
+        if self.__dataCheck():
+            return self.data
+        return None
+
+    def getPersistentTools(self):
+        if self.__dataCheck():
+            return list(self.data["persistent"].keys())
+        return None
+
+    def getTransientTools(self):
+        if self.__dataCheck():
+            return list(self.data["transient"].keys())
+        return None
+
+    def getProperties(self, tool):
+        if tool in self.data["persistent"].keys():
+            return self.data["persistent"][tool]
+        elif tool in self.data["transient"].keys():
+            return self.data["transient"][tool]
+
+    def loadIntoRedis(self, info):
+        if self.mode == "redis":
+            try:
+                self.json = Path(info).resolve(strict=True)
+            except FileNotFoundError:
+                raise ToolMetadataExc(f"missing {info}")
+            except Exception:
+                raise
+        elif self.mode == "json":
+            self.redis_server = info
+
+        try:
+            with self.json.open("r") as json_file:
+                metadata = json.load(json_file)
+                self.redis_server.set("tool-metadata", json.dumps(metadata))
+        except Exception:
+            self.logger.error("Failed to load the data into redis")
+            raise
+        return None