Add JSON schema

Events 1. Raw events: - fs.duration - fs.error - fs.read - fs.seek - fs.stat - fs.write - net.duration - net.error - net.port - net.rx - net.other - net.tcp - net.tx - net.udp - notice (rateLimitMessage) - stderr - stdout 2. Standard events: - dns.req - dns.resp - fs.close - fs.open - http.req - http.resp - net.close - net.open Added new target `make doc-generation` - fix JSON schema for http metrics with summary close criblio#725.
michalbiesek · Feb 7, 2022 · 9104b95 · 9104b95
1 parent 4a5ea07
commit 9104b95
Show file tree

Hide file tree

Showing 66 changed files with 6,361 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -177,7 +177,22 @@ image: require-qemu-binfmt
 		--platform linux/$(PLATFORM_$(ARCH)) \
 		--file docker/base/Dockerfile \
 		--load \
-                .
+		.
+
+docs-generate: TAG := cribl/scope:docs-$(ARCH)
+docs-generate: require-docker-buildx-builder
+	@echo Building the AppScope Document generator
+	@docker buildx build \
+		--tag $(TAG) \
+		--platform linux/$(PLATFORM_$(ARCH)) \
+		--file docker/docs/Dockerfile \
+		.
+	@echo Running the AppScope Document generator
+	@docker run \
+		-v $(shell pwd)/docs:/md \
+		-u $(shell id -u):$(shell id -g) \
+		--rm cribl/scope:docs-$(ARCH) 
+	@echo AppScope Document generator finished: md files are available in docs/md_files
 
 k8s-test: require-kind require-kubectl image
 	docker tag cribl/scope:dev-x86_64 cribl/scope:$(VERSION)
@@ -228,6 +243,7 @@ require-kubectl:
 .PHONY: all test clean
 .PHONY: cli% scope
 .PHONY: docker-build docker-run 
-.PHONY: build-arch builder-arch 
+.PHONY: build-arch builder-arch
+.PHONY: docs-generate
 .PHONY: image
 .PHONY: require-docker-buildx-builder require-docker require-docker-buildx require-qemu-binfmt require-kind require-kubectl
diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile
@@ -0,0 +1,21 @@
+FROM node
+
+RUN npm install -g json-dereference-cli json-schema-gendoc
+
+USER node
+WORKDIR /home/node/
+
+ENV MD_DIR_NAME="md_files"
+ENV MD_OUTPUT_PATH="/md"
+ENV TMP_DIR_NAME="temp"
+
+COPY docker/docs/entrypoint.sh /home/node/entrypoint.sh
+
+RUN mkdir -p /home/node/schemas/
+
+COPY docs/schemas /home/node/schemas
+
+RUN mkdir /home/node/schemas/temp
+RUN mkdir /home/node/schemas/md_files
+
+ENTRYPOINT ["/home/node//entrypoint.sh"]
diff --git a/docker/docs/entrypoint.sh b/docker/docs/entrypoint.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+echo "Starting resolving references in JSON Schema"
+
+cd schemas
+
+for schema_file in *.schema.json; do
+    [ -f "$schema_file" ] || break
+    json-dereference -s "$schema_file" -o $TMP_DIR_NAME/"$schema_file"
+done
+
+echo "Starting generating MD files from JSON Schema"
+
+for resolve_schema_file in $TMP_DIR_NAME/*.schema.json; do
+    [ -f "$resolve_schema_file" ] || break
+    json-schema-gendoc $resolve_schema_file > $MD_DIR_NAME/${resolve_schema_file##*/}.md
+done
+
+chmod 744 $MD_DIR_NAME
+cp -r $MD_DIR_NAME $MD_OUTPUT_PATH
+
+#Uncomment below to generate resolve JSON Schema
+
+# chmod 744 $TMP_DIR_NAME
+# cp -r $TMP_DIR_NAME $MD_OUTPUT_PATH
+
+echo "Generation MD files from JSON Schema has finished"
diff --git a/docs/README.md b/docs/README.md
@@ -10,5 +10,7 @@ Once you're past those, take a look at these.
 - [RELEASE](./RELEASE.md) - Release Process
 - [BUILD](./BUILD.md) - Build System
 - [INTEGRATION](../test/integration/README.md) - Integration Testing
+- [SCHEMA](./schemas/README.md) - JSON Schema documentation
+
 
 > We will be adding more docs over time.
diff --git a/docs/schemas/README.md b/docs/schemas/README.md
@@ -0,0 +1,182 @@
+# JSON Schema
+
+This directory contains JSON schema for each metrics and events generated by the AppScope.
+
+## Table of contents
+* [Working with the JSON schema](#working-with-the-JSON-schema)
+  * [Event](#event)
+  * [Metric](#metric)
+* [Document generation](#doc-generation)
+
+## Working with the JSON schema
+
+JSON Schema is a vocabulary that allows to annotate and validate JSON documents.
+More information can be founded [here](https://json-schema.org/) here.
+In Appscope we use it to describe the metrics and events generated by the AppScope. 
+
+### Event
+
+To add new event `foo.bar` please create file `event_foo_bar.schema.json`
+
+- `root` section required update of `"$id"` and `"description"` fields
+- `body` section required update value of following `$ref` in properties in `"sourcetype` and `"source"`
+- `data` section required update of expected `properties` which will be in `data` section of event, in
+  template below - example `properties` are presented as `property_name_1` and `property_name_2`. `required`
+  section should be updated accordingly.
+
+```
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://appscope.dev/docs/schemas/event_foo_bar.schema.json",
+  "type": "object",
+  "title": "AppScope Event",
+  "description": "A structure described AppScope `foo.bar` event message",
+  "required": [
+    "type",
+    "id",
+    "_channel",
+    "body"
+  ],
+  "properties": {
+    "type": {
+      "$ref": "definitions/metadata.schema.json#/$defs/event_type"
+    },
+    "id": {
+      "$ref": "definitions/metadata.schema.json#/$defs/id"
+    },
+    "_channel": {
+      "$ref": "definitions/metadata.schema.json#/$defs/_channel"
+    },
+    "body": {
+      "title": "body",
+      "description": "body",
+      "type": "object",
+      "required": [
+        "sourcetype",
+        "_time",
+        "source",
+        "host",
+        "proc",
+        "cmd",
+        "pid",
+        "data"
+      ],
+      "properties": {
+        "sourcetype": {
+          "$ref": "definitions/body.schema.json#/$defs/sourcetypefootype"
+        },
+        "_time": {
+          "$ref": "definitions/body.schema.json#/$defs/_time"
+        },
+        "source": {
+          "$ref": "definitions/body.schema.json#/$defs/sourcefoobar"
+        },
+        "host": {
+          "$ref": "definitions/data.schema.json#/$defs/host"
+        },
+        "proc": {
+          "$ref": "definitions/data.schema.json#/$defs/proc"
+        },
+        "cmd": {
+          "$ref": "definitions/body.schema.json#/$defs/cmd"
+        },
+        "pid": {
+          "$ref": "definitions/data.schema.json#/$defs/pid"
+        },
+        "data": {
+          "title": "data",
+          "description": "data",
+          "type": "object",
+          "required": [
+            "property_name_1",
+            "property_name_2"
+          ],
+          "properties": {
+            "property_name_1": {
+              "$ref": "definitions/data.schema.json#/$defs/property_name_1"
+            },
+            "property_name_2": {
+              "$ref": "definitions/data.schema.json#/$defs/property_name_2"
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}
+
+```
+
+### Metric
+
+To add new metric `foo.bar` please create file `metric_foo_bar.schema.json`
+
+- `root` section required update of `"$id"` and `"description"` fields
+- `body` section required update value of following `$ref` in properties in `"_metric` and `"_metric_type"`
+- `body` section required update of expected `properties` which will be in `body` section of metrics, in
+  template below - example `properties` is presented as `property_name_1`. `required`
+  section should be updated accordingly.
+
+```
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://appscope.dev/docs/schemas/metric_foo_bar.schema.json",
+  "type": "object",
+  "title": "AppScope Metric",
+  "description": "A structure described AppScope `foo.bar` metric message",
+  "required": [
+    "type",
+    "body"
+  ],
+  "properties": {
+    "type": {
+      "$ref": "definitions/metadata.schema.json#/$defs/metric_type"
+    },
+    "body": {
+      "title": "body",
+      "description": "body",
+      "type": "object",
+      "required": [
+        "_metric",
+        "_metric_type",
+        "_value",
+        "property_name_1",
+        "_time"
+      ],
+      "properties": {
+        "_metric": {
+          "$ref": "definitions/body.schema.json#/$defs/sourcefoobar"
+        },
+        "_metric_type": {
+          "$ref": "definitions/body.schema.json#/$defs/metric_type_foo"
+        },
+        "_value": {
+          "$ref": "definitions/body.schema.json#/$defs/_value"
+        },
+        "property_name_1": {
+          "$ref": "definitions/body.schema.json#/$defs/property_name_1"
+        },
+        "_time": {
+          "$ref": "definitions/body.schema.json#/$defs/_time"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}
+
+```
+
+## Document generation
+
+Call
+
+```
+make docs-generate
+```
+
+The markdown files will be generated in `docs/md_files`