YAML Model for Semantic Conventions (#571)

* First model draft

* Fix checkstyle

* Generate network table from YAML

* HTTP semantic convention tables

* Fix requirements order

* Add code and basic documentation

* Missing \n

* Remove tool

* Add General and HTTP semantic convention

* Update table style

* yaml->semantic_conventions

* Fix small errors

* Fix docfx errors

* Update Makefile to use otel Docker repo

Co-authored-by: Armin Ruech <armin.ruech@dynatrace.com>

.github/workflows/semantic-conventions-check.yaml

@@ -0,0 +1,18 @@
+name: Semantic Conventions Check
+on:
+  push:
+    tags: [ '**' ]
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    paths:
+      - .github/workflows/semantic-convention-check.yml
+      - '**/semantic_conventions/**'
+
+jobs:
+  test-markdown:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Verify that semantic convention tables are up-to-date
+      run: docker run --rm -v $(pwd)/semantic_conventions:/source -v $(pwd)/specification:/spec otel/semconvgen -f /source markdown -md /spec --md-check

Makefile

@@ -1,11 +1,13 @@
 # All documents to be used in spell check.
 ALL_DOCS := $(shell find . -name '*.md' -not -path './.github/*' -type f | grep -v ^./node_modules | sort)
+PWD := $(shell pwd)
 
 TOOLS_DIR := ./.tools
 MISSPELL_BINARY=$(TOOLS_DIR)/misspell
 MARKDOWN_LINK_CHECK=markdown-link-check
 MARKDOWN_LINT=markdownlint
 
+
 .PHONY: install-misspell
 install-misspell:
 	go build -o $(MISSPELL_BINARY) github.com/client9/misspell/cmd/misspell
@@ -34,3 +36,7 @@ install-markdown-lint:
 markdown-lint:
 	@echo $(ALL_DOCS)
 	@for f in $(ALL_DOCS); do echo $$f; $(MARKDOWN_LINT) -c .markdownlint.yaml $$f || exit 1;	done
+
+.PHONY: table-generation
+table-generation:
+	docker run --rm -v $(PWD)/semantic_conventions:/source -v $(PWD)/specification:/spec otel/semconvgen -f /source markdown -md /spec

semantic_conventions/syntax.md

@@ -0,0 +1,228 @@
+# Semantic Convention YAML Language
+
+First, the syntax with a pseudo [EBNF](https://en.wikipedia.org/wiki/Extended_Backus-Naur_form) grammar is presented.
+Then, the semantic of each field is described.
+
+## Syntax
+
+All attributes are lower case.
+
+```bnf
+groups ::= semconv
+       | semconv groups
+
+semconv ::= id brief [note] [prefix] [extends] [span_kind] attributes [constraints]
+
+id    ::= string
+brief ::= string
+note  ::= string
+
+prefix ::= string
+
+# extends MUST point to an existing semconv id
+extends ::= string
+
+span_kind ::= "client"
+          |   "server"
+          |   "producer"
+          |   "consumer"
+          |   "internal"
+
+attributes ::= (id type brief examples | ref [brief] [examples]) [required] [note]
+
+# ref MUST point to an existing attribute id
+ref ::= id
+
+type ::= "string"
+     |   "number"
+     |   "boolean"
+     |   "string[]"
+     |   "number[]"
+     |   "boolean[]"
+     |   enum
+
+enum ::= [allow_custom_values] members
+
+allow_custom_values := boolean
+
+members ::= member {member}
+
+member ::= id value [brief] [note]
+
+required ::= "always"
+         |   "conditional" <condition>
+
+examples ::= <example_value> {<example_value>}
+
+constraints ::= constraint {constraint}
+
+constraint ::= any_of
+           |   include
+
+any_of ::= id {id}
+
+include ::= id
+
+```
+
+## Semantics
+
+### Groups
+
+Groups contain the list of semantic conventions and it is the root node of each yaml file.
+
+### Semantic Convention
+
+The field `semconv` represents a semantic convention and it is made by:
+
+- `id`, string that uniquely identifies the semantic convention.
+- `brief`, string, a brief description of the semantic convention.
+- `note`, optional string, a more elaborate description of the semantic convention.
+    It defaults to an empty string.
+- `prefix`, optional string, prefix for the attributes for this semantic convention.
+    It defaults to an empty string.
+- `extends`, optional string, reference another semantic convention `id`.
+    It inherits the prefix, constraints, and all attributes defined in the specified semantic convention.
+- `span_kind`, optional enum, specifies the kind of the span.
+- `attributes`, list of attributes that belong to the semantic convention.
+- `constraints`, optional list, additional constraints (See later). It defaults to an empty list.
+
+### Attributes
+
+An attribute is defined by:
+
+- `id`, string that uniquely identifies the attribute.
+- `type`, either a string literal denoting the type or an enum definition (See later).
+   The accepted strings literals are:
+  * "string": String attributes.
+  * "number": Numeric attributes.
+  * "boolean": Boolean attributes.
+  * "string[]": Array of strings attributes.
+  * "number[]": Array of numbers attributes.
+  * "boolean[]": Array of booleans attributes.
+- `ref`, optional string, reference an existing attribute, see later.
+- `required`, optional, specifies if the attribute is mandatory.
+    Can be "always", or "conditional". When omitted, the attribute is not required.
+    When set to "conditional",the string provided as `<condition>` MUST specify
+    the conditions under which the attribute is required.
+- `brief`, string, brief description of the attribute.
+- `note`, optional string, additional notes to the attribute. It defaults to an empty string.
+- `examples`, sequence/dictionary of example values for the attribute.
+   They are optional for boolean and enum attributes.
+   Example values must be of the same type of the attribute.
+   If only a single example is provided, it can directly be reported without encapsulating it into a sequence/dictionary.
+
+Examples for setting the `examples` field:
+
+A single example value for a string attribute. All the following three representations are equivalent:
+
+```yaml
+examples: 'this is a single string'
+```
+
+or
+
+```yaml
+examples: ['this is a single string']
+```
+
+or
+
+```yaml
+examples:
+   - 'this is a single string'
+```
+
+Attention, the following will throw a type mismatch error because a string type as example value is expected and not an array of string:
+
+```yaml
+examples:
+   - ['this is an error']
+
+examples: [['this is an error']]
+```
+
+Multiple example values for a string attribute:
+
+```yaml
+examples: ['this is a single string', 'this is another one']
+```
+
+or
+
+```yaml
+examples:
+   - 'this is a single string'
+   - 'this is another one'
+```
+
+A single example value for an array of strings attribute:
+
+```yaml
+examples: ['first element of first array', 'second element of first array']
+```
+
+or
+
+```yaml
+examples:
+   - ['first element of first array', 'second element of first array']
+```
+
+Attention, the following will throw a type mismatch error because an array of strings as type for the example values is expected and not a string:
+
+```yaml
+examples: 'this is an error'
+```
+
+Multiple example values for an array of string attribute:
+
+```yaml
+examples: [ ['first element of first array', 'second element of first array'], ['first element of second array', 'second element of second array'] ]
+```
+
+or
+
+```yaml
+examples:
+   - ['first element of first array', 'second element of first array']
+   - ['first element of second array', 'second element of second array']
+```
+
+### Ref
+
+`ref` MUST have an id of an existing attribute. When it is set, `id` and `type` MUST not be present.
+`ref` is useful for specifying that an existing attribute of another semantic convention is part of
+the current semantic convention and inherit its `brief`, `note`, and `example` values. However, if these
+fields are present in the current attribute definition, they override the inherited values.
+
+### Type
+
+An attribute type can either be a string, number, boolean, array of strings, array of numbers,
+array of booleans, or an enumeration. If it is an enumeration, additional fields are required:
+
+- `allow_custom_values`, optional boolean, set to false to not accept values
+     other than the specified members. It defaults to `true`.
+- `members`, list of enum entries.
+
+An enum entry has the following fields:
+
+- `id`, string that uniquely identifies the enum entry.
+- `value`, string, number, or boolean, value of the enum entry.
+- `brief`, optional string, brief description of the enum entry value. It defaults to the value of `id`.
+- `note`, optional string, longer description. It defaults to an empty string.
+
+### Constraints
+
+Allow to define additional requirements on the semantic convention.
+Currently, it supports `any_of` and `include`.
+
+#### Any Of
+
+`any_of` accepts a list of sequences. Each sequence contains a list of attribute ids that are required.
+`any_of` enforces that all attributes of at least one of the sequences are set.
+
+#### Include
+
+`include` accepts a semantic conventions `id`. It includes as part of this semantic convention all constraints
+and required attributes that are not already defined in the current semantic convention.

semantic_conventions/trace/general.yaml

@@ -0,0 +1,97 @@
+groups:
+  - id: network
+    prefix: net
+    brief: >
+        These attributes may be used for any network related operation.
+    attributes:
+      - id: transport
+        type:
+          allow_custom_values: false
+          members:
+            - id: IP.TCP
+              value: "IP.TCP"
+            - id: IP.UDP
+              value: "IP.UDP"
+            - id: IP
+              value: "IP"
+              brief: 'Another IP-based protocol'
+            - id: Unix
+              value: "Unix"
+              brief: 'Unix Domain socket. See below.'
+            - id: pipe
+              value: "pipe"
+              brief: 'Named or anonymous pipe. See note below.'
+            - id: inproc
+              value: "inproc"
+              brief: 'In-process communication.'
+              note: >
+                Signals that there is only in-process communication not using a "real" network protocol
+                in cases where network attributes would normally be expected. Usually all other network
+                attributes can be left out in that case.
+            - id: other
+              value: "other"
+              brief: 'Something else (non IP-based).'
+        brief: >
+          Transport protocol used. See note below.
+        examples: 'IP.TCP'
+      - id: peer.ip
+        type: string
+        brief: >
+          Remote address of the peer (dotted decimal for IPv4 or
+          [RFC5952](https://tools.ietf.org/html/rfc5952) for IPv6)
+        examples: '127.0.0.1'
+      - id: peer.port
+        type: number
+        brief: 'Remote port number.'
+        examples: [80, 8080, 443]
+      - id: peer.name
+        type: string
+        brief: 'Remote hostname or similar, see note below.'
+        examples: 'example.com'
+      - id: host.ip
+        type: string
+        brief: 'Like `net.peer.ip` but for the host IP. Useful in case of a multi-IP host.'
+        examples: '192.168.0.1'
+      - id: host.port
+        type: number
+        brief: 'Like `net.peer.port` but for the host port.'
+        examples: 35555
+      - id: host.name
+        type: string
+        brief: 'Local hostname or similar, see note below.'
+        examples: 'localhost'
+  - id: peer
+    prefix: peer
+    brief: "Operations that access some remote service."
+    attributes:
+      - id: service
+        type: string
+        brief: >
+          The [`service.name`](../../resource/semantic_conventions/README.md#service)
+          of the remote service. SHOULD be equal to the actual `service.name`
+          resource attribute of the remote service if any.
+        examples: "AuthTokenCache"
+  - id: identity
+    prefix: enduser
+    brief: >
+        These attributes may be used for any operation with an authenticated and/or authorized enduser.
+    attributes:
+      - id: id
+        type: string
+        brief: >
+          Username or client_id extracted from the access token or
+          [Authorization](https://tools.ietf.org/html/rfc7235#section-4.2)
+          header in the inbound request from outside the system.
+        examples: 'username'
+      - id: role
+        type: string
+        brief: 'Actual/assumed role the client is making the request under extracted from token or application security context.'
+        examples: 'admin'
+      - id: scope
+        type: string
+        brief: >
+          Scopes or granted authorities the client currently possesses extracted from token
+          or application security context. The value would come from the scope associated
+          with an [OAuth 2.0 Access Token](https://tools.ietf.org/html/rfc6749#section-3.3)
+          or an attribute value in a [SAML 2.0 Assertion](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html).
+        examples: 'read:message, write:files'