From b45f14c13693bf9647ea83048b768e788603a6c7 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 17 Mar 2021 10:54:53 -0400 Subject: [PATCH 01/13] being policy design doc --- docs/POLICIES.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 docs/POLICIES.md diff --git a/docs/POLICIES.md b/docs/POLICIES.md new file mode 100644 index 000000000..ea49ac2f1 --- /dev/null +++ b/docs/POLICIES.md @@ -0,0 +1,86 @@ +# Policies + +**_Draft_** + +pktvisord metric collection is policy driven. Policies act as a grouping of an input stream to associated stream +handlers. + +## Property Attributes + +* Policies have exactly one Input source +* Policies have an unlimited number of Stream Handlers +* Policies contain configuration data, e.g. + * Time window length (number of periods) + * Maximum deep sample rate + +## Policy Semantics + +* The REST API path schema enforces the hierarchy of Policy -> Input => Handlers + +``` +/api/v1/policy/:POLICY_NAME:/:INPUT_MODULE_TYPE:/handler/:HANDLER_MODULE_TYPE:/:HANDLER_NAME:/ +``` + +``` +/api/v1/policy/anycast/pcap/handler/dns/default/bucket/0 +/api/v1/policy/anycast/pcap/handler/net/default/bucket/0 +/api/v1/policy/anycast/pcap/handler/dns/special_domain/bucket/0 + +/api/v1/policy/management/pcap/handler/dns/default/bucket/0 +/api/v1/policy/management/pcap/handler/net/default/bucket/0 +``` + +## Policy Schema + +* Policy schema is represented by YAML +* The schema may represent policy information not directly used by pktvisor, such as where to send output +* The schema is versioned + +## Example (YAML) + +```yaml + +version: "1.0" + +policy: + input: + name: anycast + type: pcap + config: + bpf: "host 192.168.0.50" + iface: eth0 + handler: + default_config: + periods: 5 + max_deep_sample: 50 + modules: + default_net: + type: net + default_dns: + type: dns + config: + max_deep_sample: 75 + special_domain: + type: dns + config: + qname_match: .mydomain.com + sinks: + default_prometheus: + type: prometheus_exporter + address: 0.0.0.0:9598 + default_namespace: service + my_s3: + type: aws_s3 + bucket: my-bucket + compression: gzip + region: us-east-1 +``` + + + + + + + + + From 483141019b74f131da3d4543929eb48364b2ce95 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 17 Mar 2021 14:23:09 -0400 Subject: [PATCH 02/13] design doc work --- docs/CONTROL_PLANE.md | 21 +++++++++++++++++++++ docs/POLICIES.md | 24 +++++++++++++++++------- 2 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 docs/CONTROL_PLANE.md diff --git a/docs/CONTROL_PLANE.md b/docs/CONTROL_PLANE.md new file mode 100644 index 000000000..d589e68fc --- /dev/null +++ b/docs/CONTROL_PLANE.md @@ -0,0 +1,21 @@ +# Control Plane + +**_Draft_** + +pktvisord expose a control plane over REST API. + +## Discovery + +pktvisord exposes a method for discovering the available modules, their configurable properties, and their associated +metrics schema. + +All interfaces are versioned. + +``` +/api/v1/handlers + {dns: [v1, v2], net} +/api/v1/handlers/dns/v1/schema +/api/v1/handlers/net/v1/schema + +``` + diff --git a/docs/POLICIES.md b/docs/POLICIES.md index ea49ac2f1..08ffffa4f 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -39,23 +39,29 @@ handlers. ## Example (YAML) ```yaml - version: "1.0" policy: input: - name: anycast - type: pcap + anycast: + type: pcap + config: + bpf: "host 192.168.0.50" + iface: eth0 + handlers: config: - bpf: "host 192.168.0.50" - iface: eth0 - handler: - default_config: periods: 5 max_deep_sample: 50 modules: default_net: type: net + udp_traffic: + type: net + config: + protocols: [ udp ] + metrics: + enable: + - top_ips default_dns: type: dns config: @@ -64,6 +70,10 @@ policy: type: dns config: qname_match: .mydomain.com + metrics: + disable: + - top_qtypes + - top_udp_ports sinks: default_prometheus: type: prometheus_exporter From 412890b49985ef438aae1cf9bc8ee06b4f39b137 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 17 Mar 2021 16:42:31 -0400 Subject: [PATCH 03/13] design doc work --- docs/CONTROL_PLANE.md | 77 +++++++++++++++++++++++++++++++++++++++---- docs/POLICIES.md | 34 +++++++++++++------ 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/docs/CONTROL_PLANE.md b/docs/CONTROL_PLANE.md index d589e68fc..7dd7f052b 100644 --- a/docs/CONTROL_PLANE.md +++ b/docs/CONTROL_PLANE.md @@ -2,20 +2,85 @@ **_Draft_** -pktvisord expose a control plane over REST API. +pktvisord exposes a control plane over REST API. ## Discovery pktvisord exposes a method for discovering the available modules, their configurable properties, and their associated metrics schema. -All interfaces are versioned. +All interfaces and schemas are versioned. ``` +/api/v1/inputs + { + pcap: "1.0" + } +/api/v1/inputs/pcap/interface + { + version: "1.0", + config: { + iface: { + type: "string", + description: "the ethernet interface to capture on" + } + } + filters: { + bpf: { + type: "string", + description: "tcpdump compatible bpf filter expression" + } + }, + metric_groups: { + } + } /api/v1/handlers - {dns: [v1, v2], net} -/api/v1/handlers/dns/v1/schema -/api/v1/handlers/net/v1/schema - + { dns: { version: "1.0" }, + net: { version: "1.0" } } +/api/v1/handlers/dns/interface + { + version: "1.0", + config: { + periods: { + type: "int", + description: "number of metric periods to keep" + } + } + filters: { + qname_suffix: { + type: "string", + description: "match the DNS qname sufix given", + regex: "..." + } + }, + metric_groups: { + top_error_qnames: { + description: "top N qnames with error result codes", + metrics: {, + top_refused: { + "type": "top_n", + "description": "..." + }, + top_srvfail: { + "type": "top_n", + "description": "..." + }, + top_nxdomain: { + "type": "top_n", + "description": "..." + }, + } + }, + transactions: { + description: "information on query/reply pairs", + metrics: { + ... + } + } + } + } +/api/v1/handlers/net/interface + { + } ``` diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 08ffffa4f..2d2fde04c 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -2,16 +2,15 @@ **_Draft_** -pktvisord metric collection is policy driven. Policies act as a grouping of an input stream to associated stream -handlers. +pktvisord observability configuration is policy driven. Policies act as a grouping of an input stream to stream +handlers, and their associated configuration. ## Property Attributes * Policies have exactly one Input source * Policies have an unlimited number of Stream Handlers -* Policies contain configuration data, e.g. - * Time window length (number of periods) - * Maximum deep sample rate +* Policies have an unlimited number of Sinks (which is not used directly by pktvisord; see Orb) +* Policies contain configuration data, including observability filters and metric collection configuration. ## Policy Semantics @@ -21,13 +20,15 @@ handlers. /api/v1/policy/:POLICY_NAME:/:INPUT_MODULE_TYPE:/handler/:HANDLER_MODULE_TYPE:/:HANDLER_NAME:/ ``` +Examples: + ``` -/api/v1/policy/anycast/pcap/handler/dns/default/bucket/0 -/api/v1/policy/anycast/pcap/handler/net/default/bucket/0 +/api/v1/policy/anycast/pcap/handler/dns/default_dns/bucket/0 +/api/v1/policy/anycast/pcap/handler/net/default_net/bucket/0 /api/v1/policy/anycast/pcap/handler/dns/special_domain/bucket/0 -/api/v1/policy/management/pcap/handler/dns/default/bucket/0 -/api/v1/policy/management/pcap/handler/net/default/bucket/0 +/api/v1/policy/management/pcap/handler/dns/dns_collection/bucket/0 +/api/v1/policy/management/pcap/handler/net/net_collection/bucket/0 ``` ## Policy Schema @@ -42,17 +43,26 @@ handlers. version: "1.0" policy: + # input stream + # only one is supported currently input: anycast: + # must match the input stream module name from pktvisor type: pcap + # specify that the input module requires >= specific version to be successfully applied + require_version: "1.0" config: + # must match the available configuration options from this version of the inputs stream bpf: "host 192.168.0.50" iface: eth0 + # stream handlers handlers: + # default configuration for the stream handlers config: periods: 5 max_deep_sample: 50 modules: + # the keys at this level are unique identifiers default_net: type: net udp_traffic: @@ -68,12 +78,16 @@ policy: max_deep_sample: 75 special_domain: type: dns + # specify that the stream handler module requires >= specific version to be successfully applied + require_version: "1.0" config: - qname_match: .mydomain.com + # must match the available configuration options for this version of this stream handler + qname_suffix: .mydomain.com metrics: disable: - top_qtypes - top_udp_ports + # the sink configuration is not used by pktvisord; see Orb sinks: default_prometheus: type: prometheus_exporter From 57abbc3e7a154defe7aa30f4ad59cf04d3a6eff9 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 18 Mar 2021 09:41:02 -0400 Subject: [PATCH 04/13] design doc work --- docs/CONTROL_PLANE.md | 30 +++++++++++++++++++++++++++++- docs/POLICIES.md | 13 ++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/CONTROL_PLANE.md b/docs/CONTROL_PLANE.md index 7dd7f052b..dc2f4e2d5 100644 --- a/docs/CONTROL_PLANE.md +++ b/docs/CONTROL_PLANE.md @@ -14,11 +14,20 @@ All interfaces and schemas are versioned. ``` /api/v1/inputs { - pcap: "1.0" + pcap: "1.0", + "dnstap": "1.0" } /api/v1/inputs/pcap/interface { version: "1.0", + "info": { + "interfaces": { + "eth0": {} + } + }, + "defaults": { + "interface": eth0 + }, config: { iface: { type: "string", @@ -34,6 +43,25 @@ All interfaces and schemas are versioned. metric_groups: { } } +/api/v1/inputs/dnstap/interface + { + version: "1.0", + config: { + socket: { + type: "string", + description: "the dnstap socket to listen to" + } + } + filters: { + qname_suffix: { + type: "string", + description: "match the DNS qname sufix given", + regex: "..." + } + }, + metric_groups: { + } + } /api/v1/handlers { dns: { version: "1.0" }, net: { version: "1.0" } } diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 2d2fde04c..656b6ca56 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -46,15 +46,10 @@ policy: # input stream # only one is supported currently input: - anycast: - # must match the input stream module name from pktvisor - type: pcap - # specify that the input module requires >= specific version to be successfully applied - require_version: "1.0" - config: - # must match the available configuration options from this version of the inputs stream - bpf: "host 192.168.0.50" - iface: eth0 + type: pcap + targets: + - anycast + - prod0 # stream handlers handlers: # default configuration for the stream handlers From d6a87061a4c5408ec53d264ece39cd992e62beab Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 10:27:25 -0400 Subject: [PATCH 05/13] policy design work. add doxygen file. --- .gitignore | 1 + docs/Doxyfile | 391 +++++++++++++++++++++++++++++++++++++++++++++++ docs/POLICIES.md | 55 ++++++- 3 files changed, 442 insertions(+), 5 deletions(-) create mode 100644 docs/Doxyfile diff --git a/.gitignore b/.gitignore index 061f69f56..c4fb154b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ cmake-build-*/ docs/html-documentation-generated* integration_tests/external golang/pkg/client/version.go +docs/internals/html diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 000000000..a90997659 --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,391 @@ +# Doxyfile 1.9.1 + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = pktvisor +PROJECT_NUMBER = +PROJECT_BRIEF = "pktvisor summarizes network data streams in real time, enabling on-node and centralized data visibility and analysis via API" +PROJECT_LOGO = +OUTPUT_DIRECTORY = ./internals +CREATE_SUBDIRS = NO +ALLOW_UNICODE_NAMES = NO +OUTPUT_LANGUAGE = English +OUTPUT_TEXT_DIRECTION = None +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = NO +JAVADOC_BANNER = NO +QT_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +PYTHON_DOCSTRING = YES +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 4 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +OPTIMIZE_OUTPUT_SLICE = NO +EXTENSION_MAPPING = +MARKDOWN_SUPPORT = YES +TOC_INCLUDE_HEADINGS = 5 +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = NO +GROUP_NESTED_COMPOUNDS = NO +SUBGROUPING = YES +INLINE_GROUPED_CLASSES = NO +INLINE_SIMPLE_STRUCTS = NO +TYPEDEF_HIDES_STRUCT = NO +LOOKUP_CACHE_SIZE = 0 +NUM_PROC_THREADS = 1 +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_PRIVATE = NO +EXTRACT_PRIV_VIRTUAL = NO +EXTRACT_PACKAGE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +RESOLVE_UNNAMED_PARAMS = YES +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = NO +HIDE_SCOPE_NAMES = NO +HIDE_COMPOUND_REFERENCE= NO +SHOW_INCLUDE_FILES = YES +SHOW_GROUPED_MEMB_INC = NO +FORCE_LOCAL_INCLUDES = NO +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = NO +SORT_MEMBERS_CTORS_1ST = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +STRICT_PROTO_MATCHING = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = YES +FILE_VERSION_FILTER = +LAYOUT_FILE = +CITE_BIB_FILES = +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = NO +WARN_AS_ERROR = NO +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = ../src +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice +RECURSIVE = YES +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = * +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +FILTER_SOURCE_PATTERNS = +USE_MDFILE_AS_MAINPAGE = +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = NO +REFERENCES_RELATION = NO +REFERENCES_LINK_SOURCE = YES +SOURCE_TOOLTIPS = YES +USE_HTAGS = NO +VERBATIM_HEADERS = YES +CLANG_ASSISTED_PARSING = NO +CLANG_ADD_INC_PATHS = YES +CLANG_OPTIONS = +CLANG_DATABASE_PATH = +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = YES +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_EXTRA_STYLESHEET = +HTML_EXTRA_FILES = +HTML_COLORSTYLE_HUE = 220 +HTML_COLORSTYLE_SAT = 100 +HTML_COLORSTYLE_GAMMA = 80 +HTML_TIMESTAMP = NO +HTML_DYNAMIC_MENUS = YES +HTML_DYNAMIC_SECTIONS = NO +HTML_INDEX_NUM_ENTRIES = 100 +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_NAME = Publisher +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = NO +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = org.doxygen.Project +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +GENERATE_ECLIPSEHELP = NO +ECLIPSE_DOC_ID = org.doxygen.Project +DISABLE_INDEX = NO +GENERATE_TREEVIEW = YES +ENUM_VALUES_PER_LINE = 4 +TREEVIEW_WIDTH = 250 +EXT_LINKS_IN_WINDOW = NO +HTML_FORMULA_FORMAT = png +FORMULA_FONTSIZE = 10 +FORMULA_TRANSPARENT = YES +FORMULA_MACROFILE = +USE_MATHJAX = NO +MATHJAX_FORMAT = HTML-CSS +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 +MATHJAX_EXTENSIONS = +MATHJAX_CODEFILE = +SEARCHENGINE = YES +SERVER_BASED_SEARCH = NO +EXTERNAL_SEARCH = NO +SEARCHENGINE_URL = +SEARCHDATA_FILE = searchdata.xml +EXTERNAL_SEARCH_ID = +EXTRA_SEARCH_MAPPINGS = +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = +MAKEINDEX_CMD_NAME = makeindex +LATEX_MAKEINDEX_CMD = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4 +EXTRA_PACKAGES = +LATEX_HEADER = +LATEX_FOOTER = +LATEX_EXTRA_STYLESHEET = +LATEX_EXTRA_FILES = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +LATEX_SOURCE_CODE = NO +LATEX_BIB_STYLE = plain +LATEX_TIMESTAMP = NO +LATEX_EMOJI_DIRECTORY = +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +RTF_SOURCE_CODE = NO +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_SUBDIR = +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_PROGRAMLISTING = YES +XML_NS_MEMB_FILE_SCOPE = NO +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- +GENERATE_DOCBOOK = NO +DOCBOOK_OUTPUT = docbook +DOCBOOK_PROGRAMLISTING = NO +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +EXTERNAL_PAGES = YES +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- +CLASS_DIAGRAMS = YES +DIA_PATH = +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = NO +DOT_NUM_THREADS = 0 +DOT_FONTNAME = Helvetica +DOT_FONTSIZE = 10 +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +UML_LIMIT_NUM_FIELDS = 10 +DOT_UML_DETAILS = NO +DOT_WRAP_THRESHOLD = 17 +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +CALLER_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +INTERACTIVE_SVG = NO +DOT_PATH = +DOTFILE_DIRS = +MSCFILE_DIRS = +DIAFILE_DIRS = +PLANTUML_JAR_PATH = +PLANTUML_CFG_FILE = +PLANTUML_INCLUDE_PATH = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_TRANSPARENT = NO +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 656b6ca56..a2be54204 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -39,17 +39,45 @@ Examples: ## Example (YAML) +Taps are named, host specific connections to raw stream data. They represent configuration data only, they do not cause +any processing to take place. They should be referenced by policies. These may be configured on the command line at +start up (often using a configuration management system). + +```yaml +version: "1.0" + +taps: + anycast: + type: pcap + config: + iface: eth0 + pop_switch: + type: sflow + config: + port: 6343 + bind: 192.168.1.1 + trex_tap: + type: dnstap + config: + socket: /var/dns.sock +``` + +Policies use taps to create an instance of an input (possibly with a filter), and attach handlers to it. Processing +takes place, and the data is exposed to sinks. + ```yaml version: "1.0" policy: + name: "anycast_dns" + description: "anycast DNS policy" # input stream - # only one is supported currently + # only one is supported, but multiple policies may reuse host_inputs input: + tap: anycast type: pcap - targets: - - anycast - - prod0 + filter: + bpf: "port 53" # stream handlers handlers: # default configuration for the stream handlers @@ -82,7 +110,14 @@ policy: disable: - top_qtypes - top_udp_ports - # the sink configuration is not used by pktvisord; see Orb +``` + +Orb: Sinks specify where to send summarized metric data. + +```yaml +version: "1.0" + +policy: sinks: default_prometheus: type: prometheus_exporter @@ -95,6 +130,16 @@ policy: region: us-east-1 ``` +Orb: Selectors indicate which agent should apply a policy. + +```yaml +version: "1.0" + +policy: + selector: + - global/EU/ams +``` + From 44f54dcfab764945c76e12be8f60b0aa305f700c Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 11:15:13 -0400 Subject: [PATCH 06/13] policy design work --- docs/POLICIES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index a2be54204..13883a334 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -43,6 +43,12 @@ Taps are named, host specific connections to raw stream data. They represent con any processing to take place. They should be referenced by policies. These may be configured on the command line at start up (often using a configuration management system). +```shell +$ orb-agent --tap-config taps.yaml --selector-watch global/EU/ami +``` + +taps.yaml + ```yaml version: "1.0" From 8ad58a95f41e397e1243a8e71bf4799f036d58d9 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 14:32:46 -0400 Subject: [PATCH 07/13] policy design work --- docs/POLICIES.md | 210 ++++++++++++++++++++++++++--------------------- 1 file changed, 116 insertions(+), 94 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 13883a334..0b592b44d 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -1,124 +1,134 @@ -# Policies +# Orb and pktvisor Policy Driven Configuration **_Draft_** -pktvisord observability configuration is policy driven. Policies act as a grouping of an input stream to stream -handlers, and their associated configuration. +pktvisor and Orb observability configuration is policy driven. -## Property Attributes +pktvisor maybe run stand alone, or with the Orb control plane. In the latter configuration, orb-agent controls the +pktvisord agent and allows centralized configuration via orb-api. -* Policies have exactly one Input source -* Policies have an unlimited number of Stream Handlers -* Policies have an unlimited number of Sinks (which is not used directly by pktvisord; see Orb) -* Policies contain configuration data, including observability filters and metric collection configuration. +## Concepts -## Policy Semantics +### pktvisor Taps -* The REST API path schema enforces the hierarchy of Policy -> Input => Handlers +Taps are named, host specific connections to raw stream data accessed by pktvisord. They represent configuration data +only; they do not cause any processing to take place in pktvisord. They should be referenced by Collection Policies by +name (see below). -``` -/api/v1/policy/:POLICY_NAME:/:INPUT_MODULE_TYPE:/handler/:HANDLER_MODULE_TYPE:/:HANDLER_NAME:/ -``` +Taps may be configured on the command line at agent start up (often using a configuration management system) either +directly in pktvisord (via command line or admin API) when running stand alone, or indirectly via orb-agent. See Command +Line Examples below. -Examples: +`taps.yaml` -``` -/api/v1/policy/anycast/pcap/handler/dns/default_dns/bucket/0 -/api/v1/policy/anycast/pcap/handler/net/default_net/bucket/0 -/api/v1/policy/anycast/pcap/handler/dns/special_domain/bucket/0 +```yaml +version: "1.0" -/api/v1/policy/management/pcap/handler/dns/dns_collection/bucket/0 -/api/v1/policy/management/pcap/handler/net/net_collection/bucket/0 +policy: + # each tap has input module specific configuration options + taps: + # a pcap tap which uses eth0 and is referenced by the identifier "anycast" + anycast: + type: pcap + config: + iface: eth0 + # an sflow tap which listens on the given IP and port, referenced by the identifier "pop_switch" + pop_switch: + type: sflow + config: + port: 6343 + bind: 192.168.1.1 + # a dnstap tap which gets its stream from the given socket, named "trex_tap" + trex_tap: + type: dnstap + config: + socket: /var/dns.sock ``` -## Policy Schema +### pktvisor Collection Policies -* Policy schema is represented by YAML -* The schema may represent policy information not directly used by pktvisor, such as where to send output -* The schema is versioned +Collection policies direct pktvisor to use taps to create an instance of an input stream (possibly with a filter), and +attach handlers to it. Processing takes place, and the data is exposed for sinks to collect. These policies may be given +directly to pktvisor (via command line or admin API), or via orb control plane. -## Example (YAML) +`collection-policy-anycast.yaml` -Taps are named, host specific connections to raw stream data. They represent configuration data only, they do not cause -any processing to take place. They should be referenced by policies. These may be configured on the command line at -start up (often using a configuration management system). +```yaml +version: "1.0" -```shell -$ orb-agent --tap-config taps.yaml --selector-watch global/EU/ami +policy: + collection: + # policy name and description + name: anycast_dns + description: "base anycast DNS policy" + # input stream to create based on the given tap and optional filter config + input: + # this most reference a tap name, or application of the policy will fail + tap: anycast + # this must match the type of the matching tap name. or application of the policy will fail + type: pcap + filter: + bpf: "port 53" + # stream handlers to attach to this input stream + # these decide exactly which data to summarize and expose for collection + handlers: + # default configuration for the stream handlers + config: + periods: 5 + max_deep_sample: 50 + modules: + # the keys at this level are unique identifiers + default_net: + type: net + udp_traffic: + type: net + config: + protocols: [ udp ] + metrics: + enable: + - top_ips + default_dns: + type: dns + config: + max_deep_sample: 75 + special_domain: + type: dns + # specify that the stream handler module requires >= specific version to be successfully applied + require_version: "1.0" + config: + # must match the available configuration options for this version of this stream handler + qname_suffix: .mydomain.com + metrics: + disable: + - top_qtypes + - top_udp_ports ``` -taps.yaml +### Standalone Command Line Examples for Taps and Collection Policies -```yaml -version: "1.0" +#### Standalone agent start up + +When running without Orb, the tap and the collection config can be passed in directly to pktvisor. -taps: - anycast: - type: pcap - config: - iface: eth0 - pop_switch: - type: sflow - config: - port: 6343 - bind: 192.168.1.1 - trex_tap: - type: dnstap - config: - socket: /var/dns.sock +```shell +$ pktvisord --tap-config taps.yaml --collection-config collection-policy-anycast.yaml ``` -Policies use taps to create an instance of an input (possibly with a filter), and attach handlers to it. Processing -takes place, and the data is exposed to sinks. +The admin-api (or prometheus output, pktvisor-cli, etc) should then be used to collect the result manually. -```yaml -version: "1.0" +#### orb-agent start up -policy: - name: "anycast_dns" - description: "anycast DNS policy" - # input stream - # only one is supported, but multiple policies may reuse host_inputs - input: - tap: anycast - type: pcap - filter: - bpf: "port 53" - # stream handlers - handlers: - # default configuration for the stream handlers - config: - periods: 5 - max_deep_sample: 50 - modules: - # the keys at this level are unique identifiers - default_net: - type: net - udp_traffic: - type: net - config: - protocols: [ udp ] - metrics: - enable: - - top_ips - default_dns: - type: dns - config: - max_deep_sample: 75 - special_domain: - type: dns - # specify that the stream handler module requires >= specific version to be successfully applied - require_version: "1.0" - config: - # must match the available configuration options for this version of this stream handler - qname_suffix: .mydomain.com - metrics: - disable: - - top_qtypes - - top_udp_ports +When running with Orb, the agent accepts a configuration YAML combining Taps and Selectors. Instead of accepting +Collection Policies on the command line, these instead would be sent through the central orb-api via Selectors that +match those for this agent. + +```shell +$ orb-agent --config orb.yaml ``` -Orb: Sinks specify where to send summarized metric data. +### Orb Sinks + +Sinks specify where to send summarized metric data. ```yaml version: "1.0" @@ -142,10 +152,22 @@ Orb: Selectors indicate which agent should apply a policy. version: "1.0" policy: + name: "dns" selector: - global/EU/ams ``` +```yaml +version: "1.0" +policy: + orb: + selector: dns + pktvisor-policy: anycast_dns + sinks: default_prometheus +``` + +*** sketch in single node usage on CLI with single-config yaml version +*** move selector-watch cmd line to yaml too From 0043762595aebc882e2d819f9726cc54aa6ff678 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 15:22:49 -0400 Subject: [PATCH 08/13] policy design work --- docs/POLICIES.md | 91 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 0b592b44d..44ac3fe0a 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -2,12 +2,12 @@ **_Draft_** -pktvisor and Orb observability configuration is policy driven. +Orb and pktvisor observability configuration is policy driven. pktvisor maybe run stand alone, or with the Orb control plane. In the latter configuration, orb-agent controls the -pktvisord agent and allows centralized configuration via orb-api. +pktvisord process and allows centralized configuration via orb-api. -## Concepts +## Base Concepts ### pktvisor Taps @@ -49,7 +49,8 @@ policy: Collection policies direct pktvisor to use taps to create an instance of an input stream (possibly with a filter), and attach handlers to it. Processing takes place, and the data is exposed for sinks to collect. These policies may be given -directly to pktvisor (via command line or admin API), or via orb control plane. +directly to pktvisor (via command line or admin API) in standalone mode, or via Orb control plane (in which case they +are not stored in a file, but rather in the control plane database). `collection-policy-anycast.yaml` @@ -104,26 +105,79 @@ policy: - top_udp_ports ``` -### Standalone Command Line Examples for Taps and Collection Policies +### Standalone Command Line Example #### Standalone agent start up -When running without Orb, the tap and the collection config can be passed in directly to pktvisor. +When running without Orb, the tap and the collection config can be passed directly to pktvisor. ```shell $ pktvisord --tap-config taps.yaml --collection-config collection-policy-anycast.yaml ``` -The admin-api (or prometheus output, pktvisor-cli, etc) should then be used to collect the result manually. +The admin-api (or prometheus output, pktvisor-cli, etc) should then be used to collect the results manually. -#### orb-agent start up +## Orb Concepts -When running with Orb, the agent accepts a configuration YAML combining Taps and Selectors. Instead of accepting -Collection Policies on the command line, these instead would be sent through the central orb-api via Selectors that -match those for this agent. +Orb moves most of the configuration to a central control plane. The only configuration that remains at the agent is the +Tap configuration (because it is host specific), and Vitals configuration (below). + +### Vitals and Selector Configurations + +Orb needs the ability to address the agents that it is controlling. It does this by matching Selectors with Vitals. + +#### Vitals + +orb-agent is told on startup what its Vitals are: these are arbitrary key value pairs which typically represent +information such as region, pop, and node type. + +`vitals.yaml` + +```yaml +version: "1.0" + +policy: + vitals: + region: EU + pop: ams02 + node_type: dns +``` + +#### vitals on orb-agent start up + +```shell +$ orb-agent --config vitals.yaml +``` + +#### combining vitals and taps on orb-agent start up + +Since both Taps and Vitals are necessary for orb-agent start up, you can pass both in via two separate config files: + +```shell +$ orb-agent --config taps.yaml --config vitals.yaml +``` + +Or instead combine them into a single file: + +`orb-agent.yaml` + +```yaml +version: "1.0" + +policy: + taps: + anycast: + type: pcap + config: + iface: eth0 + vitals: + region: EU + pop: ams02 + node_type: dns +``` ```shell -$ orb-agent --config orb.yaml +$ orb-agent --config orb-agent.yaml ``` ### Orb Sinks @@ -146,17 +200,6 @@ policy: region: us-east-1 ``` -Orb: Selectors indicate which agent should apply a policy. - -```yaml -version: "1.0" - -policy: - name: "dns" - selector: - - global/EU/ams -``` - ```yaml version: "1.0" policy: @@ -166,8 +209,6 @@ policy: sinks: default_prometheus ``` -*** sketch in single node usage on CLI with single-config yaml version -*** move selector-watch cmd line to yaml too From eff1388e02611e2ff79c0408d4ab659d0e8fa84d Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 15:44:38 -0400 Subject: [PATCH 09/13] policy design work --- docs/POLICIES.md | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 44ac3fe0a..facc2e2f0 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -180,9 +180,30 @@ policy: $ orb-agent --config orb-agent.yaml ``` +### Orb Selectors + +Selectors are named configurations of arbitrary key value pairs which can match against the Vitals of the agents +available in the Orb ecosystem. They may be thought of as groups of agents. These names are referenced in Ord Policies. +pktvisord does not read this configuration or use this data; it is used only by orb-agent. This schema is found only in +the control plane, not on the command line or in files. + +```yaml +version: "1.0" + +policy: + selectors: + all_dns: + node_type: dns + eu_dns: + region: EU + node_type: dns +``` + ### Orb Sinks -Sinks specify where to send summarized metric data. +Orb includes a metric collection system. Sinks specify where to send the summarized metric data. pktvisord does not read +this configuration or use this data; it is used only by orb-agent. This schema is found only in the control plane, not +on the command line or in files. ```yaml version: "1.0" @@ -200,13 +221,26 @@ policy: region: us-east-1 ``` +### Orb Policies + +An Orb policy ties together Selectors, a Collection Policy, and one or more Sinks. pktvisord does not read this +configuration or use this data; it is used only by orb-agent. This schema is found only in the control plane, not on the +command line or in files. + +orb-agent will be made aware of the collection policy and the sinks if this selector matches its vitals. In case of a +match, orb-agent will attempt to apply the collection policy to its pktvisord, and update the control plane about +success or failure. Upon success, the sink will be created. + ```yaml version: "1.0" + policy: orb: - selector: dns - pktvisor-policy: anycast_dns - sinks: default_prometheus + selectors: + - eu_dns + collection_policy: anycast_dns + sinks: + - default_prometheus ``` From 7e2ec73774ae1dc3a11bbb6dca7188a10e63d018 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 15:47:30 -0400 Subject: [PATCH 10/13] policy design work --- docs/POLICIES.md | 86 ++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index facc2e2f0..30dd3ab37 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -60,49 +60,49 @@ version: "1.0" policy: collection: # policy name and description - name: anycast_dns - description: "base anycast DNS policy" - # input stream to create based on the given tap and optional filter config - input: - # this most reference a tap name, or application of the policy will fail - tap: anycast - # this must match the type of the matching tap name. or application of the policy will fail - type: pcap - filter: - bpf: "port 53" - # stream handlers to attach to this input stream - # these decide exactly which data to summarize and expose for collection - handlers: - # default configuration for the stream handlers - config: - periods: 5 - max_deep_sample: 50 - modules: - # the keys at this level are unique identifiers - default_net: - type: net - udp_traffic: - type: net - config: - protocols: [ udp ] - metrics: - enable: - - top_ips - default_dns: - type: dns - config: - max_deep_sample: 75 - special_domain: - type: dns - # specify that the stream handler module requires >= specific version to be successfully applied - require_version: "1.0" - config: - # must match the available configuration options for this version of this stream handler - qname_suffix: .mydomain.com - metrics: - disable: - - top_qtypes - - top_udp_ports + anycast_dns: + description: "base anycast DNS policy" + # input stream to create based on the given tap and optional filter config + input: + # this most reference a tap name, or application of the policy will fail + tap: anycast + # this must match the type of the matching tap name. or application of the policy will fail + type: pcap + filter: + bpf: "port 53" + # stream handlers to attach to this input stream + # these decide exactly which data to summarize and expose for collection + handlers: + # default configuration for the stream handlers + config: + periods: 5 + max_deep_sample: 50 + modules: + # the keys at this level are unique identifiers + default_net: + type: net + udp_traffic: + type: net + config: + protocols: [ udp ] + metrics: + enable: + - top_ips + default_dns: + type: dns + config: + max_deep_sample: 75 + special_domain: + type: dns + # specify that the stream handler module requires >= specific version to be successfully applied + require_version: "1.0" + config: + # must match the available configuration options for this version of this stream handler + qname_suffix: .mydomain.com + metrics: + disable: + - top_qtypes + - top_udp_ports ``` ### Standalone Command Line Example From 6ba3899b7e075f82c2d5edba52530a5863dc29ef Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 15:50:08 -0400 Subject: [PATCH 11/13] policy design work --- docs/POLICIES.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 30dd3ab37..5ef32047b 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -64,7 +64,7 @@ policy: description: "base anycast DNS policy" # input stream to create based on the given tap and optional filter config input: - # this most reference a tap name, or application of the policy will fail + # this must reference a tap name, or application of the policy will fail tap: anycast # this must match the type of the matching tap name. or application of the policy will fail type: pcap @@ -112,9 +112,11 @@ policy: When running without Orb, the tap and the collection config can be passed directly to pktvisor. ```shell -$ pktvisord --tap-config taps.yaml --collection-config collection-policy-anycast.yaml +$ pktvisord --config taps.yaml --config collection-policy-anycast.yaml ``` +They may also be combined into a single YAML file (the schemas will merge) and passed in with one `--config` option. + The admin-api (or prometheus output, pktvisor-cli, etc) should then be used to collect the results manually. ## Orb Concepts From 250c0a07053397d2128ca69c718aae9d57bab6e7 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 19 Mar 2021 15:53:01 -0400 Subject: [PATCH 12/13] policy design work --- docs/POLICIES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 5ef32047b..ba890d091 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -185,7 +185,7 @@ $ orb-agent --config orb-agent.yaml ### Orb Selectors Selectors are named configurations of arbitrary key value pairs which can match against the Vitals of the agents -available in the Orb ecosystem. They may be thought of as groups of agents. These names are referenced in Ord Policies. +available in the Orb ecosystem. They may be thought of as groups of agents. These names are referenced in Orb Policies. pktvisord does not read this configuration or use this data; it is used only by orb-agent. This schema is found only in the control plane, not on the command line or in files. From ed47ceae5adc1dc878ca7ce7ad2c6c505e693649 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Sat, 20 Mar 2021 11:13:15 -0400 Subject: [PATCH 13/13] add analyzers --- docs/POLICIES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index ba890d091..85ce8b086 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -92,6 +92,11 @@ policy: type: dns config: max_deep_sample: 75 + # time window analyzers + analyzers: + modules: + nx_attack: + type: dns_random_label special_domain: type: dns # specify that the stream handler module requires >= specific version to be successfully applied