diff --git a/.gitignore b/.gitignore index 061f69f56..c4fb154b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ cmake-build-*/ docs/html-documentation-generated* integration_tests/external golang/pkg/client/version.go +docs/internals/html diff --git a/docs/CONTROL_PLANE.md b/docs/CONTROL_PLANE.md new file mode 100644 index 000000000..dc2f4e2d5 --- /dev/null +++ b/docs/CONTROL_PLANE.md @@ -0,0 +1,114 @@ +# Control Plane + +**_Draft_** + +pktvisord exposes a control plane over REST API. + +## Discovery + +pktvisord exposes a method for discovering the available modules, their configurable properties, and their associated +metrics schema. + +All interfaces and schemas are versioned. + +``` +/api/v1/inputs + { + pcap: "1.0", + "dnstap": "1.0" + } +/api/v1/inputs/pcap/interface + { + version: "1.0", + "info": { + "interfaces": { + "eth0": {} + } + }, + "defaults": { + "interface": eth0 + }, + config: { + iface: { + type: "string", + description: "the ethernet interface to capture on" + } + } + filters: { + bpf: { + type: "string", + description: "tcpdump compatible bpf filter expression" + } + }, + metric_groups: { + } + } +/api/v1/inputs/dnstap/interface + { + version: "1.0", + config: { + socket: { + type: "string", + description: "the dnstap socket to listen to" + } + } + filters: { + qname_suffix: { + type: "string", + description: "match the DNS qname sufix given", + regex: "..." + } + }, + metric_groups: { + } + } +/api/v1/handlers + { dns: { version: "1.0" }, + net: { version: "1.0" } } +/api/v1/handlers/dns/interface + { + version: "1.0", + config: { + periods: { + type: "int", + description: "number of metric periods to keep" + } + } + filters: { + qname_suffix: { + type: "string", + description: "match the DNS qname sufix given", + regex: "..." + } + }, + metric_groups: { + top_error_qnames: { + description: "top N qnames with error result codes", + metrics: {, + top_refused: { + "type": "top_n", + "description": "..." + }, + top_srvfail: { + "type": "top_n", + "description": "..." + }, + top_nxdomain: { + "type": "top_n", + "description": "..." + }, + } + }, + transactions: { + description: "information on query/reply pairs", + metrics: { + ... + } + } + } + } +/api/v1/handlers/net/interface + { + } +``` + diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 000000000..a90997659 --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,391 @@ +# Doxyfile 1.9.1 + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = pktvisor +PROJECT_NUMBER = +PROJECT_BRIEF = "pktvisor summarizes network data streams in real time, enabling on-node and centralized data visibility and analysis via API" +PROJECT_LOGO = +OUTPUT_DIRECTORY = ./internals +CREATE_SUBDIRS = NO +ALLOW_UNICODE_NAMES = NO +OUTPUT_LANGUAGE = English +OUTPUT_TEXT_DIRECTION = None +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = NO +JAVADOC_BANNER = NO +QT_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +PYTHON_DOCSTRING = YES +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 4 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +OPTIMIZE_OUTPUT_SLICE = NO +EXTENSION_MAPPING = +MARKDOWN_SUPPORT = YES +TOC_INCLUDE_HEADINGS = 5 +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = NO +GROUP_NESTED_COMPOUNDS = NO +SUBGROUPING = YES +INLINE_GROUPED_CLASSES = NO +INLINE_SIMPLE_STRUCTS = NO +TYPEDEF_HIDES_STRUCT = NO +LOOKUP_CACHE_SIZE = 0 +NUM_PROC_THREADS = 1 +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_PRIVATE = NO +EXTRACT_PRIV_VIRTUAL = NO +EXTRACT_PACKAGE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +RESOLVE_UNNAMED_PARAMS = YES +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = NO +HIDE_SCOPE_NAMES = NO +HIDE_COMPOUND_REFERENCE= NO +SHOW_INCLUDE_FILES = YES +SHOW_GROUPED_MEMB_INC = NO +FORCE_LOCAL_INCLUDES = NO +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = NO +SORT_MEMBERS_CTORS_1ST = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +STRICT_PROTO_MATCHING = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = YES +FILE_VERSION_FILTER = +LAYOUT_FILE = +CITE_BIB_FILES = +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = NO +WARN_AS_ERROR = NO +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = ../src +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice +RECURSIVE = YES +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = * +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +FILTER_SOURCE_PATTERNS = +USE_MDFILE_AS_MAINPAGE = +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = NO +REFERENCES_RELATION = NO +REFERENCES_LINK_SOURCE = YES +SOURCE_TOOLTIPS = YES +USE_HTAGS = NO +VERBATIM_HEADERS = YES +CLANG_ASSISTED_PARSING = NO +CLANG_ADD_INC_PATHS = YES +CLANG_OPTIONS = +CLANG_DATABASE_PATH = +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = YES +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_EXTRA_STYLESHEET = +HTML_EXTRA_FILES = +HTML_COLORSTYLE_HUE = 220 +HTML_COLORSTYLE_SAT = 100 +HTML_COLORSTYLE_GAMMA = 80 +HTML_TIMESTAMP = NO +HTML_DYNAMIC_MENUS = YES +HTML_DYNAMIC_SECTIONS = NO +HTML_INDEX_NUM_ENTRIES = 100 +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_NAME = Publisher +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = NO +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = org.doxygen.Project +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +GENERATE_ECLIPSEHELP = NO +ECLIPSE_DOC_ID = org.doxygen.Project +DISABLE_INDEX = NO +GENERATE_TREEVIEW = YES +ENUM_VALUES_PER_LINE = 4 +TREEVIEW_WIDTH = 250 +EXT_LINKS_IN_WINDOW = NO +HTML_FORMULA_FORMAT = png +FORMULA_FONTSIZE = 10 +FORMULA_TRANSPARENT = YES +FORMULA_MACROFILE = +USE_MATHJAX = NO +MATHJAX_FORMAT = HTML-CSS +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 +MATHJAX_EXTENSIONS = +MATHJAX_CODEFILE = +SEARCHENGINE = YES +SERVER_BASED_SEARCH = NO +EXTERNAL_SEARCH = NO +SEARCHENGINE_URL = +SEARCHDATA_FILE = searchdata.xml +EXTERNAL_SEARCH_ID = +EXTRA_SEARCH_MAPPINGS = +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = +MAKEINDEX_CMD_NAME = makeindex +LATEX_MAKEINDEX_CMD = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4 +EXTRA_PACKAGES = +LATEX_HEADER = +LATEX_FOOTER = +LATEX_EXTRA_STYLESHEET = +LATEX_EXTRA_FILES = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +LATEX_SOURCE_CODE = NO +LATEX_BIB_STYLE = plain +LATEX_TIMESTAMP = NO +LATEX_EMOJI_DIRECTORY = +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +RTF_SOURCE_CODE = NO +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_SUBDIR = +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_PROGRAMLISTING = YES +XML_NS_MEMB_FILE_SCOPE = NO +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- +GENERATE_DOCBOOK = NO +DOCBOOK_OUTPUT = docbook +DOCBOOK_PROGRAMLISTING = NO +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +EXTERNAL_PAGES = YES +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- +CLASS_DIAGRAMS = YES +DIA_PATH = +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = NO +DOT_NUM_THREADS = 0 +DOT_FONTNAME = Helvetica +DOT_FONTSIZE = 10 +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +UML_LIMIT_NUM_FIELDS = 10 +DOT_UML_DETAILS = NO +DOT_WRAP_THRESHOLD = 17 +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +CALLER_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +INTERACTIVE_SVG = NO +DOT_PATH = +DOTFILE_DIRS = +MSCFILE_DIRS = +DIAFILE_DIRS = +PLANTUML_JAR_PATH = +PLANTUML_CFG_FILE = +PLANTUML_INCLUDE_PATH = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_TRANSPARENT = NO +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES diff --git a/docs/POLICIES.md b/docs/POLICIES.md new file mode 100644 index 000000000..85ce8b086 --- /dev/null +++ b/docs/POLICIES.md @@ -0,0 +1,260 @@ +# Orb and pktvisor Policy Driven Configuration + +**_Draft_** + +Orb and pktvisor observability configuration is policy driven. + +pktvisor maybe run stand alone, or with the Orb control plane. In the latter configuration, orb-agent controls the +pktvisord process and allows centralized configuration via orb-api. + +## Base Concepts + +### pktvisor Taps + +Taps are named, host specific connections to raw stream data accessed by pktvisord. They represent configuration data +only; they do not cause any processing to take place in pktvisord. They should be referenced by Collection Policies by +name (see below). + +Taps may be configured on the command line at agent start up (often using a configuration management system) either +directly in pktvisord (via command line or admin API) when running stand alone, or indirectly via orb-agent. See Command +Line Examples below. + +`taps.yaml` + +```yaml +version: "1.0" + +policy: + # each tap has input module specific configuration options + taps: + # a pcap tap which uses eth0 and is referenced by the identifier "anycast" + anycast: + type: pcap + config: + iface: eth0 + # an sflow tap which listens on the given IP and port, referenced by the identifier "pop_switch" + pop_switch: + type: sflow + config: + port: 6343 + bind: 192.168.1.1 + # a dnstap tap which gets its stream from the given socket, named "trex_tap" + trex_tap: + type: dnstap + config: + socket: /var/dns.sock +``` + +### pktvisor Collection Policies + +Collection policies direct pktvisor to use taps to create an instance of an input stream (possibly with a filter), and +attach handlers to it. Processing takes place, and the data is exposed for sinks to collect. These policies may be given +directly to pktvisor (via command line or admin API) in standalone mode, or via Orb control plane (in which case they +are not stored in a file, but rather in the control plane database). + +`collection-policy-anycast.yaml` + +```yaml +version: "1.0" + +policy: + collection: + # policy name and description + anycast_dns: + description: "base anycast DNS policy" + # input stream to create based on the given tap and optional filter config + input: + # this must reference a tap name, or application of the policy will fail + tap: anycast + # this must match the type of the matching tap name. or application of the policy will fail + type: pcap + filter: + bpf: "port 53" + # stream handlers to attach to this input stream + # these decide exactly which data to summarize and expose for collection + handlers: + # default configuration for the stream handlers + config: + periods: 5 + max_deep_sample: 50 + modules: + # the keys at this level are unique identifiers + default_net: + type: net + udp_traffic: + type: net + config: + protocols: [ udp ] + metrics: + enable: + - top_ips + default_dns: + type: dns + config: + max_deep_sample: 75 + # time window analyzers + analyzers: + modules: + nx_attack: + type: dns_random_label + special_domain: + type: dns + # specify that the stream handler module requires >= specific version to be successfully applied + require_version: "1.0" + config: + # must match the available configuration options for this version of this stream handler + qname_suffix: .mydomain.com + metrics: + disable: + - top_qtypes + - top_udp_ports +``` + +### Standalone Command Line Example + +#### Standalone agent start up + +When running without Orb, the tap and the collection config can be passed directly to pktvisor. + +```shell +$ pktvisord --config taps.yaml --config collection-policy-anycast.yaml +``` + +They may also be combined into a single YAML file (the schemas will merge) and passed in with one `--config` option. + +The admin-api (or prometheus output, pktvisor-cli, etc) should then be used to collect the results manually. + +## Orb Concepts + +Orb moves most of the configuration to a central control plane. The only configuration that remains at the agent is the +Tap configuration (because it is host specific), and Vitals configuration (below). + +### Vitals and Selector Configurations + +Orb needs the ability to address the agents that it is controlling. It does this by matching Selectors with Vitals. + +#### Vitals + +orb-agent is told on startup what its Vitals are: these are arbitrary key value pairs which typically represent +information such as region, pop, and node type. + +`vitals.yaml` + +```yaml +version: "1.0" + +policy: + vitals: + region: EU + pop: ams02 + node_type: dns +``` + +#### vitals on orb-agent start up + +```shell +$ orb-agent --config vitals.yaml +``` + +#### combining vitals and taps on orb-agent start up + +Since both Taps and Vitals are necessary for orb-agent start up, you can pass both in via two separate config files: + +```shell +$ orb-agent --config taps.yaml --config vitals.yaml +``` + +Or instead combine them into a single file: + +`orb-agent.yaml` + +```yaml +version: "1.0" + +policy: + taps: + anycast: + type: pcap + config: + iface: eth0 + vitals: + region: EU + pop: ams02 + node_type: dns +``` + +```shell +$ orb-agent --config orb-agent.yaml +``` + +### Orb Selectors + +Selectors are named configurations of arbitrary key value pairs which can match against the Vitals of the agents +available in the Orb ecosystem. They may be thought of as groups of agents. These names are referenced in Orb Policies. +pktvisord does not read this configuration or use this data; it is used only by orb-agent. This schema is found only in +the control plane, not on the command line or in files. + +```yaml +version: "1.0" + +policy: + selectors: + all_dns: + node_type: dns + eu_dns: + region: EU + node_type: dns +``` + +### Orb Sinks + +Orb includes a metric collection system. Sinks specify where to send the summarized metric data. pktvisord does not read +this configuration or use this data; it is used only by orb-agent. This schema is found only in the control plane, not +on the command line or in files. + +```yaml +version: "1.0" + +policy: + sinks: + default_prometheus: + type: prometheus_exporter + address: 0.0.0.0:9598 + default_namespace: service + my_s3: + type: aws_s3 + bucket: my-bucket + compression: gzip + region: us-east-1 +``` + +### Orb Policies + +An Orb policy ties together Selectors, a Collection Policy, and one or more Sinks. pktvisord does not read this +configuration or use this data; it is used only by orb-agent. This schema is found only in the control plane, not on the +command line or in files. + +orb-agent will be made aware of the collection policy and the sinks if this selector matches its vitals. In case of a +match, orb-agent will attempt to apply the collection policy to its pktvisord, and update the control plane about +success or failure. Upon success, the sink will be created. + +```yaml +version: "1.0" + +policy: + orb: + selectors: + - eu_dns + collection_policy: anycast_dns + sinks: + - default_prometheus +``` + + + + + + + + +