Merge pull request #1159 from jberry-suse/metrics-v1

Initial metrics ingestion, processing, write to influxdb, and grafana dashboards.

Makefile

@@ -5,7 +5,7 @@ include Makefile.common
 pkgdata_BINS=repo_checker
 pkgdata_SCRIPTS=$(wildcard *.py *.pl *.sh)
 pkgdata_SCRIPTS+=bs_mirrorfull findfileconflicts
-pkgdata_DATA+=bs_copy osclib $(wildcard *.pm *.testcase)
+pkgdata_DATA+=bs_copy metrics osclib $(wildcard *.pm *.testcase)
 package_name = openSUSE-release-tools
 VERSION = "build-$(shell date +%F)"
 

dist/package/openSUSE-release-tools.spec

@@ -109,6 +109,17 @@ BuildArch:      noarch
 %description announcer
 OBS product release announcer for generating email diffs summaries.
 
+%package metrics
+Summary:        Ingest relevant data to generate insightful metrics
+Group:          Development/Tools/Other
+BuildArch:      noarch
+# TODO Update requirements.
+Requires:       osclib = %{version}
+# TODO Requires: python-influxdb, but package does not exist.
+
+%description metrics
+Ingest relevant OBS and annotation data to generate insightful metrics.
+
 %package repo-checker
 Summary:        Repository checker service
 Group:          Development/Tools/Other
@@ -203,6 +214,8 @@ mkdir -p %{buildroot}%{_datadir}/%{source_dir}/%{announcer_filename}
 %postun announcer
 %service_del_postun %{announcer_filename}.service
 
+# TODO Provide metrics service once #1006 is resolved.
+
 %pre repo-checker
 %service_add_pre osrt-repo-checker.service
 getent passwd osrt-repo-checker > /dev/null || \
@@ -236,6 +249,8 @@ exit 0
 %{_datadir}/%{source_dir}
 %exclude %{_datadir}/%{source_dir}/abichecker
 %exclude %{_datadir}/%{source_dir}/%{announcer_filename}
+%exclude %{_datadir}/%{source_dir}/metrics
+%exclude %{_datadir}/%{source_dir}/metrics.py
 %exclude %{_datadir}/%{source_dir}/repo_checker.pl
 %exclude %{_datadir}/%{source_dir}/repo_checker.py
 %exclude %{_datadir}/%{source_dir}/totest-manager.py
@@ -265,6 +280,11 @@ exit 0
 %{_unitdir}/%{announcer_filename}.service
 %{_unitdir}/%{announcer_filename}.timer
 
+%files metrics
+%defattr(-,root,root,-)
+%{_datadir}/%{source_dir}/metrics
+%{_datadir}/%{source_dir}/metrics.py
+
 %files repo-checker
 %defattr(-,root,root,-)
 %{_bindir}/osrt-repo_checker

docs/metrics.md

@@ -0,0 +1,55 @@
+# Metrics
+
+Ingest relevant OBS and annotation data to generate insightful metrics.
+
+The overall structure is to loop over all requests to extract point of change
+and calculate metrics. Many of the points of change are treated as deltas,
+meaning add one to this bucket or minus one from another. After all requests are
+ingested the points are walked to evaluate the deltas and recreate the state at
+that point in time.
+
+OBS provides incomplete and inconsistent data which causes odd results like
+negative backlog counts. The ingest tool implements a number of workarounds to
+figure out the correct values when possible, but when not possible the provided
+values are used rather than excluding a fairly large chunk of data. The main
+issues are documented in:
+
+- openSUSE/open-build-service#3857
+- openSUSE/open-build-service#3858
+- openSUSE/open-build-service#3897
+- openSUSE/open-build-service#3898
+
+## Pre-requisites
+
+- InfluxDB instance
+- Grafana instance
+  - import `metrics/grafana` dashboards
+  - create data sources for desired projects
+    setting name and database to the project name (ex. `openSUSE:Factory`)
+
+## Usage
+
+See help information for InfluxDB connection flags.
+
+```
+./metrics -p openSUSE:Factory
+```
+
+Once completed the Grafana dashboard should make pretty graphs.
+
+## Development
+
+Grafana provides an export to JSON option which can be used when the dashboards
+are modified to export them and version control the changes in git. Ensure not
+to unintentionally change the default project, annotation state, or time period
+by saving the dashboard with different defaults.
+
+Use the `--debug` option and inspect individual request XML dumps by looking in
+`~/.cache/osc-plugin-factory-metrics` or:
+
+```
+osc api '/request/$reqid?withfullhistory=1'
+```
+
+When adding new delta based metrics it may be necessary to add key logic in
+`walk_points()` to handle proper grouping for evaluation of deltas.