model: add tangram gradient boosting tree model (#32)

.gitignore

@@ -108,3 +108,5 @@ reckon
 .release-env
 .DS_Store
 cert/
+model/tangram/*.csv
+model/tangram/*.tangram

.goreleaser.yml

@@ -13,10 +13,27 @@ changelog:
     - Merge pull request
     - Merge branch
 builds:
-  - id: linux-armhf
+  #- id: linux-armhf
+  #  goos: [linux]
+  #  goarch: [arm]
+  #  goarm: [7]
+  #  env:
+  #    - CGO_ENABLED=1
+  #    - CC=arm-linux-gnueabihf-gcc
+  #    - CXX=arm-linux-gnueabihf-g++
+  #  flags:
+  #    - -mod=readonly
+  #    - -trimpath
+  #  ldflags:
+  #    - |
+  #      -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+  #      -X main.goVersion={{.Env.GO_VERSION}} -X main.pythonVersion={{.Env.PYTHON_VERSION}}
+  #      -X main.goreleaserVersion={{.Env.GORELEASER_VERSION}}
+  - id: linux-amd64
     goos: [linux]
-    goarch: [arm]
-    goarm: [7]
+    goarch: [amd64]
+    env:
+      - CGO_ENABLED=1
     flags:
       - -mod=readonly
       - -trimpath
@@ -25,9 +42,47 @@ builds:
         -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
         -X main.goVersion={{.Env.GO_VERSION}} -X main.pythonVersion={{.Env.PYTHON_VERSION}}
         -X main.goreleaserVersion={{.Env.GORELEASER_VERSION}}
-  - id: everything-else
-    goos: [linux, darwin]
-    goarch: [amd64, arm64]
+  - id: linux-arm64
+    goos: [linux]
+    goarch: [arm64]
+    env:
+      - CGO_ENABLED=1
+      - CC=aarch64-linux-gnu-gcc
+      - CXX=aarch64-linux-gnu-g++
+    flags:
+      - -mod=readonly
+      - -trimpath
+    ldflags:
+      - |
+        -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+        -X main.goVersion={{.Env.GO_VERSION}} -X main.pythonVersion={{.Env.PYTHON_VERSION}}
+        -X main.goreleaserVersion={{.Env.GORELEASER_VERSION}}
+  - id: darwin-amd64
+    goos: [darwin]
+    goarch: [amd64]
+    env:
+      - CGO_ENABLED=1
+      - PATH=$PATH:/usr/local/osxcross/bin
+      - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/osxcross/lib
+      - CC=o64-clang
+      - CXX=o64-clang++
+    flags:
+      - -mod=readonly
+      - -trimpath
+    ldflags:
+      - |
+        -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+        -X main.goVersion={{.Env.GO_VERSION}} -X main.pythonVersion={{.Env.PYTHON_VERSION}}
+        -X main.goreleaserVersion={{.Env.GORELEASER_VERSION}}
+  - id: darwin-arm64
+    goos: [darwin]
+    goarch: [arm64]
+    env:
+      - CGO_ENABLED=1
+      - PATH=$PATH:/usr/local/osxcross/bin
+      - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/osxcross/lib
+      - CC=oa64-clang
+      - CXX=oa64-clang++
     flags:
       - -mod=readonly
       - -trimpath
@@ -38,13 +93,17 @@ builds:
         -X main.goreleaserVersion={{.Env.GORELEASER_VERSION}}
 archives:
   - builds:
-      - linux-armhf
-      - everything-else
+      #- linux-armhf
+      - linux-amd64
+      - linux-arm64
+      - darwin-amd64
+      - darwin-arm64
     wrap_in_directory: true
     name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
     files:
       - src: LICENSE
       - src: README.md
       - src: model/**/*.py
+      - src: model/tangram/config.json
       - src: model/requirements.txt
       - src: model/README.md

Makefile

@@ -2,12 +2,13 @@ APP := reckon
 PYTHON_VERSION := 3.7.12
 GO_VERSION := 1.16.10
 GORELEASER_VERSION := 0.174.0
+OSX_SDK_VERSION := 11.3
 PORT := 8080:8080
 PROM_CLIENT_URL ?= http://prometheus.rpi.topbass.studio:9090
 PROM_EXPORTER_ADDR ?= :8080
 # comma separated list or inline yaml
 # WATCH_LIST ?= sensehat_temperature,sensehat_humidity
-WATCH_LIST ?= {sensehat_temperature: [Prophet], sensehat_humidity: [Prophet]}
+WATCH_LIST ?= {sensehat_temperature: [Prophet, Tangram], sensehat_humidity: [Prophet, Tangram]}
 SCHEDULE ?= @every 10m
 GRPC_SERVER_ADDRESS ?= localhost:18443
 GRPC_ROOT_CA := $$(cat cert/gRPC_Root_CA.crt)
@@ -142,6 +143,7 @@ release-base:
 		--build-arg "PYTHON_VERSION=$(PYTHON_VERSION)" \
 		--build-arg "GO_VERSION=$(GO_VERSION)" \
 		--build-arg "GORELEASER_VERSION=$(GORELEASER_VERSION)" \
+		--build-arg "OSX_SDK_VERSION=$(OSX_SDK_VERSION)" \
 		-t $(APP)/release \
 		-f release.Dockerfile \
 		.

README.md

@@ -13,19 +13,29 @@ with a defined time range, train a predictive model with those metrics, and then
 metrics back through a prometheus HTTP endpoint.
 
 An exmaple of original and forecasted metrics generated from [Prophet](https://facebook.github.io/prophet/) and
-exposed from reckon:
+[Tangram](https://www.tangram.dev/) gradient boosting model and then exposed from reckon:
 
 ```
 # HELP sensehat_humidity_prophet Prophet forecasted metric value
 # TYPE sensehat_humidity_prophet gauge
-sensehat_humidity_prophet{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 60.491915201576944
-sensehat_humidity_prophet{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 59.233345022648194
-sensehat_humidity_prophet{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 61.69595781236965
+sensehat_humidity_prophet{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 58.246349893997056
+sensehat_humidity_prophet{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 57.4450973695544
+sensehat_humidity_prophet{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 59.08650789641951
 # HELP sensehat_temperature_prophet Prophet forecasted metric value
 # TYPE sensehat_temperature_prophet gauge
-sensehat_temperature_prophet{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 26.479665209525724
-sensehat_temperature_prophet{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 26.17373164147707
-sensehat_temperature_prophet{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 26.767488257211966
+sensehat_temperature_prophet{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 22.196715588009724
+sensehat_temperature_prophet{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 21.83335916260913
+sensehat_temperature_prophet{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 22.551863453282152
+# HELP sensehat_humidity_tangram Tangram forecasted metric value
+# TYPE sensehat_humidity_tangram gauge
+sensehat_humidity_tangram{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 58.43156051635742
+sensehat_humidity_tangram{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 54.96922722581767
+sensehat_humidity_tangram{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 61.893893806897175
+# HELP sensehat_temperature_tangram Tangram forecasted metric value
+# TYPE sensehat_temperature_tangram gauge
+sensehat_temperature_tangram{column="yhat",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 21.94704246520996
+sensehat_temperature_tangram{column="yhat_lower",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 20.179807771933568
+sensehat_temperature_tangram{column="yhat_upper",instance="sensehat.rpi.topbass.studio:8000",job="sensehat_exporter"} 23.714277158486354
 ```
 
 Each metric configured in reckon will have its forecasted metric generated and exported in 3 dimensions (labels):
@@ -43,7 +53,7 @@ Gather the following info before start:
 
 - Prometheus server address, for example, `http://prometheus.rpi.topbass.studio:9090`
 - Metric names to watch and models for each metric, accepts inline YAML or comma separated list,
-  for example, `{sensehat_temperature: [Prophet], sensehat_humidity: [Prophet]}` or
+  for example, `{sensehat_temperature: [Prophet, Tangram], sensehat_humidity: [Prophet, Tangram]}` or
   `sensehat_temperature,sensehat_humidity`
 
 #### With Docker
@@ -104,6 +114,17 @@ Reckon can be configured with the following environment variables:
 | `GRPC_SERVER_GRACE_PERIOD_IN_SECS` | Yes         | `2`                 | gRPC server - in shutdown grace period, server only sends SIGTERM to attempt shutting down subprocesses |
 | `GRPC_SERVER_KILL_PERIOD_IN_SECS`  | Yes         | `5`                 | gRPC server - shutdown kill period, server force kill any subprocesses that are still alive             |
 
+
+`WATCH_LIST` accepts either inline YAML or comma separated list, for example:
+- `{sensehat_temperature: [Prophet, Tangram], sensehat_humidity: [Prophet, Tangram]}`
+- `sensehat_temperature,sensehat_humidity`
+  - When using comma separated list, `MODELS` becomes a required field
+
+Available models:
+- [Prophet](https://facebook.github.io/prophet/)
+- [Tangram](https://www.tangram.dev/) with gradient boosting tree model
+- Future plans: add ARIMA and LSTM models
+
 ## Use it
 
 - Find [the latest release](https://github.com/waltzofpearls/reckon/releases) and downlaod a `.tar.gz` archive
@@ -136,3 +157,11 @@ In addition to forecasted metrics, reckon also exposes runtime metrics to help m
 | `reckon_data_scraped_values`                   | Gauge   | number of the last scraped data points                           |
 | `reckon_forecast_data_values`                  | Gauge   | number of the existing data points kept in memory                |
 | `reckon_training_data_values`                  | Gauge   | number of data points recently sent to model for training        |
+
+Reckon exposes build info through `reckon_build_info`
+
+```
+# HELP reckon_build_info Information about reckon build.
+# TYPE reckon_build_info gauge
+reckon_build_info{commit="c5159d0375d59b0255069eeec3fe3d3ccfe42207",date="2021-11-24T08:05:34Z",go_version="1.16.10",goreleaser_version="0.174.0",python_version="3.7.12",version="0.5.2"} 1
+```

alpine.Dockerfile

@@ -1,6 +1,7 @@
 ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}-alpine3.13
+
 RUN apk add -U --no-cache \
         curl \
         gcc \
@@ -9,6 +10,7 @@ RUN apk add -U --no-cache \
         musl-dev \
         zlib-dev \
         jpeg-dev
+
 ARG APP
 ARG VERSION
 ARG OS
@@ -19,5 +21,10 @@ RUN curl -L https://github.com/waltzofpearls/${APP}/releases/download/v${VERSION
  && pip install --no-cache-dir -r ./model/requirements.txt
 RUN pip install pystan==2.19.1.1
 RUN pip install prophet==1.0.1
+
+RUN curl -fsSL https://pkgs.tangram.dev/stable/alpine/tangram.rsa | tee /etc/apk/keys/tangram.rsa \
+ && echo "https://pkgs.tangram.dev/stable/alpine" | tee /etc/apk/repositories \
+ && apk add tangram
+
 WORKDIR /reckon
 CMD ["./reckon"]

build.Dockerfile

@@ -2,26 +2,36 @@ ARG PYTHON_VERSION
 ARG GO_VERSION
 
 FROM python:${PYTHON_VERSION}-slim-buster
+
 RUN apt-get update; \
     apt-get install -y --no-install-recommends \
         g++ \
         git \
         curl \
         make \
         pkg-config \
+        gnupg \
         ; \
     rm -rf /var/lib/apt/lists/*
+
 ARG GO_VERSION
 RUN curl -O https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz \
  && tar xvf go${GO_VERSION}.linux-amd64.tar.gz \
  && mv go /usr/local
 ENV GOROOT=/usr/local/go
 ENV PATH=$GOROOT/bin:$PATH
+
 WORKDIR /reckon/
 COPY ./model/requirements.txt ./model/requirements.txt
 RUN pip install --no-cache-dir -r ./model/requirements.txt
 RUN pip install pystan==2.19.1.1
 RUN pip install prophet==1.0.1
+
+RUN curl -fsSL https://pkgs.tangram.dev/stable/debian/buster.gpg | apt-key add - \
+ && curl -fsSL https://pkgs.tangram.dev/stable/debian/buster.list | tee /etc/apt/sources.list.d/tangram.list \
+ && apt-get update \
+ && apt-get install tangram
+
 COPY . .
 RUN make
 CMD ["./reckon"]

debian.Dockerfile

@@ -1,12 +1,15 @@
 ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}-slim-buster
+
 RUN apt-get update; \
     apt-get install -y --no-install-recommends \
         curl \
         g++ \
+        gnupg \
         ; \
     rm -rf /var/lib/apt/lists/*
+
 ARG APP
 ARG VERSION
 ARG OS
@@ -17,5 +20,11 @@ RUN curl -L https://github.com/waltzofpearls/${APP}/releases/download/v${VERSION
  && pip install --no-cache-dir -r ./model/requirements.txt
 RUN pip install pystan==2.19.1.1
 RUN pip install prophet==1.0.1
+
+RUN curl -fsSL https://pkgs.tangram.dev/stable/debian/buster.gpg | apt-key add - \
+ && curl -fsSL https://pkgs.tangram.dev/stable/debian/buster.list | tee /etc/apt/sources.list.d/tangram.list \
+ && apt-get update \
+ && apt-get install tangram
+
 WORKDIR /reckon
 CMD ["./reckon"]

go.mod

@@ -3,24 +3,30 @@ module github.com/waltzofpearls/reckon
 go 1.16
 
 require (
+	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang/mock v1.6.0
+	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
+	github.com/guptarohit/asciigraph v0.5.2 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/common v0.6.0
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.3.0
+	github.com/prometheus/common v0.7.0
 	github.com/robfig/cron/v3 v3.0.0
+	github.com/rocketlaunchr/dataframe-go v0.0.0-20211025052708-a1030444159b
 	github.com/stretchr/testify v1.7.0
+	github.com/tangramdotdev/tangram-go v0.7.0
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0
+	golang.org/x/exp v0.0.0-20211123021643-48cbe7f80d7c // indirect
 	golang.org/x/net v0.0.0-20211116231205-47ca1ff31462 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c // indirect
-	golang.org/x/text v0.3.7 // indirect
+	gonum.org/v1/gonum v0.9.3
 	google.golang.org/genproto v0.0.0-20211116182654-e63d96a377c4 // indirect
 	google.golang.org/grpc v1.42.0
 	google.golang.org/protobuf v1.27.1
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )