#123 : support separate InfluxDB AirSensEUR instance for Data Collect…

…or - step 2

services/grafana/docker-compose.yml

@@ -4,7 +4,7 @@ services:
 
   grafana:
 
-    image: smartemission/se-grafana:5.1.3-2
+    image: smartemission/se-grafana:5.1.3-3
 
     container_name: grafana
 
@@ -14,6 +14,7 @@ services:
 #      - GF_AUTH_ANONYMOUS_ENABLED_ORG_NAME=Main Org.
 #      - GF_AUTH_ANONYMOUS_ENABLED_ORG_ROLE=Viewer
       - SE_INFLUX_URL=http://influxdb:8086
+      - SE_INFLUX_DC1_URL=http://dc1.smartemission.nl:8086
 
     env_file:
       # for other config vars (see Docker def README)

services/influxdb-dc1/README.md

@@ -0,0 +1,107 @@
+# InfluxDB Server - Data Collector 1
+
+This runs the InfluxDB service as a Docker container for an SE Data Collector. See https://www.influxdata.com:
+
+> InfluxDB is an open source database written in Go specifically to handle time 
+> series data with high availability and high performance requirements. 
+> InfluxDB installs in minutes without external dependencies, yet is 
+> flexible and scalable enough for complex deployments.
+
+The Docker image comes from https://hub.docker.com/_/influxdb/
+
+## Use in SE
+
+InfluxDB was/is used in the SE platform initially for storing time-series for calibration ANN learning.
+
+In addition InfluxDB is used within SE Data Collectors. These run external from the main SE platform.
+For example for the EU JRC AirSensEUR AQ Sensor: http://www.airsenseur.org and possibly other attached sensors.
+
+## Docker
+
+Docker is used to deploy/run InfluxDB. See the [run.sh script](run.sh).
+
+The Docker container will expose the InfluxDB port (8086).
+Database initialization: tbs.
+
+Run `/usr/bin/influx` command as follows:
+
+```
+sudo docker run --rm --net=container:influxdb -it influxdb influx -host localhost
+```
+
+## Visualization
+
+InfluxDB is closely integrated with Grafana (http://grafana.org). See the [Grafana Service](../grafana) in the SE platform.
+
+## Authorization
+
+See https://docs.influxdata.com/influxdb/v1.1/query_language/authentication_and_authorization/#authorization
+
+We use two types of users: a global `admin` user and a per-DB user.
+
+We need admin user. All below via `influx` CLI.
+
+```
+sudo docker exec -it influxdb  bash
+influx -username <user> -password <pass>
+```
+
+Creating a database. See https://docs.influxdata.com/influxdb/v1.1/introduction/getting_started/#creating-a-database
+
+```
+CREATE DATABASE airsenseur
+```
+
+Create DB user and give access to DB
+
+```
+CREATE USER <username> WITH PASSWORD '<password>'
+GRANT ALL ON airsenseur TO <username>
+```
+
+## Writing Data
+
+See https://docs.influxdata.com/influxdb/v1.1/guides/writing_data/
+Writing data, for example for raw Josene Sensor data could be:
+
+```
+curl -i -XPOST 'http://localhost:8086/write?db=smartemraw' --data-binary 'joseraw,station=19,component=no2raw value=12345 1434055562000000000'
+```
+
+or multiple measurements, each on a new line:
+
+```
+curl -i -XPOST 'http://localhost:8086/write?db=smartemraw' --data-binary 'joseraw,station=19,component=no2raw value=12345 1434055562000000000
+joseraw,station=23,component=temperature value=18 1434055562000000000
+joseraw,station=19,component=o3raw value=12345 1434055562000000000'
+```
+
+## Query Data
+
+See https://docs.influxdata.com/influxdb/v1.1/guides/querying_data/
+
+To select all `no2` from `measurement` `joseraw`:
+
+```
+SELECT * FROM joseraw WHERE component = 'no2'
+```
+            
+## Delete Data
+
+To delete from `measurement` `joseraw`:
+
+```
+DELETE FROM joseraw WHERE time > '1970-01-01'
+```
+
+## Geospatial Data
+
+While InfluxDB has no specific geospatial support, a common way to tag measurements with lat/lon values is
+via [Geohash](https://en.wikipedia.org/wiki/Geohash). The common Python module is https://github.com/vinsci/geohash.
+Tools like Grafana can interpret geohashes in Dashboards when the tag name is `geohash`.
+
+A line would be like:
+
+```
+joseraw,station=70,component=pressure,geohash=u1hhc432rtwb value=1006 1476435600000000000
+```

services/influxdb-dc1/backup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# See https://www.influxdata.com/blog/backuprestore-of-influxdb-fromto-docker-containers/
+
+source influxdb.env
+
+BACKUP_DIR="${SE_BACKUP_DIR}/${SE_CONTAINER_NAME}"
+mkdir -p ${BACKUP_DIR}
+rm -rf ${BACKUP_DIR}/*
+TARGET_DUMP_FILE=${SE_BACKUP_DIR}/influxdb_${INFLUXDB_DB}_data.tar.gz
+
+# On RUNNING container named influxdb
+docker exec ${SE_CONTAINER_NAME} influxd backup -database ${INFLUXDB_DB} /backup
+
+pushd ${SE_BACKUP_DIR}
+	tar -cvzf ${TARGET_DUMP_FILE} ${SE_CONTAINER_NAME}
+popd

services/influxdb-dc1/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.1'
+
+services:
+
+  influxdb-dc1:
+
+    # image: influxdb:1.4.2
+    image: influxdb:1.5.3
+
+    # Data Collector 1
+    container_name: influxdb-dc1
+
+    env_file:
+      # Should set all influx_* DB-related vars
+      - influxdb.env
+
+    environment:
+      - INFLUXDB_DATA_INDEX_VERSION=tsi1
+      - INFLUXDB_HTTP_AUTH_ENABLED=true
+
+    labels:
+      - "traefik.backend=influxdb-dc1"
+      - "traefik.enable=true"
+      - "traefik.frontend.priority=5000"
+      - "traefik.frontend.rule=PathPrefixStrip:/influxdb-dc1"
+      - "traefik.docker.network=se_back"
+
+    expose:
+     - "8086"
+
+    ports:
+      - "8086:8086"
+
+    networks:
+      # Visible in SE backend and frontend Docker network
+      - se_back
+
+    volumes:
+      - /var/smartem/data/influxdb-dc1:/var/lib/influxdb
+      - /var/smartem/backup/influxdb-dc1:/backup
+
+networks:
+  se_back:
+    external: true

services/influxdb-dc1/query.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Fire query on influxdb local instance.
+# usage: ./query.sh host db 'my query'
+# examples:
+# ./query.sh test.smartemission.nl airsenseur "SELECT sampleRawVal FROM JustObjects1"
+# ./query.sh test.smartemission.nl airsenseur "SELECT sampleRawVal FROM JustObjects1 WHERE \"name\" = 'NO2-B43F'"
+# ./query.sh test.smartemission.nl airsenseur "SHOW SERIES FROM Geonovum1"
+# ./query.sh test.smartemission.nl airsenseur 'DROP MEASUREMENT JustObjects1'
+
+source influxdb.env
+
+INFLUX_HOST=$1
+INFLUX_DB=$2
+
+# Shorthand func to call InfluxDB REST Query API
+function query() {
+  QUERY="${1}"
+  echo "host: ${INFLUX_HOST} db=${INFLUX_DB} query: ${QUERY}"
+
+  # replace all blanks http://stackoverflow.com/questions/5928156/replace-a-space-with-a-period-in-bash
+  # Basically urlencoding.
+  # QUERY=${QUERY// /%20}
+
+  # Invoke curl to local InfluxDB instance to execute the query/command   OLD
+#  curl \
+#   -u ${influx_admin_user}:${influx_admin_password} \
+#   -H "Content-Type:text/plain" \
+#   -XPOST http://${INFLUX_HOST}:8086/query?q=${QUERY}
+
+
+	# example curl
+	# curl -G 'http://localhost:8086/query?pretty=true' --data-urlencode "db=mydb" --data-urlencode "q=SELECT value FROM cpu_load_short WHERE region='us-west';SELECT count(value) FROM cpu_load_short WHERE region='us-west'"
+
+   curl  -u ${influx_admin_user}:${influx_admin_password} \
+     -GET "http://${INFLUX_HOST}:8086/query?pretty=true" \
+     --data-urlencode "db=${INFLUX_DB}" \
+     --data-urlencode "q=${QUERY}"
+}
+
+#
+
+query "$3"

services/influxdb-dc1/restore.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# See https://www.influxdata.com/blog/backuprestore-of-influxdb-fromto-docker-containers/
+# NB need to stop monitoring as influxdb will not stop because of cAdvisor lock!!!
+
+if [ $# -eq 0 ]
+then
+    echo "Geef DB InfluxDB dump bestand bestand op als parameter."
+    exit 1
+fi
+
+DUMP_FILE=$1
+
+if [ ! -f ${DUMP_FILE} ]
+then
+    echo "Bestand ${DUMP_FILE} niet gevonden."
+    exit 1
+fi
+
+# Parameter ok
+./stop.sh
+
+source influxdb.env
+
+DATA_DIR="${SE_DATA_DIR}/${SE_CONTAINER_NAME}"
+BACKUP_DIR="${SE_BACKUP_DIR}/${SE_CONTAINER_NAME}"
+IMAGE="influxdb:1.5.3"
+
+
+# NB possibly best to make InfluxDB empty (db-init-influxdb.sh script)!
+# otherwise this issue: https://github.com/influxdata/influxdb/issues/8320
+# and restore just one DB...
+rm -rf ${DATA_DIR}
+mkdir -p ${DATA_DIR}
+
+rm -rf ${BACKUP_DIR}
+mkdir -p ${BACKUP_DIR}
+
+pushd ${SE_BACKUP_DIR}
+	tar xzvf ${DUMP_FILE}
+popd
+
+
+# On STOPPED container named influxdb
+docker run --rm \
+  --entrypoint /bin/bash \
+  -v ${DATA_DIR}:/var/lib/influxdb \
+  -v ${BACKUP_DIR}:/backup \
+  ${IMAGE} \
+  -c "influxd restore -metadir /var/lib/influxdb/meta -datadir /var/lib/influxdb/data -database ${INFLUXDB_DB} /backup"
+
+# ./run.sh

services/influxdb-dc1/run.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Run the InfluxDB from https://hub.docker.com/_/influxdb/.
+#
+
+#LOG_DIR="/var/smartem/log/influxdb"
+#DATA_DIR="/var/smartem/data/influxdb"
+#BACKUP_DIR="/var/smartem/backup/influxdb"
+#NAME="influxdb"
+#IMAGE="influxdb:1.4.2"
+#
+#sudo mkdir -p ${DATA_DIR}
+#sudo mkdir -p ${LOG_DIR}
+#sudo mkdir -p ${BACKUP_DIR}
+#
+#SCRIPT_DIR=${0%/*}
+#pushd ${SCRIPT_DIR}
+#SCRIPT_DIR=$PWD
+#popd
+#
+#VOL_MAP="-v ${DATA_DIR}:/var/lib/influxdb -v ${LOG_DIR}:/var/log/influxdb -v ${SCRIPT_DIR}/config/influxdb.conf:/etc/influxdb/influxdb.conf:ro -v ${BACKUP_DIR}:/backup"
+#
+## 8083 is admin, 8086 API NB: admin 8083 was removed in InfluxDB 1.3
+#PORT_MAP="-p 8086:8086"
+#
+## Stop and remove possibly old containers
+#sudo docker stop ${NAME} > /dev/null 2>&1
+#sudo docker rm ${NAME} > /dev/null 2>&1
+#
+## Finally run, keeping DB-data, config and logs on the host
+#sudo docker run --name ${NAME} --network="se_back" ${PORT_MAP} ${VOL_MAP} -d -t ${IMAGE} -config /etc/influxdb/influxdb.conf
+
+# Need HOSTNAME within docker-compose for host-specific path to env file.
+mkdir -p /var/smartem/data/influxdb-dc1
+mkdir -p /var/smartem/backup/influxdb-dc1
+
+docker-compose rm --force --stop
+docker-compose up -d
+
+# generate conf: docker run --rm influxdb influxd config > influxdb.conf
+# create db: curl -G http://localhost:8086/query --data-urlencode "q=CREATE DATABASE mydb"

services/influxdb-dc1/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+#
+docker-compose rm --force --stop
+
+

services/influxdb-dc1/test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Test DB for general health.
+#
+
+. influxdb.env
+
+# Shorthand func to call InfluxDB REST Query API
+function query() {
+  DB=$1
+  QUERY="$2"
+  echo "db=$DB - query: ${QUERY}"
+  # See https://docs.influxdata.com/influxdb/v1.4/guides/querying_data/
+  curl -G 'http://localhost:8086/query?pretty=true' \
+  -u ${INFLUXDB_ADMIN_USER}:${INFLUXDB_ADMIN_PASSWORD} \
+     --data-urlencode "db=${DB}" --data-urlencode \
+     "q=${QUERY}"
+}
+
+query _internal "SHOW GRANTS FOR ${INFLUXDB_READ_USER}"
+query _internal "SHOW GRANTS FOR ${INFLUXDB_WRITE_USER}"
+
+query _internal "SHOW DATABASES"
+query _internal "SHOW USERS"
+
+query ${INFLUXDB_DB} "SELECT * from Geonovum1 limit 2"
+
+# examples
+# http://test.smartemission.nl:8086/query?db=smartemission&q=SELECT%20*%20from%20rivm%20limit%202
+# http://local.smartemission.nl/influxdb/query?db=smartemission&q=SELECT%20*%20from%20rivm%20limit%202

services/influxdb/backup.sh

@@ -1,17 +1,16 @@
 #!/bin/bash
 # See https://www.influxdata.com/blog/backuprestore-of-influxdb-fromto-docker-containers/
 
-NAME="influxdb"
-BACKUP_INFLUXDB_DIR="/var/smartem/backup/influxdb"
+source influxdb.env
 
-DBS="airsenseur smartemission"
-rm -rf ${BACKUP_INFLUXDB_DIR}/*
-for DB in ${DBS}
-do
-	# On RUNNING container named influxdb
-	docker exec ${NAME} influxd backup -database ${DB} /backup
-done
+BACKUP_DIR="${SE_BACKUP_DIR}/${SE_CONTAINER_NAME}"
+mkdir -p ${BACKUP_DIR}
+rm -rf ${BACKUP_DIR}/*
+TARGET_DUMP_FILE=${SE_BACKUP_DIR}/influxdb_${INFLUXDB_DB}_data.tar.gz
 
-pushd ${BACKUP_INFLUXDB_DIR}/..
-	tar -cvzf influxdb_data_dump.tar.gz influxdb
+# On RUNNING container named influxdb
+docker exec ${SE_CONTAINER_NAME} influxd backup -database ${INFLUXDB_DB} /backup
+
+pushd ${SE_BACKUP_DIR}
+	tar -cvzf ${TARGET_DUMP_FILE} ${SE_CONTAINER_NAME}
 popd