inrichten nieuwe prod server

etl/options/.gitignore

@@ -4,3 +4,4 @@ vagrant-ubuntu-trusty-64.args
 SETEST.args
 SEPROD.args
 pdok-sensor-test.args
+pdok-sensor-test-2.args

etl/options/pdok-sensor-test-2.args

@@ -0,0 +1,175 @@
+# Host-specific settings - PDOK Test Server
+
+
+#
+# PostGIS DB: credentials and settings
+#
+
+# postgis docker container IP is mapped by docker via --link option
+pg_host=postgis
+pg_user=docker
+pg_password=docker
+pg_port=5432
+pg_database=gis
+pg_schema=smartem
+pg_schema_rt=smartem_rt
+pg_schema_raw=smartem_raw
+pg_schema_refined=smartem_refined
+pg_schema_extracted=smartem_extracted
+pg_schema_harvest_rivm=smartem_harvest_rivm
+pg_schema_calibrated=smartem_calibrated
+#
+# ETL: Refiner-settings
+#
+# Maximum records to process per ETL-run
+refiner_max_input_records=2000
+
+# Should we stop after one round or continue until all done
+refiner_raw_read_once=True
+
+#
+# ETL: Harvester-settings for Raw Sensor API
+#
+raw_device_url_1=http://whale.citygis.nl/sensors/v1
+raw_device_url_2=https://sensor1.intemo.com/sensors/v1
+intemo_token=e8KeUFOciesnHIJ3M2js
+
+# Maximum time spent in processing
+max_proc_time_secs=660
+max_proc_time_secs_whale=600
+
+# Time between calling the Whale API again
+api_interval_secs=-1
+
+#
+# ETL: Harvester-RIVM-settings
+#
+rivm_sos_base_url=http://inspire.rivm.nl/sos/eaq/api/v1/timeseries/
+
+#
+# ETL: Extractor-settings
+#
+extractor_max_input_records=4000
+
+# Should we stop after one round or continue until all done
+extractor_raw_read_once=True
+
+#
+# Calibration-settings
+#
+calibration_components=carbon_monoxide__air_,nitrogen_dioxide__air_,ozone__air_
+calibration_location_geohash=(geohash_tag='u1hnvkb' or geohash_tag='u1hnvkc' or geohash_tag='u1hnvm3' or geohash_tag='u178hw8' or geohash_tag='u178kt7')
+calibrator_max_input_records=1000000
+
+#
+# SOS Publication and API Server credentials
+#
+sos_host=sos52n
+sos_port=8080
+sos_path=/sos52n/service
+sospublisher_max_input_records=1200
+
+#
+# SensorThings API settings and Server credentials
+#
+#sta_host=sta.smartemission.nl
+#sta_port=8080
+#sta_path=/OGCSensorThings/v1.0
+#sta_user=nijmegen
+#sta_password=c9b6fca5be82
+#stapublisher_max_input_records=10000
+
+
+sta_host=gost
+sta_port=8080
+sta_path=/v1.0
+sta_user=
+sta_password=
+# sta_host=test.smartemission.nl
+# sta_port=80
+# sta_path=/gost/v1.0
+# sta_user=stagost
+# sta_password=c9c3b58abe82
+stapublisher_max_input_records=20000
+
+#
+# InfluxDB: settings and Server credentials
+#
+influx_host=influxdb
+influx_dc1_host=dc1.smartemission.nl
+influx_port=8086
+influx_admin_user=air
+influx_admin_password=airsense4254
+
+# Project Smartemission
+influx_se_database=smartemission
+influx_se_writer=smartuser
+influx_se_writer_password=jose2raw
+influx_se_reader=smart
+influx_se_reader_password=smart
+influx_se_measurement_raw=joseraw
+influx_se_measurement_refined=joserefined
+influx_se_measurement_extract=joseextract
+influx_se_measurement_rivm=rivm
+
+# Project AirSensEUR (RIVM)
+influx_as_database=airsenseur
+influx_as_writer=airuser
+influx_as_writer_password=air2sense
+influx_as_reader=jrc
+influx_as_reader_password=jrc
+influx_as_measurement_geonovum=geonovum
+influx_as_measurement_rivm=rivm
+
+#
+# Grafana Dashboard Viewer
+#
+
+grafana_admin_password=airsense52
+GF_SECURITY_ADMIN_PASSWORD=airsense52
+SE_INFLUX_ADMIN_PASSWORD=airsense4254
+SE_INFLUX_ADMIN_USER=air
+
+#
+# GOST SensorThings API
+#
+
+# * db: gost_db_host, gost_db_database, gost_db_port, gost_db_user, gost_db_password.
+# * mqtt: gost_mqtt_host, gost_mqtt_port
+#  server: gost_server_host, gost_server_port, GOST_SERVER_EXTERNAL_URI, gost_client_content
+GOST_SERVER_EXTERNAL_URI=http://data.smartemission.nl/gost
+GOST_DB_HOST=postgis
+GOST_DB_PORT=5432
+GOST_DB_DATABASE=gis
+GOST_DB_USER=docker
+GOST_DB_PASSWORD=docker
+
+# general host domain name
+se_host=data.smartemission.nl
+se_port=
+
+# gmail for notification
+GMAIL_ACCOUNT=smart.emission.srv@gmail.com
+GMAIL_AUTH_TOKEN=Mon4All2Use@
+
+# SOSEmu settings
+SOSEMU_DB_NAME=gis
+SOSEMU_DB_HOST=postgis
+SOSEMU_DB_PORT=5432
+SOSEMU_DB_SCHEMA=smartem_rt
+SOSEMU_DB_USER=docker
+SOSEMU_DB_PASSWORD=docker
+
+# GeoServer
+DB_HOSTNAME=postgis
+DB_USERNAME=docker
+DB_PASSWD=docker
+GS_USERNAME=admin
+GS_PASSWD=jXsoO9S/TGd9o8ZDWWPc1GKgmzOOlLQNEJujWDDsm+OgpLtiexWEOGX1j9u6Gi88
+
+# 52North SOS Settings
+SOS_SERVICE_URL=http://data.smartemission.nl/sos52n/service
+SOS_DB_SCHEMA=sos52n1
+SOS_USERNAME=admin
+# sos4sensor brypted
+SOS_PASSWD=$2a$10$RfwsGAHuetaMjMP6EACXQOHMEP24PP8LSTmkUTiJaJiiIvGWQr4Ci

platform/bootstrap.sh

@@ -5,66 +5,51 @@
 # NB not required when your system already has Docker and docker-compose!!
 #
 #
-# NB2 On Fiware lab VM: add "127.0.0.1 localhost hostname" to /etc/hosts
-#
+# Only for Ubuntu 16.04 !!!
+# 
 # Just van den Broecke - 2016-2018
 
 # Bring system uptodate
 
 # set time right adn configure timezone and locale
-echo "Europe/Amsterdam" > /etc/timezone
-dpkg-reconfigure -f noninteractive tzdata
+#echo "Europe/Amsterdam" > /etc/timezone
+#dpkg-reconfigure -f noninteractive tzdata
+
+timedatectl set-timezone Europe/Amsterdam
 
 apt-get update
 apt-get -y upgrade
-apt-get install -y apt-transport-https ca-certificates  # usually already installed
-
-# Add keys and extra repos
-# /bin/rm /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/pgdg.list
 
-# Docker see https://docs.docker.com/engine/installation/linux/ubuntulinux/
-apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-
-# Add to repo by putting this line in /etc/apt/sources.list.d/docker.list
-echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" >  /etc/apt/sources.list.d/docker.list
-
-# Need 9.4 version of PG client, not in Ubuntu 14.4, so get from PG Repo
-echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" > /etc/apt/sources.list.d/pgdg.list
-curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+# Docker CE
+# see https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 
+# Add to repo 
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 apt-get update
+apt-cache policy docker-ce
+apt-get install -y docker-ce
 
-# check we get from right repo
-# apt-cache policy docker-engine
-
-# The linux-image-extra package allows you use the aufs storage driver.
-# at popup keep locally installed config option
-apt-get install -y linux-image-extra-$(uname -r)
-
-# If you are installing on Ubuntu 14.04 or 12.04, apparmor is required.
-# You can install it using (usually already installed)
-# apt-get install -y apparmor
+# Check Docker daemon
+systemctl status docker
 
-# install docker engine
-apt-get install -y docker-engine
-
-# Start the docker daemon. Usually already running
-service docker start
-
-# test
-# docker run hello-world
 
 # Utils like Emacs and Postgres client to connect to PG DB
 # https://www.postgresql.org/download/linux/ubuntu/
 # Need 9.4 version of PG client, not in Ubuntu 14.4, so get from PG Repo
-apt-get install -y python-pip libyaml-dev libpython2.7-dev git emacs24-nox apache2-utils apt-show-versions sqlite3 postgresql-client-9.4
+# apt-get install -y python-pip libyaml-dev libpython2.7-dev git emacs24-nox apache2-utils apt-show-versions sqlite3 postgresql-client-9.4
+apt-get install -y git emacs24-nox
+apt-get install -y python-pip libyaml-dev libpython2.7-dev postgresql-client-common
+apt-get install -y  apt-show-versions sqlite3
+
+# apache2-utils??
 
 # Also Docker Compose
 pip install pyyaml
 pip install docker-compose
 
 # Postfix: choose Local System
-apt-get -y install postfix
+# apt-get -y install postfix
 
 # Smart Emission Github
 mkdir -p /opt/geonovum/smartem
@@ -73,4 +58,34 @@ mkdir -p /opt/geonovum/smartem
 
 git clone https://github.com/smartemission/smartemission.git /opt/geonovum/smartem/git
 
+# Mount disk Azure
+https://docs.microsoft.com/en-us/azure/virtual-machines/linux/add-disk
+
 echo "READY: now run ./build.sh and ./install.sh to build and run SE Data Platform"
+
+# OLD STUFF
+# Need 9.4 version of PG client, not in Ubuntu 14.4, so get from PG Repo
+# echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+# curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+
+# apt-get update
+
+# check we get from right repo
+# apt-cache policy docker-engine
+
+# The linux-image-extra package allows you use the aufs storage driver.
+# at popup keep locally installed config option
+# apt-get install -y linux-image-extra-$(uname -r)
+
+# If you are installing on Ubuntu 14.04 or 12.04, apparmor is required.
+# You can install it using (usually already installed)
+# apt-get install -y apparmor
+
+# install docker engine
+# apt-get install -y docker-engine
+
+# Start the docker daemon. Usually already running
+# service docker start
+
+# test
+# docker run hello-world

platform/init-databases.sh

@@ -11,17 +11,12 @@ pushd ../services/postgis
   ./run.sh
 popd
 
-pushd ../services/influxdb
-  ./run.sh
-popd
-
 pushd ../database
   ./db-init-meta.sh
   ./db-init-last.sh
   ./db-init-raw.sh
   ./db-init-refined.sh
   ./db-init-extract.sh
-  ./db-init-influxdb.sh
   ./db-init-harvest-rivm.sh
   ./db-init-calibrate.sh
   ./db-init-gost.sh