refactor: added api again

etl/rawsensorlastinput.py

@@ -248,6 +248,8 @@ def format_data(self, data):
 
         # Parse JSON from data string fetched by base method read()
         json_obj = self.parse_json_str(data)
+        if 'p_unitserialnumber' not in json_obj:
+            return []
 
         # Base data for all records
         base_record = {}

services/web/api/sosrest/config.py

@@ -0,0 +1,9 @@
+# local config, change on server for real config
+config = {
+    'database': 'gis',
+    'host': 'postgis',
+    'port': '5432',
+    'schema': 'smartem_rt',
+    'user': 'docker',
+    'password': 'docker'
+}

services/web/api/sosrest/index.py

@@ -0,0 +1,122 @@
+import os, sys
+from functools import wraps, update_wrapper
+from datetime import datetime
+from flask import Flask, render_template, request, make_response
+
+if __name__ != '__main__':
+    # When run with WSGI in Apache we need to extend the PYTHONPATH to find Python modules relative to index.py
+    sys.path.insert(0, os.path.dirname(__file__))
+
+from postgis import PostGIS
+from config import config
+
+app = Flask(__name__)
+app.debug = True
+application = app
+
+
+# Wrapper to disable any kind of caching for all pages
+# See http://arusahni.net/blog/2014/03/flask-nocache.html
+def nocache(view):
+    @wraps(view)
+    def no_cache(*args, **kwargs):
+        response = make_response(view(*args, **kwargs))
+        response.headers['Last-Modified'] = datetime.now()
+        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
+        response.headers['Pragma'] = 'no-cache'
+        response.headers['Expires'] = '-1'
+        return response
+
+    return update_wrapper(no_cache, view)
+
+# Shorthand to get stations array from DB
+def get_stations():
+    # Do query from DB
+    db = PostGIS(config)
+    return db.do_query('SELECT * from stations', 'stations')
+
+# Shorthand to get (last values) array from DB
+def get_last_values(station):
+    # Do query from DB
+    db = PostGIS(config)
+
+    # Default is to get all last measurements
+    query = 'SELECT * from v_last_measurements'
+    if station:
+        # Last measurements for single station
+        query = query + ' WHERE device_id = ' + station
+    return db.do_query(query, 'v_last_measurements')
+
+# Shorthand to create proper JSON HTTP response
+def make_json_response(json_doc):
+    response = make_response(json_doc)
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+# Shorthand to create proper JSONP HTTP response
+def make_jsonp_response(json_doc, callback):
+    # TODO: make smart wrapper: http://flask.pocoo.org/snippets/79/
+    json_doc = str(callback) + '(' + json_doc + ')'
+    response = make_response(json_doc)
+    response.headers["Content-Type"] = "application/javascript"
+    return response
+
+
+# Home page
+@app.route('/')
+@nocache
+def home():
+    return render_template('home.html')
+
+
+# Get list of all stations with locations (as JSON or HTML)
+@app.route('/api/v1/stations')
+@nocache
+def stations():
+    # Fetch stations from DB
+    stations_list = get_stations()
+
+    # Determine response format: JSON (default) or HTML
+    format = request.args.get('format', 'json')
+    if format == 'html':
+        return render_template('stations.html', stations=stations_list)
+    else:
+        # Construct JSON response: JSON doc via Jinja2 template with JSON content type
+        json_doc = render_template('stations.json', stations=stations_list)
+
+        # To enable X-domain: JSONP with callback
+        # TODO: make smart wrapper: http://flask.pocoo.org/snippets/79/
+        jsonp_cb = request.args.get('callback', False)
+        if jsonp_cb:
+            return make_jsonp_response(json_doc, jsonp_cb)
+        else:
+            return make_json_response(json_doc)
+
+
+# Get last values for single station  (as JSON or HTML)
+# Example: /api/v1/timeseries?station=23&expanded=true
+@app.route('/api/v1/timeseries')
+@nocache
+def timeseries(package=None):
+    # Get last values, all or for single station if 'station=' arg in query string
+    last_values = get_last_values(request.args.get('station', None))
+
+    # Determine response format: JSON (default) or HTML
+    format = request.args.get('format', 'json')
+    if format == 'html':
+        return render_template('timeseries.html', last_values=last_values)
+    else:
+        # Construct JSON response: JSON doc via Jinja2 template with JSON content type
+        json_doc = render_template('timeseries.json', last_values=last_values)
+
+        # To enable X-domain: JSONP with callback
+        # TODO: make smart wrapper: http://flask.pocoo.org/snippets/79/
+        jsonp_cb = request.args.get('callback', False)
+        if jsonp_cb:
+            return make_jsonp_response(json_doc, jsonp_cb)
+        else:
+            return make_json_response(json_doc)
+
+if __name__ == '__main__':
+    # Run as main via python index.py
+    app.run(debug=True, host='0.0.0.0')

services/web/api/sosrest/postgis.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+#
+# PostGIS support wrapper.
+#
+# Author: Just van den Broecke
+#
+from util import get_log
+
+log = get_log("postgis")
+
+try:
+    import psycopg2
+    import psycopg2.extensions
+except ImportError:
+    log.error("cannot find package psycopg2 for Postgres client support, please install psycopg2 first!")
+    # sys.exit(-1)
+
+
+class PostGIS:
+    def __init__(self, config):
+        # Lees de configuratie
+        self.config = config
+
+    def connect(self):
+        try:
+            conn_str = "dbname=%s user=%s host=%s port=%s" % (self.config['database'],
+                                                              self.config['user'],
+                                                              self.config.get('host', 'localhost'),
+                                                              self.config.get('port', '5432'))
+            log.info('Connecting to %s' % conn_str)
+            conn_str += ' password=%s' % self.config['password']
+            self.connection = psycopg2.connect(conn_str)
+            self.cursor = self.connection.cursor()
+
+            self.set_schema()
+            log.debug("Connected to database %s" % (self.config['database']))
+        except Exception, e:
+            log.error("Cannot connect to database '%s'" % (self.config['database']))
+            raise
+
+    def disconnect(self):
+        self.e = None
+        try:
+            self.connection.close()
+        except (Exception), e:
+            self.e = e
+            log.error("error %s in close" % (str(e)))
+
+        return self.e
+
+    # Do the whole thing: connecting, query, and conversion of result to array of dicts (records)
+    def do_query(self, query_str, table):
+        self.connect()
+
+        column_names = self.get_column_names(table, self.config.get('schema'))
+        # print('cols=' + str(column_names))
+
+        self.execute(query_str)
+        records_vals = self.cursor.fetchall()
+
+        # record is Python list of Python dict (multiple records)
+        records = list()
+
+        # Convert list of lists to list of dict using column_names
+        for col_vals in records_vals:
+            records.append(dict(zip(column_names, col_vals)))
+        # print('stations=' + str(records))
+        self.disconnect()
+        return records
+
+    def get_column_names(self, table, schema='public'):
+        self.cursor.execute("select column_name from information_schema.columns where table_schema = '%s' and table_name='%s'" % (schema, table))
+        column_names = [row[0] for row in self.cursor]
+        return column_names
+
+    def set_schema(self):
+        # Non-public schema set search path
+        if self.config['schema'] != 'public':
+            # Always set search path to our schema
+            self.execute('SET search_path TO %s,public' % self.config['schema'])
+            self.connection.commit()
+
+    def execute(self, sql, parameters=None):
+        try:
+            if parameters:
+                self.cursor.execute(sql, parameters)
+            else:
+                self.cursor.execute(sql)
+
+                # log.debug(self.cursor.statusmessage)
+        except (Exception), e:
+            log.error("error %s in query: %s with params: %s" % (str(e), str(sql), str(parameters)))
+            #            self.log_actie("uitvoeren_db", "n.v.t", "fout=%s" % str(e), True)
+            return -1
+
+        return self.cursor.rowcount

services/web/api/sosrest/run-local.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+# Run local on port 5000
+
+echo "Running SOS REST API locally: point your browser to http://127.0.0.1:5000"
+python index.py

services/web/api/sosrest/static/style.css

@@ -0,0 +1,22 @@
+/*body {*/
+    /*font-family: sans-serif;*/
+    /*background: #eee;*/
+/*}*/
+
+/*h1, h2 {*/
+    /*color: #377BA8;*/
+/*}*/
+
+/*a {*/
+    /*color: #0000dd;*/
+/*}*/
+
+.warn {
+    color: #CC0000;
+    font-weight: bold
+}
+
+/*h2 {*/
+    /*font-size: 1.2em;*/
+/*}*/
+

services/web/api/sosrest/syncprod.sh

@@ -0,0 +1,3 @@
+#/bin/bash
+
+rsync -e ssh -alzvx --exclude "*.pyc" --exclude "config.py" ./ sadmin@api.smartemission.nl:/var/www/api.smartemission.nl/sosemu/

services/web/api/sosrest/templates/home.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+{% include 'page-head.html' %}
+<body>
+<div class="container">
+    <h1>SOSRest API</h1>
+
+    <p>Dit is de SOS REST API Emulator voor SmartEmission.<br/>
+        Deze serveert de laatste sensor-metingen uit de Raw Sensor API via een
+        geëmuleerde <a href="http://sensorweb.demo.52north.org/sensorwebclient-webapp-stable/api-doc/index.html">52N SOS REST API</a>. </p>
+    <h2>Specificatie URLs</h2>
+    <p>
+        Twee services zijn aktief, deze geven JSON-responses terug:
+    </p>
+    <ul>
+        <li>Stations: <a href="{{ url_for('stations') }}">{{ url_for('stations') }} </a></li>
+        <li>Laatste metingen per station (bijv station 23): <a
+                href="{{ url_for('timeseries') }}?station=23&expanded=true">{{ url_for('timeseries') }}?station=23&expanded=true</a>
+        </li>
+    </ul>
+    <h2>HTML Rendering</h2>
+    <p>
+        Deze API kan ook met HTML responses ipv JSON bevraagd worden, via de optionele <code>format=html</code>
+        query parameter. Hieronder de voorbeelden van hierboven als HTML.
+    </p>
+    <ul>
+        <li>Stations: <a href="{{ url_for('stations') }}?format=html">{{ url_for('stations') }}?format=html</a></li>
+        <li>Laatste metingen per station (bijv station 23): <a
+                href="{{ url_for('timeseries') }}?station=23&format=html">{{ url_for('timeseries') }}?station=23&format=html</a>
+        </li>
+    </ul>
+    <h2>Web Applicatie</h2>
+    <p>
+        Deze web applicatie is geschreven in Python met het lichtgewicht <a href="http://flask.pocoo.org/">Flask web-framework</a>. Zie ook de
+        <a href="https://github.com/Geonovum/sospilot/tree/master/src/smartem/sosrest/webapp">webapp code in GitHub</a>.
+    </p>
+
+    <h2>JSONP Support</h2>
+    <p>
+        JSONP support is via the <strong>callback</strong> parameter, for example:
+        <a href="http://api.smartemission.nl/sosemu/api/v1/stations?callback=mycallback">http://api.smartemission.nl/sosemu/api/v1/stations?callback=mycallback</a>
+    </p>
+
+    <h2>Voorbeeld</h2>
+    <p>
+        Hier voorbeelden van web clients:
+    </p>
+    <ul>
+    <li>
+        <a href="http://rawgit.com/Geonovum/smartemission/master/specs/sosrest-api/examples/leaflet.html">Leaflet Voorbeeld (gebruikt JSONP)</a>
+        en de <a href="https://github.com/Geonovum/smartemission/blob/master/specs/sosrest-api/examples/leaflet.html">code uit GitHub</a>
+    </li>
+</ul>
+
+
+
+</div>
+
+</body>
+</html>

services/web/api/sosrest/templates/page-head.html

@@ -0,0 +1,9 @@
+<head>
+    <title>Smart Emission SOS REST API</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="{{ url_for('static', filename='bootstrap/3.3.5/bootstrap.min.css') }}">
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+    <script src="{{ url_for('static', filename='jquery/1.11.3/jquery.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='bootstrap/3.3.5/bootstrap.min.js') }}"></script>
+</head>

services/web/api/sosrest/templates/stations.html

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html lang="en">
+{% include 'page-head.html' %}
+<body>
+<div class="container">
+    <h1>Stations</h1>
+    <p>
+        Hieronder alle stations met tijd van laatste update. Klik op de Observaties om deze te zien.
+    </p>
+    <table class="table table-bordered table-condensed table-responsive table-striped">
+        <thead>
+        <tr>
+            <th>Station Id</th>
+            <th>Station Naam</th>
+            <th>Laatste Update</th>
+            <th>Inactief ?</th>
+            <th>Laatste Observaties</th>
+        </tr>
+        </thead>
+        {% for station in stations %}
+            <tr>
+                <td>
+                    {{ station['device_id'] }}
+                </td>
+                <td>
+                    {{ station['device_name'] }}
+                </td>
+                <td>
+                    {{ station['last_update'] }}
+                </td>
+                <td>
+                    {{ station['value_stale'] }}
+                </td>
+                <td>
+                    <a href="{{ url_for('timeseries') }}?station={{ station['device_id'] }}&format=html">Observaties &gt;&gt;</a>
+                </td>
+            </tr>
+        {% endfor %}
+    </table>
+    <p>
+    <a href="{{ url_for('home') }}">Terug naar thuis pagina &gt;&gt;</a>
+    </p>
+</div>
+
+</body>
+</html>