refine O3 calibration

docs/platform/data.rst

@@ -21,6 +21,77 @@ measured values: temperature, relative humidity, pressure and even the concentra
 other gasses! For example O3 and NO2. A great deal of scientific literature is already devoted
 to the sensor calibration issue.
 
+The units are: ::
+
+	S.TemperatureUnit		milliKelvin
+	S.TemperatureAmbient	milliKelvin
+	S.Humidity				%mRH
+	S.LightsensorTop		Lux
+	S.LightsensorBottom		Lux
+	S.Barometer				Pascal
+	S.Altimeter				Meter
+	S.CO					ppb
+	S.NO2					ppb
+	S.AcceleroX				2 ~ +2G (0x200 = midscale)
+	S.AcceleroY				2 ~ +2G (0x200 = midscale)
+	S.AcceleroZ				2 ~ +2G (0x200 = midscale)
+	S.LightsensorRed		Lux
+	S.LightsensorGreen		Lux
+	S.LightsensorBlue		Lux
+	S.RGBColor				8 bit R, 8 bit G, 8 bit B
+	S.BottomSwitches		?
+	S.O3					ppb
+	S.CO2					ppb
+	v3: S.ExternalTemp		milliKelvin
+	v3: S.COResistance		Ohm
+	v3: S.No2Resistance		Ohm
+	v3: S.O3Resistance		Ohm
+	S.AudioMinus5			Octave -5 in dB(A)
+	S.AudioMinus4			Octave -4 in dB(A)
+	S.AudioMinus3			Octave -3 in dB(A)
+	S.AudioMinus2			Octave -2 in dB(A)
+	S.AudioMinus1			Octave -1 in dB(A)
+	S.Audio0				Octave 0 in dB(A)
+	S.AudioPlus1			Octave +1 in dB(A)
+	S.AudioPlus2			Octave +2 in dB(A)
+	S.AudioPlus3			Octave +3 in dB(A)
+	S.AudioPlus4			Octave +4 in dB(A)
+	S.AudioPlus5			Octave +5 in dB(A)
+	S.AudioPlus6			Octave +6 in dB(A)
+	S.AudioPlus7			Octave +7 in dB(A)
+	S.AudioPlus8			Octave +8 in dB(A)
+	S.AudioPlus9			Octave +9 in dB(A)
+	S.AudioPlus10			Octave +10 in dB(A)
+	S.SatInfo
+	S.Latitude				nibbles: n1:0=East/North, 8=West/South; n2&n3: whole degrees (0-180); n4-n8: degree fraction (max 999999)
+	S.Longitude				nibbles: n1:0=East/North, 8=West/South; n2&n3: whole degrees (0-180); n4-n8: degree fraction (max 999999)
+
+	P.Powerstate					Power State
+	P.BatteryVoltage				Battery Voltage (milliVolts)
+	P.BatteryTemperature			Battery Temperature (milliKelvin)
+	P.BatteryGauge					Get Battery Gauge, BFFF = Battery full, 1FFF = Battery fail, 0000 = No Battery Installed
+	P.MuxStatus						Mux Status (0-7=channel,F=inhibited)
+	P.ErrorStatus					Error Status (0=OK)
+	P.BaseTimer						BaseTimer (seconds)
+	P.SessionUptime					Session Uptime (seconds)
+	P.TotalUptime					Total Uptime (minutes)
+	v3: P.COHeaterMode				CO heater mode
+	P.COHeater						Powerstate CO heater (0/1)
+	P.NO2Heater						Powerstate NO2 heater (0/1)
+	P.O3Heater						Powerstate O3 heater (0/1)
+	v3: P.CO2Heater					Powerstate CO2 heater (0/1)
+	P.UnitSerialnumber				Serialnumber of unit
+	P.TemporarilyEnableDebugLeds	Debug leds (0/1)
+	P.TemporarilyEnableBaseTimer	Enable BaseTime (0/1)
+	P.ControllerReset				WIFI reset
+	P.FirmwareUpdate				Firmware update, reboot to bootloader
+
+	Unknown at this moment (decimal):
+	P.11
+	P.16
+	P.17
+	P.18
+
 Below are typical values as obtained via the raw sensor API ::
 
 	# General
@@ -117,6 +188,7 @@ Below are typical values as obtained via the raw sensor API ::
 	p_18: 167772549,
 	p_17: 167772549,
 
+
 Below each of these sensor values are elaborated. All conversions are implemented in this Python script and
 used in various ETL-chains:
 https://github.com/Geonovum/smartemission/blob/master/etl/sensorconverters.py
@@ -134,7 +206,7 @@ https://github.com/pietermarsman/smartemission_calibration .
 By using the R-language, reports in PDF are generated.
 
 O3 Calibration
---------------
+~~~~~~~~~~~~~~
 
 O3 seems to be the most linear. See the resulting `O3 PDF report <_static/calibration/O3.pdf>`_.
 
@@ -152,5 +224,110 @@ From the linear model comes the following formula for the conversion from resist
 	+ 0.0699428 * s.humidity
 	+ 0.008435412 * s.temperature.unit * sqrt(s.no2resistance)
 
+GPS Data
+--------
+
+See https://github.com/Geonovum/sospilot/issues/22
+
+Example: ::
+
+	07/24/2015 07:27:36,S.Longitude,5914103
+	07/24/2015 07:27:36,S.Latitude,53949937
+	wordt
+
+	Longitude: 5914103 --> 0x005a3df7
+	0x05 --> 5 graden (n2 en n3),
+	0xa3df7 --> 671223 (n4-n8) fractie --> 0.671223
+	dus 5.671223 graden
+
+	Latitude: 53949937 --> 0x033735f1
+	0x33 --> 51 graden
+	0x735f1 --> 472561 --> 0.472561
+	dus 51.472561
+	n0=0 klopt met East/North.
+	5.671223, 51.472561
+
+	komt precies uit in de Marshallstraat in Helmond bij Intemo, dus alles lijkt te kloppen!!
+
+	In TypeScript:
+
+	/*
+	        8 nibbles:
+	        MSB                  LSB
+	        n1 n2 n3 n4 n5 n6 n7 n8
+	        n1: 0 of 8, 0=East/North, 8=West/South
+	        n2 en n3: whole degrees (0-180)
+	        n4-n8: fraction of degrees (max 999999)
+	*/
+	private convert(input: number): number {
+	 var sign = input >> 28 ? -1 : +1;
+	 var deg = (input >> 20) & 255;
+	 var dec = input & 1048575;
+
+	 return (deg + dec / 1000000) * sign;
+	}
+
+In Python: ::
+
+	# Lat or longitude conversion
+	# 8 nibbles:
+	# MSB                  LSB
+	# n1 n2 n3 n4 n5 n6 n7 n8
+	# n1: 0 of 8, 0=East/North, 8=West/South
+	# n2 en n3: whole degrees (0-180)
+	# n4-n8: fraction of degrees (max 999999)
+	def convert_coord(input, json_obj, name):
+	    sign = 1.0
+	    if input >> 28:
+	        sign = -1.0
+	    deg = float((input >> 20) & 255)
+	    dec = float(input & 1048575)
+
+	    result = (deg + dec / 1000000.0) * sign
+	    if result == 0.0:
+	        result = None
+	    return result
+
+	def convert_latitude(input, json_obj, name):
+	    res = convert_coord(input, json_obj, name)
+	    if res is not None and (res < -90.0 or res > 90.0):
+	        log.error('Invalid latitude %d' % res)
+	        return None
+	    return res
+
+	def convert_longitude(input, json_obj, name):
+	    res = convert_coord(input, json_obj, name)
+	    if res is not None and (res < -180.0 or res > 180.0):
+	        log.error('Invalid longitude %d' % res)
+	        return None
+	    return res
+
+Meteo Data
+----------
+
+Python code: ::
+
+	def convert_temperature(input, json_obj, name):
+	    if input == 0:
+	        return None
+
+	    tempC = int(round(float(input)/1000.0 - 273.1))
+	    if tempC > 100:
+	        return None
+
+	    return tempC
+
+
+	def convert_barometer(input, json_obj, name):
+	    result = float(input) / 100.0
+	    if result > 2000:
+	        return None
+	    return int(round(result))
+
 
+	def convert_humidity(input, json_obj, name):
+	    humPercent = int(round(float(input) / 1000.0))
+	    if humPercent > 100:
+	        return None
+	    return humPercent
 

etl/harvester.cfg

@@ -0,0 +1,41 @@
+# Smart Emission Data Harvester - Stetl config
+#
+# Just van den Broecke - 2016
+#
+# This config reads raw measurements from the CityGIS Sensor REST API.
+# It inserts this data  in Postgres. The progres of harvesting (date/time) is tracked, so this allows to harvest all data
+# in multiple steps.
+# The measurement data in Postgres is then available for further refinement (validation, aggregation, etc).
+
+# The main Stetl ETL chain
+[etl]
+# chains = input_raw_sensor_api|output_postgres_insert
+chains = input_raw_sensor_api|output_std
+
+# Read all history data from CityGIS Sensor REST API
+[input_raw_sensor_api]
+class = rawsensorinput.RawSensorHistoryInput
+url = http://whale.citygis.nl/sensors/v1/raw
+host = {host}
+database = {database}
+user = {user}
+password = {password}
+schema = {schema}
+query = SELECT * from raw_sensor_sysnc WHERE file_name = '%s'
+
+# for testing/debugging
+[output_std]
+class = outputs.standardoutput.StandardOutput
+
+# Insert file records
+[output_postgres_insert]
+class = outputs.dboutput.PostgresInsertOutput
+input_format = record_array
+host = {pg_host}
+database = {pg_database}
+user = {pg_user}
+password = {pg_password}
+schema = {pg_schema}
+table = device_output
+key=unique_id
+replace=False

etl/harvester.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# ETL for reading last values from Smart Emission CityGIS Raw Sensor API
+#
+
+STETL_ARGS="-c harvester.cfg -a options/docker.args"
+WORK_DIR="`pwd`"
+PG_HOST=postgis
+NAME="stetl_harvest"
+
+# Stop and remove possibly old containers
+sudo docker stop ${NAME} > /dev/null 2>&1
+sudo docker rm ${NAME} > /dev/null 2>&1
+
+sudo docker run --name ${NAME} --link ${PG_HOST}:${PG_HOST} -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR}  geonovum/stetl ${STETL_ARGS}

etl/last.cfg

@@ -11,7 +11,7 @@ chains = input_raw_sensor_last_api|output_postgres_insert
 
 # for reading files from CityGIS Sensor REST API
 [input_raw_sensor_last_api]
-class = rawsensorlastinput.RawSensorLastInput
+class = rawsensorinput.RawSensorLastInput
 url = {raw_device_url}
 
 # for testing/debugging

etl/sensorconverters.py

@@ -135,28 +135,53 @@ def ppb_o3_to_ugm3(input, json_obj, name):
 # 	+ 0.008435412 * s.temperature.unit * sqrt(s.no2resistance)
 #
 def ohm_o3_to_ugm3(input, json_obj, name):
-
-    val1 = 89.1177 + 0.03420626 * json_obj['s_coresistance']  * math.log(json_obj['s_o3resistance'] )
-    val2 = - 0.008836714 * json_obj['s_lightsensorbottom']
-    val3 = - 0.02934928 * json_obj['s_coresistance']  * json_obj['s_temperatureambient']
-    val4 = - 1.439367 * json_obj['s_temperatureambient'] * math.log(json_obj['s_coresistance'] )
-    val5 = 1.26521 * math.log(json_obj['s_coresistance'] ) * math.sqrt(json_obj['s_coresistance'] )
-    val6 = - 0.000343098 * json_obj['s_coresistance']  * json_obj['s_no2resistance'] 
-    val7 = 0.02761877 * json_obj['s_no2resistance']  * math.log(json_obj['s_o3resistance'] )
-    val8 = - 0.0002260495 * json_obj['s_barometer'] * json_obj['s_coresistance'] 
-    val9 = 0.0699428 * json_obj['s_humidity']
-    val10 = 0.008435412 * json_obj['s_temperatureunit'] * math.sqrt(json_obj['s_no2resistance'] )
-
-    val = val1 + val2 + val3 + val4 + val5 + val6 + val7 + val8 + val9 + val10
-    print '%s : ohm=%d ugm3=%d' %(name, input, val)
+    # Ik doe als eerst een aantal preprocess stappen:
+    # - gassen in kOhm
+    # - Temperatuur in celcius (zowel ambient als unit)
+    # - Humidity in %, dus ook delen door 1000
+    # - barometer / 100
+    # - Ik doe niets met lightsensor_bottom
+
+    val = None
+    try:
+        s_no2resistance = ohm_to_kohm(json_obj['s_coresistance'])
+        s_o3resistance = ohm_to_kohm(json_obj['s_o3resistance'])
+        s_coresistance = ohm_to_kohm(json_obj['s_coresistance'])
+        s_temperatureambient = convert_temperature(json_obj['s_temperatureambient'])
+        s_temperatureunit = convert_temperature(json_obj['s_temperatureunit'])
+        s_humidity = convert_humidity(json_obj['s_humidity'])
+        s_barometer = convert_barometer(json_obj['s_barometer'])
+
+        val1 = 89.1177 + 0.03420626 * s_coresistance  * math.log(s_o3resistance )
+        val2 = - 0.008836714 * json_obj['s_lightsensorbottom']
+        val3 = - 0.02934928 * s_coresistance  * s_temperatureambient
+        val4 = - 1.439367 * s_temperatureambient * math.log(s_coresistance )
+        val5 = 1.26521 * math.log(s_coresistance ) * math.sqrt(s_coresistance )
+        val6 = - 0.000343098 * s_coresistance  * s_no2resistance
+        val7 = 0.02761877 * s_no2resistance  * math.log(s_o3resistance )
+        val8 = - 0.0002260495 * s_barometer * s_coresistance
+        val9 = 0.0699428 * s_humidity
+        val10 = 0.008435412 * s_temperatureunit * math.sqrt(s_no2resistance )
+        val = val1 + val2 + val3 + val4 + val5 + val6 + val7 + val8 + val9 + val10
+
+        if val < 0:
+            val = -val
+        print '%s : ohm=%d ugm3=%d' %(name, input, val)
+        if val > 400:
+            val = None
+    except Exception, e:
+        log.error('Error converting %s, err= %s' % (name, str(e)))
+
+    if val is not None:
+        json_obj['s_o3'] = val
     return val
 
 
-def ohm_to_kohm(input, json_obj, name):
+def ohm_to_kohm(input, json_obj=None, name=None):
     return int(round(input/1000))
 
 
-def convert_temperature(input, json_obj, name):
+def convert_temperature(input, json_obj=None, name=None):
     if input == 0:
         return None
 
@@ -167,14 +192,14 @@ def convert_temperature(input, json_obj, name):
     return tempC
 
 
-def convert_barometer(input, json_obj, name):
+def convert_barometer(input, json_obj=None, name=None):
     result = float(input) / 100.0
     if result > 2000:
         return None
     return int(round(result))
 
 
-def convert_humidity(input, json_obj, name):
+def convert_humidity(input, json_obj=None, name=None):
     humPercent = int(round(float(input) / 1000.0))
     if humPercent > 100:
         return None
@@ -354,8 +379,8 @@ def convert_audio_avg(value, json_obj, name):
     's_o3': ppb_o3_to_ugm3,
     's_coresistance': ohm_to_kohm,
     's_no2resistance': ohm_to_kohm,
-    's_o3resistance': ohm_to_kohm,
-    # 's_o3resistance': ohm_o3_to_ugm3,
+    # 's_o3resistance': ohm_to_kohm,
+    's_o3resistance': ohm_o3_to_ugm3,
     's_temperatureambient': convert_temperature,
     's_barometer': convert_barometer,
     's_humidity': convert_humidity,