Skip to content

Commit

Permalink
Added more EEA Importer fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@hylkevds
hylkevds committed May 12, 2024
1 parent 5816ad7 commit 7274a33
Show file tree
Hide file tree
Showing 5 changed files with 613 additions and 128 deletions.
249 changes: 125 additions & 124 deletions examples/EEA_LastDayNew.json
View file
Original file line number Diff line number Diff line change
@@ -1,125 +1,126 @@
{
"importer": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.ImporterCsv",
"classConfig": {
"recordConvertors": [
{
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.RecordConverterNames",
"classConfig": {
"colResult": "VALUE_NUMERIC",
"resultMissing": "-1",
"colUnit": "UNIT",
"converter": {
"conversions": [
{
"from": "mg/m3",
"to": "µg/m3",
"factor": 1000
},
{
"from": "mg/m3",
"to": "ug/m3",
"factor": 1000
},
{
"from": "ugNO2/m3",
"to": "ug/m3",
"factor": 1
},
{
"from": "ug.m-3",
"to": "ug/m3",
"factor": 1
},
{
"from": "µg/m3",
"to": "ug/m3",
"factor": 1
}
]
},
"colPhenTime": [
"DATETIME_BEGIN",
"DATETIME_END"
],
"dsm": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.DsMapperFilter",
"classConfig": {
"filterTemplate": "name eq \u0027{SAMPLINGPOINT_LOCALID}\u0027",
"dsGenerator": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.importers.eea.DataStreamGeneratorEea",
"classConfig": {
"stationsUrl": "http://discomap.eea.europa.eu/map/fme/metadata/PanEuropean_metadata.csv"
}
}
}
},
"timeParser": {
"format": "yyyyMMddHHmmss",
"zone": "+01:00"
},
"resultParser": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.utils.parsers.ParserBigdecimal",
"classConfig": {
"dropTailingZeroes": true,
"decimalSeparator": "DETECT"
}
}
}
}
],
"inputUrl": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.UrlGeneratorCombinations",
"classConfig": {
"baseUrl": "https://discomap.eea.europa.eu/Map/UTDViewer/dataService/Hourly?polu\u003d{polu}\u0026dt\u003d{time}",
"replaceSets": [
{
"replaceKey": "{polu}",
"replacementGen": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.StringListGenStatic",
"strings": [
"PM10",
"PM25",
"NO2",
"O3",
"SO2",
"CO"
]
}
},
{
"replaceKey": "{time}",
"replacementGen": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.StringListGeneratorTimeStrings",
"minus": 24,
"count": 26,
"delta": 1,
"zone": "+01:00",
"format": "yyyyMMddHHmmss"
}
}
]
}
},
"hasHeader": true
}
},
"validator": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.validator.ValidatorByPhenTime",
"classConfig": {
"update": true,
"cacheObservations": true,
"deleteDuplicates": true
}
},
"uploader": {
"serviceUrl": "http://localhost:8080/FROST-Server/v1.1",
"authMethod": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.auth.AuthNone",
"classConfig": {}
},
"useDataArrays": true,
"maxBatch": 1000
},
"name": "eeaIosbH"
}
"importer": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.ImporterCsv",
"classConfig": {
"recordConvertors": [
{
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.RecordConverterNames",
"classConfig": {
"colResult": "VALUE_NUMERIC",
"resultMissing": "-1",
"colUnit": "UNIT",
"converter": {
"conversions": [
{
"from": "mg/m3",
"to": "µg/m3",
"factor": 1000
},
{
"from": "mg/m3",
"to": "ug/m3",
"factor": 1000
},
{
"from": "ugNO2/m3",
"to": "ug/m3",
"factor": 1
},
{
"from": "ug.m-3",
"to": "ug/m3",
"factor": 1
},
{
"from": "µg/m3",
"to": "ug/m3",
"factor": 1
}
]
},
"colPhenTime": [
"DATETIME_BEGIN",
"DATETIME_END"
],
"dsm": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.DsMapperFilter",
"classConfig": {
"filterTemplate": "Thing/properties/localId eq \u0027{STATIONCODE}\u0027 and ObservedProperty/name eq \u0027{PROPERTY}\u0027",
"dsGenerator": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.importers.eea.DataStreamGeneratorEea2",
"classConfig": {
"stationsUrl": "https://discomap.eea.europa.eu/App/AQViewer/data?fqn\u003dAirquality_Dissem.b2g.measurements",
"template": "{\n\t\"Page\": 0,\n\t\"SortBy\": null,\n\t\"SortAscending\": true,\n\t\"RequestFilter\": {\n\t\t\"AssessmentMethodId\": {\n\t\t\t\"FieldName\": \"AssessmentMethodId\",\n\t\t\t\"Values\": [\"$SAMPLING_POINT_ID\"]\n\t\t}\n\t}\n}"
}
}
}
},
"timeParser": {
"format": "yyyyMMddHHmmss",
"zone": "+01:00"
},
"resultParser": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.utils.parsers.ParserBigdecimal",
"classConfig": {
"dropTailingZeroes": true,
"decimalSeparator": "DETECT"
}
}
}
}
],
"inputUrl": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.UrlGeneratorCombinations",
"classConfig": {
"baseUrl": "https://discomap.eea.europa.eu/Map/UTDViewer/dataService/Hourly?polu\u003d{polu}\u0026dt\u003d{time}",
"replaceSets": [
{
"replaceKey": "{polu}",
"replacementGen": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.StringListGenStatic",
"strings": [
"PM10",
"PM25",
"NO2",
"O3",
"SO2",
"CO"
]
}
},
{
"replaceKey": "{time}",
"replacementGen": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.csv.StringListGeneratorTimeStrings",
"minus": 24,
"count": 26,
"delta": 1,
"zone": "+01:00",
"format": "yyyyMMddHHmmss"
}
}
]
}
},
"hasHeader": true
}
},
"validator": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.validator.ValidatorByPhenTime",
"classConfig": {
"update": true,
"cacheObservations": true,
"deleteDuplicates": true
}
},
"uploader": {
"serviceUrl": "http://localhost:8080/FROST-Server/v1.1",
"authMethod": {
"className": "de.fraunhofer.iosb.ilt.sensorthingsimporter.auth.AuthNone",
"classConfig": {}
},
"useDataArrays": true,
"maxBatch": 1000
},
"name": "eeaIosbH"
}
2 changes: 1 addition & 1 deletion src/main/java/de/fraunhofer/iosb/ilt/sensorthingsimporter/csv/DsMapperFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ private Datastream getDatastreamFor(String filter, CSVRecord record, ErrorLog er
EntityList<Datastream> streams = query.list();
if (streams.size() > 1) {
LOGGER.error("Found incorrect number of datastreams: {} for filter: {}", streams.size(), filter);
throw new ImportException("Found incorrect number of datastreams: " + streams.size() + " for filter: " + filter);
return null;
} else if (streams.isEmpty()) {
if (dsGenerator != null) {
ds = dsGenerator.createDatastreamFor(record, errorLog);
Expand Down

0 comments on commit 7274a33

Please sign in to comment.