Skip to content

Commit

Permalink
Merge pull request #424 from ScorpioBroker/confix
Browse files Browse the repository at this point in the history 
Confix
  • Loading branch information
@ScorpioBroker
ScorpioBroker committed Aug 15, 2023
2 parents 055fc1b + d4a7b02 commit f903919
Show file tree
Hide file tree
Showing 17 changed files with 2,699 additions and 1,247 deletions.
39 changes: 39 additions & 0 deletions AllInOneRunner/src/main/resources/db/migration/V20230810.1__historyup.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
ALTER TABLE IF EXISTS public.temporalentityattrinstance
ADD COLUMN IF NOT EXISTS location geometry;
CREATE INDEX IF NOT EXISTS i_temporalentityattrinstance_location
ON public.temporalentityattrinstance USING gist
(location)
WITH (buffering=auto)
;
CREATE INDEX IF NOT EXISTS i_temporalentityattrinstance_entityid
ON public.temporalentityattrinstance USING hash
(temporalentity_id)
;
with x as (SELECT distinct temporalentity_id as eid, geovalue, modifiedat as mat, observedat as oat, COALESCE(modifiedat, observedat) FROM temporalentityattrinstance WHERE geovalue is not null ORDER BY COALESCE(modifiedat, observedat)) UPDATE temporalentityattrinstance SET location = (SELECT x.geovalue FROM x WHERE eid = temporalentity_id and COALESCE(x.mat, x.oat) <= COALESCE(modifiedat, observedat) ORDER BY COALESCE(modifiedat, observedat) DESC limit 1) WHERE location is not null;

CREATE OR REPLACE FUNCTION public.temporalentityattrinstance_extract_jsonb_fields()
RETURNS trigger
LANGUAGE plpgsql
AS $function$
DECLARE
f_internalid temporalentityattrinstance.internalid%TYPE;
BEGIN
IF TG_OP = 'INSERT' OR NEW.data <> OLD.data THEN -- do not reprocess if it is just an update on other column (e.g. static)
NEW.instanceid = NEW.data#>>'{https://uri.etsi.org/ngsi-ld/instanceId,0,@id}';
NEW.createdat = (NEW.data#>>'{https://uri.etsi.org/ngsi-ld/createdAt,0,@value}')::TIMESTAMP;
NEW.modifiedat = (NEW.data#>>'{https://uri.etsi.org/ngsi-ld/modifiedAt,0,@value}')::TIMESTAMP;
NEW.observedat = (NEW.data#>>'{https://uri.etsi.org/ngsi-ld/observedAt,0,@value}')::TIMESTAMP;
IF NEW.data#>>'{@type,0}' = 'https://uri.etsi.org/ngsi-ld/GeoProperty' THEN
NEW.geovalue = ST_SetSRID(ST_GeomFromGeoJSON( getGeoJson(NEW.data#>'{https://uri.etsi.org/ngsi-ld/hasValue,0}') ), 4326);
ELSE
NEW.geovalue = NULL;
END IF;
IF NEW.location is NULL THEN
SELECT teai.location INTO NEW.location FROM temporalentityattrinstance teai WHERE COALESCE(teai.modifiedat, teai.observedat) <= COALESCE(NEW.modifiedat, NEW.observedat) ORDER BY COALESCE(teai.modifiedat, teai.observedat) LIMIT 1;
END IF;
END IF;

RETURN NEW;
END;
$function$

2 changes: 2 additions & 0 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/constants/AppConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ public class AppConstants {
public static final String SQL_ALREADY_EXISTS = "23505";
public static final String SQL_NOT_FOUND = "02000";
public static final String SQL_INVALID_OPERATOR = "42804";
public static final String SQL_INVALID_INTERVAL = "22007";

public static final String NGB_APPLICATION_ZIP = "application/zip";
public static final int INTERNAL_NOTIFICATION_REQUEST = 99;
public static final int INTERVAL_NOTIFICATION_REQUEST = 100;
Expand Down
8 changes: 8 additions & 0 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/constants/NGSIConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,8 @@ public interface NGSIConstants {
public final static String QUERY_PARAMETER_CONCISE = "option";
public static final String SCOPE = "scope";
public static final String PREVIOUS_VALUE = "previousValue";

public static final String PREVIOUS_LANGUAGE_MAP = "previousLanguageMap";
public static final String PREVIOUS_OBJECT = "previousObject";
public static final String NGSI_LD_REG_MODE = "https://uri.etsi.org/ngsi-ld/mode";
public static final String NGSI_LD_REG_MODE_AUX = "auxiliary";
Expand Down Expand Up @@ -436,6 +438,12 @@ public interface NGSIConstants {
public static final String HAS_OBJECT_NULL = "[{https://uri.etsi.org/ngsi-ld/hasObject=[{@id=urn:ngsi-ld:null}]";
public static final String LANGUAGE_PROPERTY = "LanguageProperty";
public static final String LANGUAGE_MAP = "languageMap";


public static final Set<String> ENTITY_BASE_PROPS = Sets.newHashSet(JSON_LD_ID, JSON_LD_TYPE, NGSI_LD_CREATED_AT, NGSI_LD_MODIFIED_AT, NGSI_LD_SCOPE);


public final static String NGSI_LD_HAS_KEY ="https://uri.etsi.org/ngsi-ld/hasKey";

}

60 changes: 60 additions & 0 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/datatypes/terms/GeoQueryTerm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,66 @@ public int toSql(StringBuilder query, Tuple tuple, int dollar) {
return dollar;
}

public int toTempSql(StringBuilder query, Tuple tuple, int dollar) throws ResponseException{
String dbColumn;
if (!geoproperty.equals(NGSIConstants.NGSI_LD_LOCATION)) {
// query.append("data @> '{\"");
// query.append(geoproperty);
// query.append("\": [{\"");
// query.append(NGSIConstants.JSON_LD_TYPE);
// query.append("\":[\"");
// query.append(NGSIConstants.NGSI_LD_GEOPROPERTY);
// query.append("\"]}]}' AND ");
// dbColumn = "ST_SetSRID(ST_GeomFromGeoJSON( getGeoJson( " + "data#>'{" + geoproperty + ",0,"
// + NGSIConstants.NGSI_LD_HAS_VALUE + ",0}') ), 4326)";
throw new ResponseException(ErrorType.InvalidRequest, "Unfortunatley the temporal api can only support geoqueries on the location field");
} else {
dbColumn = "location";
}

String referenceValue = "ST_SetSRID(ST_GeomFromGeoJSON('{\"type\": \"" + geometry + "\", \"coordinates\": "
+ coordinates + " }'), 4326)";
String sqlPostgisFunction = DBConstants.NGSILD_TO_POSTGIS_GEO_OPERATORS_MAPPING.get(georel);
switch (georel) {
case NGSIConstants.GEO_REL_NEAR:
if (distanceType.equals(NGSIConstants.GEO_REL_MIN_DISTANCE)) {
query.append("NOT ");
}
query.append(sqlPostgisFunction);
query.append("( ");
query.append(dbColumn);
query.append("::geography, ");
query.append(referenceValue);
query.append("::geography, ");
query.append(distanceValue);
query.append(") ");
// query.append("or ");
// query.append(sqlPostgisFunction);
// query.append("( ");
// query.append(dbColumn);
// query.append(", ");
// query.append(referenceValue);
// query.append(", ");
// query.append(distanceValue);
// query.append(") ");
break;
case NGSIConstants.GEO_REL_WITHIN:
case NGSIConstants.GEO_REL_CONTAINS:
case NGSIConstants.GEO_REL_OVERLAPS:
case NGSIConstants.GEO_REL_INTERSECTS:
case NGSIConstants.GEO_REL_EQUALS:
case NGSIConstants.GEO_REL_DISJOINT:
query.append(sqlPostgisFunction);
query.append("( ");
query.append(dbColumn);
query.append(", ");
query.append(referenceValue);
query.append(") ");
break;
}
return dollar;
}

public int toTempSql(StringBuilder query, Tuple tuple, int dollar, TemporalQueryTerm tempQuery) {
String dbColumn = "geovalue";
query.append("attributeid = $");
Expand Down
76 changes: 57 additions & 19 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/datatypes/terms/QQueryTerm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import eu.neclab.ngsildbroker.commons.tools.SerializationTools;
import io.vertx.mutiny.sqlclient.Tuple;

public class QQueryTerm implements Serializable {
public class QQueryTerm implements Serializable {

/**
*
Expand All @@ -47,7 +47,7 @@ public class QQueryTerm implements Serializable {
private String operator = "";
private String operant = "";
private Set<String> allAttribs = Sets.newHashSet();

QQueryTerm() {
// for serialization
}
Expand Down Expand Up @@ -586,15 +586,14 @@ private int getAttribQuery(StringBuilder result, int dollarCount, Tuple tuple) {
result.append("ENTITY ? $");
result.append(dollarCount);


String[] splitted = getAttribute().split("\\[");
if (splitted.length > 1) {
splitted[1] = splitted[1].substring(0, splitted[1].length() - 1);
}
String[] subAttribPath = splitted.length == 1 ? null : splitted[1].split("\\.");
String[] attribPath = splitted[0].split("\\.");
String attribName = linkHeaders.expandIri(attribPath[0], false, true, null, null);
if(attribName.equals("@id")){
if (attribName.equals("@id")) {
result.append(" AND entity ->> $");
result.append(dollarCount);
dollarCount++;
Expand All @@ -603,7 +602,7 @@ private int getAttribQuery(StringBuilder result, int dollarCount, Tuple tuple) {
result.append(dollarCount);
dollarCount++;
tuple.addString(operant);
}else {
} else {
result.append(" AND EXISTS (SELECT TRUE FROM JSONB_ARRAY_ELEMENTS(ENTITY -> $");
result.append(dollarCount);
dollarCount++;
Expand Down Expand Up @@ -909,15 +908,15 @@ private int applyOperator(StringBuilder attributeFilterProperty, int dollarCount
// attributeFilterProperty.append("'");
dollarCount++;
tuple.addString(operant);
//addItemToTupel(tuple, operant);
// addItemToTupel(tuple, operant);
break;
case NGSIConstants.QUERY_NOTPATTERNOP:
attributeFilterProperty.append("::text !~ $");
attributeFilterProperty.append(dollarCount);
// attributeFilterProperty.append("'");
dollarCount++;
tuple.addString(operant);
//addItemToTupel(tuple, operant);
// addItemToTupel(tuple, operant);
break;
}
return dollarCount;
Expand Down Expand Up @@ -951,25 +950,25 @@ public void addAttrib(String attrib) {

private int temporalSqlWherePart(StringBuilder sql, int dollarCount, Tuple tuple, QQueryTerm current,
TemporalQueryTerm tempQuery) {
sql.append("(TEAI.ATTRIBUTEID=$");
sql.append(dollarCount);
dollarCount++;
// sql.append("(TEAI.ATTRIBUTEID=$");
// sql.append(dollarCount);
// dollarCount++;
String[] splitted = current.getAttribute().split("\\[");
if (splitted.length > 1) {
splitted[1] = splitted[1].substring(0, splitted[1].length() - 1);
}
String[] subAttribPath = splitted.length == 1 ? null : splitted[1].split("\\.");
String[] attribPath = splitted[0].split("\\.");
String attribName = linkHeaders.expandIri(attribPath[0], false, true, null, null);
tuple.addString(attribName);
if (tempQuery != null) {
sql.append(" AND TEAI.");
dollarCount = tempQuery.toSql(sql, tuple, dollarCount);
}
// String attribName = linkHeaders.expandIri(attribPath[0], false, true, null, null);
// tuple.addString(attribName);
// if (tempQuery != null) {
// sql.append(" AND TEAI.");
// dollarCount = tempQuery.toSql(sql, tuple, dollarCount);
// }
String currentSqlAttrib = "TEAI.data";
if (!current.getOperator().isEmpty() || attribPath.length > 1 || subAttribPath != null) {
sql.append(" AND ");
}
// if (!current.getOperator().isEmpty() || attribPath.length > 1 || subAttribPath != null) {
// sql.append(" AND ");
// }

return commonWherePart(attribPath, subAttribPath, currentSqlAttrib, dollarCount, tuple, sql, current);
}
Expand Down Expand Up @@ -1151,4 +1150,43 @@ public int[] toTempSql(StringBuilder sql, int dollarCount, Tuple tuple, int char
return new int[] { dollarCount, charCount };
}

public int toTempSql(StringBuilder sql, StringBuilder laterSql, Tuple tuple, int dollarCount) {
QQueryTerm current = this;

while(current != null) {
if (firstChild != null) {
laterSql.append("(");
dollarCount = firstChild.toTempSql(sql, laterSql, tuple, dollarCount);
laterSql.append(")");
}
laterSql.append("$");
laterSql.append(dollarCount);
laterSql.append(" = any(attribs)");
tuple.addString(linkHeaders.expandIri(current.attribute, false, true, null, null));
if(current.hasNext()) {
if(current.isNextAnd()) {
laterSql.append(" AND ");
}else {
laterSql.append(" OR ");
}
}
dollarCount++;
if(current.getOperant() != null) {
sql.append(" WHEN TEAI.attributeid=$");
sql.append(dollarCount-1);
sql.append(" THEN (");

dollarCount = temporalSqlWherePart(sql, dollarCount, tuple, current, null);
}



current = current.getNext();
}



return dollarCount;
}

}
12 changes: 8 additions & 4 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/datatypes/terms/TemporalQueryTerm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,26 @@ public int toSql(StringBuilder sql, Tuple tuple, int dollarCount) {
case NGSIConstants.TIME_REL_BEFORE:
sql.append(" < $");
sql.append(dollarCount);
tuple.addLocalDateTime(LocalDateTime.parse(getTimeAt(), SerializationTools.informatter));
sql.append("::text::timestamp");
tuple.addString(getTimeAt());
dollarCount++;
break;
case NGSIConstants.TIME_REL_AFTER:
sql.append(" > $");
sql.append(dollarCount);
tuple.addLocalDateTime(LocalDateTime.parse(getTimeAt(), SerializationTools.informatter));
sql.append("::text::timestamp");
tuple.addString(getTimeAt());
dollarCount++;
break;
case NGSIConstants.TIME_REL_BETWEEN:
sql.append(" between $");
sql.append(dollarCount);
sql.append("::text::timestamp");
sql.append(" AND $");
sql.append((dollarCount + 1));
tuple.addLocalDateTime(LocalDateTime.parse(getTimeAt(), SerializationTools.informatter));
tuple.addLocalDateTime(LocalDateTime.parse(getEndTimeAt(), SerializationTools.informatter));
sql.append("::text::timestamp");
tuple.addString(getTimeAt());
tuple.addString(getEndTimeAt());
dollarCount += 2;
break;
}
Expand Down
3 changes: 3 additions & 0 deletions Commons/src/main/java/eu/neclab/ngsildbroker/commons/tools/QueryParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,9 @@ public static TemporalQueryTerm parseTempQuery(String timeProperty, String timeR
throw new ResponseException(ErrorType.InvalidRequest, timeProperty + " is not an allowed timeproperty. Use "
+ StringUtils.join(NGSIConstants.ALLOWED_TIME_PROPERTIES, " or "));
}
if (timeRel == null) {
return new TemporalQueryTerm(timeProperty, null, null, null);
}
switch (timeRel) {
case NGSIConstants.TIME_REL_AFTER:
if (endTimeAt != null) {
Expand Down
11 changes: 5 additions & 6 deletions ...nager/src/main/java/eu/neclab/ngsildbroker/entityhandler/controller/EntityController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,6 @@ private void noConcise(Object object, Map<String, Object> parentMap, String keyO
// Map have object but not type
if (map.containsKey(NGSIConstants.OBJECT)) {
((Map<String, Object>) map).put(NGSIConstants.TYPE, NGSIConstants.RELATIONSHIP);

}
// Map have value but not type
if (map.containsKey(NGSIConstants.VALUE) && !map.containsKey(NGSIConstants.TYPE)) {
Expand All @@ -249,11 +248,10 @@ private void noConcise(Object object, Map<String, Object> parentMap, String keyO
parentMap.put(keyOfObject, newMap);

}
if (NGSIConstants.LANGUAGE_MAP.equals(keyOfObject) && parentMap != null
&& NGSIConstants.LANGUAGE_PROPERTY.equals(parentMap.get(NGSIConstants.TYPE))) {
return;
if (map.containsKey(NGSIConstants.LANGUAGE_MAP)) {
((Map<String, Object>) map).put(NGSIConstants.TYPE, NGSIConstants.LANGUAGE_PROPERTY);
}

// Iterate through every element of Map
Object[] mapKeys = map.keySet().toArray();
for (Object key : mapKeys) {
Expand All @@ -264,7 +262,8 @@ private void noConcise(Object object, Map<String, Object> parentMap, String keyO
&& !key.equals(NGSIConstants.INSTANCE_ID)
&& !key.equals(NGSIConstants.QUERY_PARAMETER_DATA_SET_ID) && !key.equals(NGSIConstants.OBJECT)
&& !key.equals(NGSIConstants.VALUE) && !key.equals(NGSIConstants.SCOPE)
&& !key.equals(NGSIConstants.QUERY_PARAMETER_UNIT_CODE)) {
&& !key.equals(NGSIConstants.QUERY_PARAMETER_UNIT_CODE)
&& !key.equals(NGSIConstants.LANGUAGE_MAP)) {
noConcise(map.get(key), (Map<String, Object>) map, key.toString());
}
}
Expand Down

0 comments on commit f903919

Please sign in to comment.