Skip to content

Commit

Permalink
Merge pull request #798 from FIWARE/task/metadataSysAttrs
Browse files Browse the repository at this point in the history 
Task/metadata sys attrs
  • Loading branch information
@kzangeli
kzangeli committed Apr 18, 2021
2 parents d513258 + 849f8bd commit ab6db86
Show file tree
Hide file tree
Showing 62 changed files with 2,426 additions and 138 deletions.
2 changes: 2 additions & 0 deletions CHANGES_NEXT_RELEASE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,5 @@
* Issue #280 Implemented datasetId for PATCH /entities/{EID}/attrs/{attrName}
* Issue #280 Implemented datasetId for DELETE /entities/{EID}/attrs/{attrName} (supporting URI params 'datasetId' and 'deleteAll')
* Issue #280 Implemented GET /attributes, but only without details=true
* Issue #280 Slightly increased response from Batch Upsert
* Issue #280 Implemented system attributes (createdAt, modifiedAt) for sub-attributes
173 changes: 93 additions & 80 deletions src/lib/mongoBackend/MongoCommonUpdate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@
#include <set>
#include <algorithm>

extern "C"
{
#include "kjson/KjNode.h" // KjNode, kjValueType
#include "kjson/kjLookup.h" // kjLookup
}

#include "logMsg/logMsg.h"
#include "logMsg/traceLevels.h"

Expand All @@ -50,17 +56,11 @@
#include "ngsi/Scope.h"
#include "rest/uriParamNames.h"

#ifdef ORIONLD
extern "C"
{
#include "kjson/KjNode.h" // KjNode, kjValueType
#include "kjson/kjLookup.h" // kjLookup
}

#include "orionld/types/AttributeType.h" // AttributeType
#include "orionld/common/orionldState.h" // orionldState
#include "orionld/common/geoJsonCreate.h" // geoJsonCreate
#include "orionld/common/isSpecialSubAttribute.h" // isSpecialSubAttribute
#include "orionld/db/dbConfiguration.h" // dbDataFromKjTree
#endif

#include "mongoBackend/connectionOperations.h"
#include "mongoBackend/safeMongo.h"
Expand Down Expand Up @@ -352,11 +352,17 @@ static void appendMetadata
(
BSONObjBuilder* mdBuilder,
BSONArrayBuilder* mdNamesBuilder,
const Metadata* mdP,
Metadata* mdP,
bool useDefaultType
)
{
std::string type = mdP->type;
BSONObjBuilder bsonmd;
std::string type = mdP->type;
const char* mdName = mdP->name.c_str();
AttributeType mdType;
bool special = isSpecialSubAttribute(mdName, &mdType, NULL);

LM_TMP(("MS: appending metadata", mdName));

if (!mdP->typeGiven && useDefaultType)
{
Expand All @@ -373,87 +379,83 @@ static void appendMetadata
mdNamesBuilder->append(mdP->name);
std::string effectiveName = dbDotEncode(mdP->name);

if (type != "")
//
// Filling in the RHS of the metadata ("mdName": RHS)
//
if (special == false)
{
switch (mdP->valueType)
//
// Setting sysAttrs
//
if (orionldState.apiVersion == NGSI_LD_V1)
{
case orion::ValueTypeString:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << mdP->stringValue));
return;

case orion::ValueTypeNumber:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << mdP->numberValue));
return;

case orion::ValueTypeBoolean:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << mdP->boolValue));
return;

case orion::ValueTypeNull:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << mongo::BSONNULL));
return;

case orion::ValueTypeObject:
if (mdP->compoundValueP->valueType == orion::ValueTypeVector)
LM_TMP(("MS: Setting MD SysAttrs"));
if (mdP->createdAt == 0)
{
BSONArrayBuilder ba;
compoundValueBson(mdP->compoundValueP->childV, ba);
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << ba.arr()));
LM_TMP(("MS: Setting createdAt in BSON"));
mdP->createdAt = orionldState.requestTime;
bsonmd.append("createdAt", mdP->createdAt);
}
else
{
BSONObjBuilder bo;

compoundValueBson(mdP->compoundValueP->childV, bo);
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_TYPE << type << ENT_ATTRS_MD_VALUE << bo.obj()));
}
break;

default:
LM_E(("Runtime Error (unknown metadata type: %d)", mdP->valueType));
LM_TMP(("MS: Setting modifiedAt in BSON"));
mdP->modifiedAt = orionldState.requestTime;
bsonmd.append("modifiedAt", mdP->modifiedAt);
}
}
else
{
switch (mdP->valueType)
{
case orion::ValueTypeString:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << mdP->stringValue));
return;

case orion::ValueTypeNumber:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << mdP->numberValue));
return;

case orion::ValueTypeBoolean:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << mdP->boolValue));
return;

case orion::ValueTypeNull:
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << mongo::BSONNULL));
return;

case orion::ValueTypeObject:
if (mdP->compoundValueP->isVector())
{
BSONArrayBuilder ba;

compoundValueBson(mdP->compoundValueP->childV, ba);
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << ba.arr()));
}
else
{
BSONObjBuilder bo;
LM_TMP(("MS: Setting MD Type"));
if (type != "")
bsonmd.append("type", type);

LM_TMP(("MS: Setting MD Value"));
switch (mdP->valueType)
{
case orion::ValueTypeString:
LM_TMP(("MS: Setting MD String Value"));
bsonmd.append(ENT_ATTRS_MD_VALUE, mdP->stringValue);
break;

case orion::ValueTypeNumber:
LM_TMP(("MS: Setting MD Number Value (%f)", mdP->numberValue));
bsonmd.append(ENT_ATTRS_MD_VALUE, mdP->numberValue);
LM_TMP(("MS: Set MD Number Value (%f)", mdP->numberValue));
break;

case orion::ValueTypeBoolean:
LM_TMP(("MS: Setting MD Boolean Value"));
bsonmd.append(ENT_ATTRS_MD_VALUE, mdP->boolValue);
break;

case orion::ValueTypeNull:
LM_TMP(("MS: Setting MD Null Value"));
bsonmd.appendNull(ENT_ATTRS_MD_VALUE);
break;

case orion::ValueTypeObject:
LM_TMP(("MS: Setting MD Compound Value"));
if (mdP->compoundValueP->isVector())
{
BSONArrayBuilder ba;

compoundValueBson(mdP->compoundValueP->childV, bo);
mdBuilder->append(effectiveName, BSON(ENT_ATTRS_MD_VALUE << bo.obj()));
}
break;
compoundValueBson(mdP->compoundValueP->childV, ba);
bsonmd.append(ENT_ATTRS_MD_VALUE, ba.arr());
}
else
{
BSONObjBuilder bo;

default:
LM_E(("Runtime Error (unknown metadata type)"));
compoundValueBson(mdP->compoundValueP->childV, bo);
bsonmd.append(ENT_ATTRS_MD_VALUE, bo.obj());
}
break;

default:
LM_E(("Runtime Error (unknown metadata type)"));
}

LM_TMP(("MS: Appending MD to mdBuilder"));
mdBuilder->append(effectiveName, bsonmd.obj());
LM_TMP(("MS: Appended MD '%s' to mdBuilder", effectiveName.c_str()));
}


Expand Down Expand Up @@ -670,14 +672,18 @@ static bool contextAttributeCustomMetadataToBson

for (unsigned int ix = 0; ix < ca->metadataVector.size(); ++ix)
{
const Metadata* md = ca->metadataVector[ix];
Metadata* md = ca->metadataVector[ix];

LM_TMP(("MS: Calling isNotCustomMetadata for '%s'", md->name.c_str()));
if (!isNotCustomMetadata(md->name.c_str()))
{
LM_TMP(("MS: '%s' is a custom metadata", md->name.c_str()));
appendMetadata(&mdToAdd, &mdNamesToAdd, md, useDefaultType);
LM_T(LmtMongo, ("new custom metadata: {name: %s, type: %s, value: %s}",
md->name.c_str(), md->type.c_str(), md->toStringValue().c_str()));
}
else
LM_TMP(("MS: '%s' is NOT a custom metadata", md->name.c_str()));
}

*md = mdToAdd.obj();
Expand Down Expand Up @@ -840,9 +846,11 @@ static bool appendAttribute
return false;
}

LM_TMP(("MS: APPEND with existing attribute"));
//
// If updateAttribute fails, the name of the attribute caP is added to the list of erroneous attributes
//
LM_TMP(("MS: Calling updateAttribute"));
if (updateAttribute(attrs, toSet, toPush, caP, actualUpdate, false, apiVersion) == false)
orionldStateErrorAttributeAdd(caP->name.c_str());

Expand All @@ -852,6 +860,8 @@ static bool appendAttribute
/* Build the attribute to append */
BSONObjBuilder ab;

LM_TMP(("MS: ADDING a new attribute - all MD created/modifiedAt to be added"));

/* 1. Value */
caP->valueBson(ab, caP->type, ngsiv1Autocast && (apiVersion == V1));

Expand Down Expand Up @@ -880,6 +890,7 @@ static bool appendAttribute
BSONObj md;
BSONArray mdNames;

LM_TMP(("MS: Calling contextAttributeCustomMetadataToBson (created/modifiedAt to be added)"));
if (contextAttributeCustomMetadataToBson(&md, &mdNames, caP, apiVersion == V2))
{
ab.append(ENT_ATTRS_MD, md);
Expand Down Expand Up @@ -2814,6 +2825,7 @@ static bool processContextAttributeVector
bool actualUpdate = true;
if ((action == ActionTypeUpdate) || (action == ActionTypeReplace))
{
LM_TMP(("MS: Calling updateContextAttributeItem"));
if (!updateContextAttributeItem(cerP,
ca,
attrs,
Expand All @@ -2837,6 +2849,7 @@ static bool processContextAttributeVector
}
else if ((action == ActionTypeAppend) || (action == ActionTypeAppendStrict))
{
LM_TMP(("MS: Calling appendContextAttributeItem"));
if (!appendContextAttributeItem(cerP,
attrs,
targetAttr,
Expand Down
12 changes: 9 additions & 3 deletions src/lib/mongoBackend/MongoGlobal.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1235,6 +1235,8 @@ static void addDatesForAttrs(ContextElementResponse* cerP, bool includeCreDate,
{
ContextAttribute* caP = cerP->contextElement.contextAttributeVector[ix];

LM_TMP(("SA: attribute: '%s'", caP->name.c_str()));

if (includeCreDate && caP->creDate != 0)
{
// Lookup Metadata NGSI_MD_DATECREATED
Expand All @@ -1243,6 +1245,8 @@ static void addDatesForAttrs(ContextElementResponse* cerP, bool includeCreDate,
{
Metadata* mdP = caP->metadataVector[mIx];

LM_TMP(("SA: metadata '%s'", mdP->name.c_str()));

if (mdP->name == NGSI_MD_DATECREATED)
{
dateCreatedMetadataP = mdP;
Expand Down Expand Up @@ -1344,6 +1348,7 @@ bool entitiesQuery
ApiVersion apiVersion
)
{
LM_TMP(("SA: In entitiesQuery"));
/* Query structure is as follows
*
* {
Expand Down Expand Up @@ -1650,6 +1655,7 @@ bool entitiesQuery
LM_T(LmtMongo, ("retrieved document [%d]: '%s'", docs, r.toString().c_str()));
ContextElementResponse* cer = new ContextElementResponse(r, attrL, includeEmpty, apiVersion);

LM_TMP(("SA: Adding dates for attributes"));
addDatesForAttrs(cer, metadataList.lookup(NGSI_MD_DATECREATED), metadataList.lookup(NGSI_MD_DATEMODIFIED));

/* All the attributes existing in the request but not found in the response are added with 'found' set to false */
Expand Down Expand Up @@ -1721,10 +1727,10 @@ bool entitiesQuery

if (needToAdd)
{
ContextElementResponse* cerP = new ContextElementResponse();
ContextElementResponse* cerP = new ContextElementResponse();

cerP->contextElement.entityId.id = enV[ix]->id;
cerP->contextElement.entityId.type = enV[ix]->type;
cerP->contextElement.entityId.id = enV[ix]->id;
cerP->contextElement.entityId.type = enV[ix]->type;
cerP->contextElement.entityId.isPattern = "false";

//
Expand Down
8 changes: 8 additions & 0 deletions src/lib/ngsi/ContextElementResponse.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
#include "mongoBackend/dbFieldEncoding.h"
#include "mongoBackend/compoundResponses.h"

#include "orionld/common/orionldState.h" // orionldState

using namespace mongo;


Expand Down Expand Up @@ -139,6 +141,8 @@ ContextElementResponse::ContextElementResponse
{
prune = false;

LM_TMP(("SA: In ContextElementResponse constructor"));

// Entity
BSONObj id = getFieldF(entityDoc, "_id").embeddedObject();

Expand Down Expand Up @@ -268,12 +272,16 @@ ContextElementResponse::ContextElementResponse
BSONObj mds = getObjectFieldF(attr, ENT_ATTRS_MD);
std::set<std::string> mdsSet;

LM_TMP(("SA: mds: %s", mds.toString().c_str()));
mds.getFieldNames(mdsSet);
for (std::set<std::string>::iterator i = mdsSet.begin(); i != mdsSet.end(); ++i)
{
std::string currentMd = *i;
Metadata* md = new Metadata(dbDotDecode(currentMd), getObjectFieldF(mds, currentMd));
caP->metadataVector.push_back(md);
LM_TMP(("SA: pushed the metadata '%s' - system attrs set by Metadata constructor", md->name.c_str()));
LM_TMP(("SA: createdAt for '%s': %f", md->name.c_str(), md->createdAt));
LM_TMP(("SA: modifiedAt for '%s': %f", md->name.c_str(), md->modifiedAt));
}
}

Expand Down
Loading

0 comments on commit ab6db86

Please sign in to comment.