Skip to content

Commit

Permalink
Merge pull request #804 from FIWARE/task/Issue_803_http_status_codes_…
Browse files Browse the repository at this point in the history 
…for_batch_upsert

Fixed issue #803
  • Loading branch information
@kzangeli
kzangeli committed Apr 22, 2021
2 parents 5c0eb99 + c45d846 commit d4d0b95
Show file tree
Hide file tree
Showing 18 changed files with 533 additions and 64 deletions.
1 change: 1 addition & 0 deletions CHANGES_NEXT_RELEASE
View file
Expand Up @@ -9,3 +9,4 @@
* Issue #280 Slightly increased response from Batch Upsert
* Issue #280 Implemented system attributes (createdAt, modifiedAt) for sub-attributes
* Issue #280 Implemented POST /temporal/entities - to add entities to TRoE without actually creating any entity (for the current state)
* Issue #803 Correct HTTP Status Codes for Batch Upsert (201/204/207) according to NGSI-LD v1.3
22 changes: 21 additions & 1 deletion src/lib/orionld/kjTree/kjEntityIdArrayExtract.cpp
View file
Expand Up @@ -37,6 +37,7 @@ extern "C"
#include "orionld/common/orionldState.h" // orionldState
#include "orionld/common/entityIdAndTypeGet.h" // entityIdAndTypeGet
#include "orionld/common/entityErrorPush.h" // entityErrorPush
#include "orionld/payloadCheck/pcheckEntity.h" // pcheckEntity
#include "orionld/kjTree/kjStringValueLookupInArray.h" // kjStringValueLookupInArray
#include "orionld/kjTree/kjEntityIdArrayExtract.h" // Own interface

Expand Down Expand Up @@ -65,7 +66,7 @@ static KjNode* entityIdPush(KjNode* entityIdsArrayP, const char* entityId)
//
// kjEntityIdArrayExtract -
//
KjNode* kjEntityIdArrayExtract(KjNode* entityArray, KjNode* successArray, KjNode* errorArray)
KjNode* kjEntityIdArrayExtract(KjNode* entityArray, KjNode* errorArray)
{
KjNode* entityP = entityArray->value.firstChildP;
KjNode* next;
Expand Down Expand Up @@ -136,6 +137,25 @@ KjNode* kjEntityIdArrayExtract(KjNode* entityArray, KjNode* successArray, KjNode
continue;
}

#if 0
// Call pcheckEntity() and move to errorArray if needed
// But, not until pcheckEntity is good enough
if (pcheckEntity(entityP->value.firstChildP, NULL, NULL, NULL, NULL, NULL, true) == false)
{
LM_W(("Bad Input (invalid payload body)"));
entityErrorPush(errorArray,
entityId,
OrionldBadRequestData,
"invalid payload body",
"no details",
400,
false);
kjChildRemove(entityArray, entityP);
entityP = next;
continue;
}
#endif

entityIdPush(idArray, entityId);

entityP = next;
Expand Down
2 changes: 1 addition & 1 deletion src/lib/orionld/kjTree/kjEntityIdArrayExtract.h
View file
Expand Up @@ -36,6 +36,6 @@ extern "C"
//
// kjEntityIdArrayExtract -
//
extern KjNode* kjEntityIdArrayExtract(KjNode* entityArray, KjNode* successArray, KjNode* errorArray);
extern KjNode* kjEntityIdArrayExtract(KjNode* entityArray, KjNode* errorArray);

#endif // SRC_LIB_ORIONLD_KJTREE_KJENTITYIDARRAYEXTRACT_H_
10 changes: 5 additions & 5 deletions src/lib/orionld/payloadCheck/pcheckEntity.cpp
View file
Expand Up @@ -188,11 +188,11 @@ bool pcheckEntity
//
// Prepare output
//
*locationNodePP = locationNodeP;
*observationSpaceNodePP = observationSpaceNodeP;
*operationSpaceNodePP = operationSpaceNodeP;
*createdAtPP = createdAtP;
*modifiedAtPP = modifiedAtP;
if (locationNodePP != NULL) *locationNodePP = locationNodeP;
if (observationSpaceNodePP != NULL) *observationSpaceNodePP = observationSpaceNodeP;
if (operationSpaceNodePP != NULL) *operationSpaceNodePP = operationSpaceNodeP;
if (createdAtPP != NULL) *createdAtPP = createdAtP;
if (modifiedAtPP != NULL) *modifiedAtPP = modifiedAtP;

return true;
}
2 changes: 1 addition & 1 deletion src/lib/orionld/serviceRoutines/orionldPostBatchUpdate.cpp
View file
Expand Up @@ -104,7 +104,7 @@ bool orionldPostBatchUpdate(ConnectionInfo* ciP)
//
// Create idArray as an array of entity IDs, extracted from orionldState.requestTree
//
KjNode* idArray = kjEntityIdArrayExtract(orionldState.requestTree, successArrayP, errorsArrayP);
KjNode* idArray = kjEntityIdArrayExtract(orionldState.requestTree, errorsArrayP);

//
// 02. Check whether some ID from idArray does not exist - that would be an error for Batch Update
Expand Down
165 changes: 129 additions & 36 deletions src/lib/orionld/serviceRoutines/orionldPostBatchUpsert.cpp
View file
Expand Up @@ -29,7 +29,6 @@ extern "C"
#include "kjson/kjBuilder.h" // kjString, kjObject, ...
#include "kjson/kjLookup.h" // kjLookup
#include "kjson/kjClone.h" // kjClone
#include "kjson/kjRender.h" // kjFastRender
}

#include "logMsg/logMsg.h" // LM_*
Expand Down Expand Up @@ -120,6 +119,43 @@ static void entityTypeAndCreDateGet(KjNode* dbEntityP, char** idP, char** typeP,
}


// ----------------------------------------------------------------------------
//
// entityLookupInDb - lookup an entioty id in an array of entities
//
// idTypeAndCreDateFromDb looks like this:
// [
// {
// "id":"urn:ngsi-ld:entity:E1",
// "type":"https://uri.etsi.org/ngsi-ld/default-context/Vehicle",
// "creDate":1619077285.144012
// },
// {
// "id":"urn:ngsi-ld:entity:E2",
// "type":"https://uri.etsi.org/ngsi-ld/default-context/Vehicle",
// "creDate":1619077285.144012
// }
// ]
static KjNode* entityLookupInDb(KjNode* idTypeAndCreDateFromDb, const char* entityId)
{
if (idTypeAndCreDateFromDb == NULL)
return NULL;

for (KjNode* entityP = idTypeAndCreDateFromDb->value.firstChildP; entityP != NULL; entityP = entityP->next)
{
KjNode* idNodeP = kjLookup(entityP, "id");

if ((idNodeP != NULL) && (idNodeP->type == KjString))
{
if (strcmp(idNodeP->value.s, entityId) == 0)
return entityP;
}
}

return NULL;
}



// ----------------------------------------------------------------------------
//
Expand All @@ -139,7 +175,6 @@ static void entityTypeAndCreDateGet(KjNode* dbEntityP, char** idP, char** typeP,
//
bool orionldPostBatchUpsert(ConnectionInfo* ciP)
{
LM_TMP(("BUPS: In orionldPostBatchUpsert"));
// Error or not, the Link header should never be present in the reponse
orionldState.noLinkHeader = true;

Expand All @@ -166,7 +201,8 @@ bool orionldPostBatchUpsert(ConnectionInfo* ciP)


KjNode* incomingTree = orionldState.requestTree;
KjNode* successArrayP = kjArray(orionldState.kjsonP, "success");
KjNode* createdArrayP = kjArray(orionldState.kjsonP, "created");
KjNode* updatedArrayP = kjArray(orionldState.kjsonP, "updated");
KjNode* errorsArrayP = kjArray(orionldState.kjsonP, "errors");

//
Expand All @@ -176,7 +212,7 @@ bool orionldPostBatchUpsert(ConnectionInfo* ciP)
//
// Create idArray as an array of entity IDs, extracted from orionldState.requestTree
//
KjNode* idArray = kjEntityIdArrayExtract(orionldState.requestTree, successArrayP, errorsArrayP);
KjNode* idArray = kjEntityIdArrayExtract(orionldState.requestTree, errorsArrayP);

//
// 02. Query database extracting three fields: { id, type and creDate } for each of the entities
Expand Down Expand Up @@ -363,17 +399,22 @@ bool orionldPostBatchUpsert(ConnectionInfo* ciP)
// bool partialUpdate = (orionldState.errorAttributeArrayP[0] == 0)? false : true;
// bool retValue = true;
//

if (orionldState.httpStatusCode == SccOk)
{
orionldState.responseTree = kjObject(orionldState.kjsonP, NULL);

for (unsigned int ix = 0; ix < mongoResponse.contextElementResponseVector.vec.size(); ix++)
{
const char* entityId = mongoResponse.contextElementResponseVector.vec[ix]->contextElement.entityId.id.c_str();

if (mongoResponse.contextElementResponseVector.vec[ix]->statusCode.code == SccOk)
entitySuccessPush(successArrayP, entityId);
{
// Creation or Update?
KjNode* dbEntityP = entityLookupInDb(idTypeAndCreDateFromDb, entityId);

if (dbEntityP == NULL)
entitySuccessPush(createdArrayP, entityId);
else
entitySuccessPush(updatedArrayP, entityId);
}
else
entityErrorPush(errorsArrayP,
entityId,
Expand All @@ -388,52 +429,104 @@ bool orionldPostBatchUpsert(ConnectionInfo* ciP)
{
const char* entityId = mongoRequest.contextElementVector.vec[ix]->entityId.id.c_str();

if (kjStringValueLookupInArray(successArrayP, entityId) == NULL)
entitySuccessPush(successArrayP, entityId);
}
// Creation or Update?
KjNode* dbEntityP = entityLookupInDb(idTypeAndCreDateFromDb, entityId);

//
// Add the success/error arrays to the response-tree
//
kjChildAdd(orionldState.responseTree, successArrayP);
kjChildAdd(orionldState.responseTree, errorsArrayP);

orionldState.httpStatusCode = SccOk;
if (dbEntityP == NULL)
{
if (kjStringValueLookupInArray(createdArrayP, entityId) == NULL)
entitySuccessPush(createdArrayP, entityId);
}
else
{
if (kjStringValueLookupInArray(updatedArrayP, entityId) == NULL)
entitySuccessPush(updatedArrayP, entityId);
}
}
}

mongoRequest.release();
mongoResponse.release();
//
// We have entity ids in three arrays:
// errorsArrayP
// updatedArrayP
// createdArrayP
//
// 1. The broker returns 201 if all entities have been created:
// - errorsArrayP EMPTY
// - updatedArrayP EMPTY
// - createdArrayP NOT EMPTY
//
// 2. The broker returns 204 if all entities have been updated:
// - errorsArrayP EMPTY
// - createdArrayP EMPTY
// - updatedArrayP NOT EMPTY
//
// 3. Else, 207 is returned
//
KjNode* successArrayP = NULL;

if (orionldState.httpStatusCode != SccOk)
if ((errorsArrayP->value.firstChildP == NULL) && (updatedArrayP->value.firstChildP == NULL))
{
LM_E(("mongoUpdateContext flagged an error"));
orionldErrorResponseCreate(OrionldBadRequestData, "Internal Error", "Database Error");
orionldState.httpStatusCode = SccReceiverInternalError;
return false;
orionldState.httpStatusCode = 201;
orionldState.responseTree = createdArrayP;
successArrayP = createdArrayP; // for kjEntityArrayErrorPurge
successArrayP->name = (char*) "success";
}
else if (errorsArrayP->value.firstChildP != NULL) // There are entities in error
else if ((errorsArrayP->value.firstChildP == NULL) && (createdArrayP->value.firstChildP == NULL))
{
orionldState.httpStatusCode = 207; // Multi-Status
orionldState.acceptJsonld = false;
orionldState.httpStatusCode = 204;
orionldState.responseTree = NULL;
successArrayP = updatedArrayP; // for kjEntityArrayErrorPurge
successArrayP->name = (char*) "success";
}
else
{
orionldState.httpStatusCode = 207;

//
// FIXME: "idTypeAndCreDateFromDb == NULL" is not enough to decide whether all the entities were created or just updated
// Need a better check (look inside idTypeAndCreDateFromDb and compare with the EIDs in the payload body)
// For v1.3.1, merge updatedArrayP and createdArrayP into successArrayP
// FIXME: This will be amended in the spec and this code will need to change
//
if (idTypeAndCreDateFromDb == NULL) // None of the entities were found in DB - 201 Created
if (updatedArrayP->value.firstChildP != NULL)
{
orionldState.httpStatusCode = 201;
orionldState.responseTree = successArrayP;
successArrayP = updatedArrayP;

if (createdArrayP->value.firstChildP != NULL)
{
// Concatenate arrays
successArrayP->lastChild->next = createdArrayP->value.firstChildP;
successArrayP->lastChild = createdArrayP->lastChild;
}
}
else // All entities were found in DB - 204 No Content
else if (createdArrayP->value.firstChildP != NULL)
{
orionldState.httpStatusCode = 204; // No Content
orionldState.responseTree = NULL;
successArrayP = createdArrayP;

if (updatedArrayP->value.firstChildP != NULL)
{
// Concatenate arrays
successArrayP->lastChild->next = updatedArrayP->value.firstChildP;
successArrayP->lastChild = updatedArrayP->lastChild;
}
}
else
successArrayP = updatedArrayP; // Empty array

successArrayP->name = (char*) "success";

//
// Add the success/error arrays to the response-tree
//
orionldState.responseTree = kjObject(orionldState.kjsonP, NULL);

kjChildAdd(orionldState.responseTree, successArrayP);
kjChildAdd(orionldState.responseTree, errorsArrayP);
}


mongoRequest.release();
mongoResponse.release();

if (troe == true)
kjEntityArrayErrorPurge(orionldState.requestTree, errorsArrayP, successArrayP);

Expand Down
11 changes: 10 additions & 1 deletion test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert-many-geoproperties.test
View file
Expand Up @@ -260,9 +260,18 @@ Date: REGEX(.*)

03. New Batch Upsert, modifying one of the two entities and adding another entity (also with one GeoProperty)
=============================================================================================================
HTTP/1.1 204 No Content
HTTP/1.1 207 Multi-Status
Content-Length: 109
Content-Type: application/json
Date: REGEX(.*)

{
"errors": [],
"success": [
"urn:ngsi-ld:GtfsStation:900000245025:MS",
"urn:ngsi-ld:GtfsStation:900000550091:MS"
]
}


04. GET the entities
Expand Down
11 changes: 10 additions & 1 deletion test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert-options=replace.test
View file
Expand Up @@ -272,9 +272,18 @@ Slept 3 seconds

05. Modify E2 and add E3 using POST /ngsi-ld/v1/entityOperations/upsert?options=replace
=======================================================================================
HTTP/1.1 204 No Content
HTTP/1.1 207 Multi-Status
Content-Length: 73
Content-Type: application/json
Date: REGEX(.*)

{
"errors": [],
"success": [
"urn:ngsi-ld:entity:E2",
"urn:ngsi-ld:entity:E3"
]
}


06. T6: Extract creDate and modDate of E1, E2 and E3 from mongo
Expand Down
11 changes: 10 additions & 1 deletion test/functionalTest/cases/0000_ngsild/ngsild_batch_upsert-options=update.test
View file
Expand Up @@ -141,9 +141,18 @@ Date: REGEX(.*)

03. Modify E2 and add E3 using POST /ngsi-ld/v1/entityOperations/upsert?options=update
======================================================================================
HTTP/1.1 204 No Content
HTTP/1.1 207 Multi-Status
Content-Length: 73
Content-Type: application/json
Date: REGEX(.*)

{
"errors": [],
"success": [
"urn:ngsi-ld:entity:E2",
"urn:ngsi-ld:entity:E3"
]
}


04. GET all entities - see E1 as it was and E2 changed by step 3, and the new E3
Expand Down

0 comments on commit d4d0b95

Please sign in to comment.