From 4fdf9b6cd88ef965224ad0f2e2440a558492bcad Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Tue, 2 Jul 2019 15:23:19 -0300 Subject: [PATCH 01/19] Partial update redis-set response --- .travis.yml | 2 +- README.md | 5 +++ hydra_agent/agent.py | 32 +++++++++++++++++-- hydra_agent/redis_core/graph_init.py | 1 - hydra_agent/redis_core/graphutils.py | 22 +++++++------ .../redis_core/graphutils_operations.py | 5 +-- redis_setup.sh | 6 ++-- requirements.txt | 2 +- 8 files changed, 55 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index b68f227..2b7f567 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: - docker before_script: - - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph:1.2.2 + - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph:edge python: - "3.5" diff --git a/README.md b/README.md index 591476d..dddbb83 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,11 @@ Get all nodes and filter by label: GRAPH.QUERY apigraph "MATCH (p:collection) RETURN p" ``` +Get all nodes and filter by label: +```' +GRAPH.QUERY apigraph "MATCH (p) WHERE(p.id = '/serverapi/DroneCollection/72b53615-a480-4920-b126-4d1e1e107dc6') RETURN p" +``` + To read all the edges of the graph ``` GRAPH.QUERY apigraph "MATCH ()-[r]->() RETURN type(r)" diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index dda5d31..433e77c 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -1,11 +1,12 @@ import logging -from hydra_agent.redis_core.redis_proxy import RedisProxy -from hydra_agent.redis_core.graphutils_operations import GraphOperations -from hydra_agent.redis_core.graph_init import InitialGraph +from redis_core.redis_proxy import RedisProxy +from redis_core.graphutils_operations import GraphOperations +from redis_core.graph_init import InitialGraph from hydra_python_core import doc_maker from typing import Union, Tuple from requests import Session +logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) @@ -92,4 +93,29 @@ def delete(self, url: str) -> dict: return response.json() if __name__ == "__main__": + # GRAPH.QUERY apigraph "MATCH (p:collection {id:'vocab:EntryPoint/DroneCollection'}) RETURN p.members" + # GRAPH.QUERY apigraph "MATCH(p) WHERE(p.id='/serverapi/DroneCollection/d4a8106e-87ab-4f27-ad36-ecae3bca075c') RETURN p" + agent = Agent("http://localhost:8080/serverapi") + logger.info(agent.get("http://localhost:8080/serverapi/DroneCollection/")) + #input(">>>") + + new_object = {"@type": "Drone", "DroneState": "Simplified state", + "name": "Smart Drone", "model": "Hydra Drone", + "MaxSpeed": "999", "Sensor": "Wind"} + print("----PUT-----") + response, new_resource_url = agent.put("http://localhost:8080/serverapi/DroneCollection/", new_object) + print("----GET RESOURCE-----") + logger.info(agent.get(new_resource_url)) + + new_object["name"] = "Updated Name" + del new_object["@id"] + + print("----POST-----") + logger.info(agent.post(new_resource_url, new_object)) + + print("----DELETE-----") + logger.info(agent.delete(new_resource_url)) + + print("----GET COLLECTION-----") + logger.info(agent.get("http://localhost:8080/serverapi/DroneCollection/")) pass diff --git a/hydra_agent/redis_core/graph_init.py b/hydra_agent/redis_core/graph_init.py index d8edb32..dd366ce 100644 --- a/hydra_agent/redis_core/graph_init.py +++ b/hydra_agent/redis_core/graph_init.py @@ -1,4 +1,3 @@ -import redis from redisgraph import Graph, Node import urllib.request import json diff --git a/hydra_agent/redis_core/graphutils.py b/hydra_agent/redis_core/graphutils.py index 5485ee8..6a05976 100644 --- a/hydra_agent/redis_core/graphutils.py +++ b/hydra_agent/redis_core/graphutils.py @@ -1,9 +1,10 @@ import logging -from hydra_agent.redis_core.redis_proxy import RedisProxy +from redis_core.redis_proxy import RedisProxy from redisgraph import Node, Edge, Graph from typing import Union, Optional from redis.exceptions import ResponseError +logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) @@ -130,18 +131,21 @@ def process_result(self, result: list) -> list: """ response_json_list = [] - for record in result.result_set[1:]: + for record in result.result_set[0][:]: new_record = {} - for j, property_x in enumerate(record): + if record is None: + return + properties = record.properties + #properties = {y.replace("b'", "")[:-1]: properties.get(y).replace("b'", "")[:-1] for y in properties.keys() } + properties = {y.replace("b'", ""): properties.get(y) for y in properties.keys() } + string.encode(encoding='UTF-8',errors='strict') + for j, property_x in enumerate(record.properties): if property_x is None: pass else: - property_name = result.result_set[0][j].decode() - try: - node_alias, property_name = property_name.split(".") - new_record[property_name] = property_x.decode() - except ValueError as e: - logger.info("Graph property with no dot/wrong format") + property_name = property_x.decode() + new_record[property_name] = record.properties[j].decode() + if new_record: if 'id' in new_record: new_record['@id'] = new_record.pop('id') diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 50ef596..7c8cf64 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -2,10 +2,11 @@ import json import logging from urllib.error import URLError, HTTPError -from hydra_agent.redis_core.redis_proxy import RedisProxy -from hydra_agent.redis_core.graphutils import GraphUtils +from redis_core.redis_proxy import RedisProxy +from redis_core.graphutils import GraphUtils from redisgraph import Graph, Node +logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) diff --git a/redis_setup.sh b/redis_setup.sh index 6109dee..ebef16c 100755 --- a/redis_setup.sh +++ b/redis_setup.sh @@ -12,12 +12,12 @@ else fi # after getting the docker-ce, check if `redislabs/redisgraph` docker image is not installed then install ii. -if [ -z "$(docker images -q redislabs/redisgraph:1.2.2)" ] +if [ -z "$(docker images -q redislabs/redisgraph:edge)" ] then - echo "Docker already have a redislabs/redisgraph:1.2.2 image" + echo "Docker already have a redislabs/redisgraph:edge image" else - sudo docker run -p 6379:6379 -it --rm redislabs/redisgraph:1.2.2 + sudo docker run -p 6379:6379 -it --rm redislabs/redisgraph:edge fi # command for run the server # sudo docker run -p 6379:6379 -it --rm redislabs/redisgraph # uncomment this line if you want to run server without using dockerflie. diff --git a/requirements.txt b/requirements.txt index d8c2c78..b1cacb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ rdflib-jsonld uritemplate httplib2 redis==2.10.6 -redisgraph==1.7 graphviz requests +-e git+https://github.com/RedisGraph/redisgraph-py@62c591da4ec54d268f3abde3561d378de1763746#egg=redisgraph -e git+https://github.com/HTTP-APIs/hydra-python-core@v0.1#egg=hydra_python_core From 155c0b7473d91dcac24120a50c550a0dd668652e Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 15:57:40 -0300 Subject: [PATCH 02/19] Upgrading packages Version Redis, Redis Graph and Hydra Python Core with the proper readme observation and so on --- .travis.yml | 2 +- README.md | 4 +- docker-compose.yaml | 2 +- .../tests/test_examples/hydra_doc_sample.py | 287 +++++++++++------- requirements.txt | 6 +- 5 files changed, 179 insertions(+), 122 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b7f567..b8d49ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ python: - "3.6-dev" # 3.6 development branch - "3.7-dev" install: - - pip install -e git+https://github.com/HTTP-APIs/hydra-python-core@v0.1#egg=hydra_python_core + - pip install -e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core - pip install -e . script: python -m unittest discover diff --git a/README.md b/README.md index dddbb83..71a9035 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ After setup the environment. You can query or run the client. docker-compose run client - and provide a valid URL and then you can query in querying format. + and provide a valid URL(of a hydrus updated server) and then you can query in querying format. `>>>url` #here url should be a valid link, for testing you can use http://35.224.198.158:8080/api `>>>help` # it will provide the querying format @@ -148,7 +148,7 @@ from hydra_agent.agent import Agent agent = Agent("http://localhost:8080/serverapi") agent.get("http://localhost:8080/serverapi/DroneCollection/123-123-123-123") ``` - +**Remember that it's important you use the an updated hydrus version, currently the one under the develop branch** The agent supports GET, PUT, POST or DELETE: - GET - used to READ resources or collections diff --git a/docker-compose.yaml b/docker-compose.yaml index d5b4152..83cfa71 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,6 +14,6 @@ services: environment: - REDIS_HOST=redis_db db: - image: redislabs/redisgraph + image: redislabs/redisgraph:edge ports: - "6379:6379" diff --git a/hydra_agent/tests/test_examples/hydra_doc_sample.py b/hydra_agent/tests/test_examples/hydra_doc_sample.py index 5cb99eb..3ea8efc 100644 --- a/hydra_agent/tests/test_examples/hydra_doc_sample.py +++ b/hydra_agent/tests/test_examples/hydra_doc_sample.py @@ -62,12 +62,14 @@ "method": "GET", "possibleStatus": [ { - "description": "State not found", - "statusCode": 404 + "title": "State not found", + "statusCode": 404, + "description": "" }, { - "description": "State Returned", - "statusCode": 200 + "title": "State Returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:State", @@ -137,12 +139,14 @@ "method": "GET", "possibleStatus": [ { - "description": "Command not found", - "statusCode": 404 + "title": "Command not found", + "statusCode": 404, + "description": "" }, { - "description": "Command Returned", - "statusCode": 200 + "title": "Command Returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:Command", @@ -154,8 +158,9 @@ "method": "PUT", "possibleStatus": [ { - "description": "Command added", - "statusCode": 201 + "title": "Command added", + "statusCode": 201, + "description": "" } ], "returns": "null", @@ -167,8 +172,9 @@ "method": "DELETE", "possibleStatus": [ { - "description": "Command deleted", - "statusCode": 201 + "title": "Command deleted", + "statusCode": 201, + "description": "" } ], "returns": "null", @@ -206,12 +212,14 @@ "method": "GET", "possibleStatus": [ { - "description": "Message not found", - "statusCode": 404 + "title": "Message not found", + "statusCode": 404, + "description": "" }, { - "description": "Message returned", - "statusCode": 200 + "title": "Message returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:Message", @@ -223,8 +231,9 @@ "method": "DELETE", "possibleStatus": [ { - "description": "Message deleted", - "statusCode": 200 + "title": "Message deleted", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -254,8 +263,9 @@ "method": "POST", "possibleStatus": [ { - "description": "Area of interest changed", - "statusCode": 200 + "title": "Area of interest changed", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -267,12 +277,14 @@ "method": "GET", "possibleStatus": [ { - "description": "Area of interest not found", - "statusCode": 404 + "title": "Area of interest not found", + "statusCode": 404, + "description": "" }, { - "description": "Area of interest returned", - "statusCode": 200 + "title": "Area of interest returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:Area", @@ -310,12 +322,14 @@ "method": "GET", "possibleStatus": [ { - "description": "Data not found", - "statusCode": 404 + "title": "Data not found", + "statusCode": 404, + "description": "" }, { - "description": "Data returned", - "statusCode": 200 + "title": "Data returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:Datastream", @@ -327,8 +341,9 @@ "method": "POST", "possibleStatus": [ { - "description": "Data updated", - "statusCode": 200 + "title": "Data updated", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -340,8 +355,9 @@ "method": "DELETE", "possibleStatus": [ { - "description": "Data deleted", - "statusCode": 200 + "title": "Data deleted", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -387,8 +403,9 @@ "method": "POST", "possibleStatus": [ { - "description": "Drone updated", - "statusCode": 200 + "title": "Drone updated", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -400,8 +417,9 @@ "method": "PUT", "possibleStatus": [ { - "description": "Drone added", - "statusCode": 200 + "title": "Drone added", + "statusCode": 200, + "description": "" } ], "returns": "null", @@ -413,16 +431,37 @@ "method": "GET", "possibleStatus": [ { - "description": "Drone not found", - "statusCode": 404 + "title": "Drone not found", + "statusCode": 404, + "description": "" }, { - "description": "Drone Returned", - "statusCode": 200 + "title": "Drone Returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:Drone", "title": "GetDrone" + }, + { + "@type": "http://schema.org/DeleteAction", + "expects": "null", + "method": "DELETE", + "possibleStatus": [ + { + "title": "Drone not found", + "statusCode": 404, + "description": "" + }, + { + "title": "Drone successfully deleted", + "statusCode": 200, + "description": "" + } + ], + "returns": "null", + "title": "DeleteDrone" } ], "supportedProperty": [ @@ -480,12 +519,14 @@ "method": "GET", "possibleStatus": [ { - "description": "Log entry not found", - "statusCode": 404 + "title": "Log entry not found", + "statusCode": 404, + "description": "" }, { - "description": "Log entry returned", - "statusCode": 200 + "title": "Log entry returned", + "statusCode": 200, + "description": "" } ], "returns": "vocab:LogEntry", @@ -497,8 +538,9 @@ "method": "PUT", "possibleStatus": [ { - "description": "Log entry created", - "statusCode": 201 + "title": "Log entry created", + "statusCode": 201, + "description": "" } ], "returns": "null", @@ -603,19 +645,20 @@ "expects": "null", "method": "GET", "returns": "vocab:CommandCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:command_create", "@type": "http://schema.org/AddAction", - "description": "Create new Command entitity", + "description": "Create new Command entity", "expects": "vocab:Command", "method": "PUT", "returns": "vocab:Command", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Command entity was created successfully.", - "statusCode": 201 + "title": "If the Command entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -646,19 +689,20 @@ "expects": "null", "method": "GET", "returns": "vocab:StateCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:state_create", "@type": "http://schema.org/AddAction", - "description": "Create new State entitity", + "description": "Create new State entity", "expects": "vocab:State", "method": "PUT", "returns": "vocab:State", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the State entity was created successfully.", - "statusCode": 201 + "title": "If the State entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -689,19 +733,20 @@ "expects": "null", "method": "GET", "returns": "vocab:MessageCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:message_create", "@type": "http://schema.org/AddAction", - "description": "Create new Message entitity", + "description": "Create new Message entity", "expects": "vocab:Message", "method": "PUT", "returns": "vocab:Message", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Message entity was created successfully.", - "statusCode": 201 + "title": "If the Message entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -732,19 +777,20 @@ "expects": "null", "method": "GET", "returns": "vocab:DroneCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:drone_create", "@type": "http://schema.org/AddAction", - "description": "Create new Drone entitity", + "description": "Create new Drone entity", "expects": "vocab:Drone", "method": "PUT", "returns": "vocab:Drone", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Drone entity was created successfully.", - "statusCode": 201 + "title": "If the Drone entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -775,19 +821,20 @@ "expects": "null", "method": "GET", "returns": "vocab:LogEntryCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:logentry_create", "@type": "http://schema.org/AddAction", - "description": "Create new LogEntry entitity", + "description": "Create new LogEntry entity", "expects": "vocab:LogEntry", "method": "PUT", "returns": "vocab:LogEntry", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the LogEntry entity was created successfully.", - "statusCode": 201 + "title": "If the LogEntry entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -818,19 +865,20 @@ "expects": "null", "method": "GET", "returns": "vocab:DatastreamCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:datastream_create", "@type": "http://schema.org/AddAction", - "description": "Create new Datastream entitity", + "description": "Create new Datastream entity", "expects": "vocab:Datastream", "method": "PUT", "returns": "vocab:Datastream", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Datastream entity was created successfully.", - "statusCode": 201 + "title": "If the Datastream entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -860,7 +908,7 @@ "expects": "null", "method": "GET", "returns": "null", - "statusCodes": "vocab:EntryPoint" + "possibleStatus": "vocab:EntryPoint" } ], "supportedProperty": [ @@ -883,10 +931,11 @@ "label": "UpdateArea", "method": "POST", "returns": "null", - "statusCodes": [ + "possibleStatus": [ { - "description": "Area of interest changed", - "statusCode": 200 + "title": "Area of interest changed", + "statusCode": 200, + "description": "" } ] }, @@ -898,14 +947,16 @@ "label": "GetArea", "method": "GET", "returns": "vocab:Area", - "statusCodes": [ + "possibleStatus": [ { - "description": "Area of interest not found", - "statusCode": 404 + "title": "Area of interest not found", + "statusCode": 404, + "description": "" }, { - "description": "Area of interest returned", - "statusCode": 200 + "title": "Area of interest returned", + "statusCode": 200, + "description": "" } ] } @@ -933,19 +984,20 @@ "expects": "null", "method": "GET", "returns": "vocab:CommandCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:command_create", "@type": "http://schema.org/AddAction", - "description": "Create new Command entitity", + "description": "Create new Command entity", "expects": "vocab:Command", "method": "PUT", "returns": "vocab:Command", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Command entity was created successfully.", - "statusCode": 201 + "title": "If the Command entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -973,19 +1025,20 @@ "expects": "null", "method": "GET", "returns": "vocab:StateCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:state_create", "@type": "http://schema.org/AddAction", - "description": "Create new State entitity", + "description": "Create new State entity", "expects": "vocab:State", "method": "PUT", "returns": "vocab:State", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the State entity was created successfully.", - "statusCode": 201 + "title": "If the State entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -1013,19 +1066,20 @@ "expects": "null", "method": "GET", "returns": "vocab:MessageCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:message_create", "@type": "http://schema.org/AddAction", - "description": "Create new Message entitity", + "description": "Create new Message entity", "expects": "vocab:Message", "method": "PUT", "returns": "vocab:Message", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Message entity was created successfully.", - "statusCode": 201 + "title": "If the Message entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -1053,19 +1107,20 @@ "expects": "null", "method": "GET", "returns": "vocab:DroneCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:drone_create", "@type": "http://schema.org/AddAction", - "description": "Create new Drone entitity", + "description": "Create new Drone entity", "expects": "vocab:Drone", "method": "PUT", "returns": "vocab:Drone", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Drone entity was created successfully.", - "statusCode": 201 + "title": "If the Drone entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -1093,19 +1148,20 @@ "expects": "null", "method": "GET", "returns": "vocab:LogEntryCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:logentry_create", "@type": "http://schema.org/AddAction", - "description": "Create new LogEntry entitity", + "description": "Create new LogEntry entity", "expects": "vocab:LogEntry", "method": "PUT", "returns": "vocab:LogEntry", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the LogEntry entity was created successfully.", - "statusCode": 201 + "title": "If the LogEntry entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -1133,7 +1189,7 @@ "expects": "null", "method": "GET", "returns": "vocab:DatastreamCollection", - "statusCodes": [] + "possibleStatus": [] }, { "@id": "_:datastream_create", @@ -1142,10 +1198,11 @@ "expects": "vocab:Datastream", "method": "PUT", "returns": "vocab:Datastream", - "statusCodes": [ + "possibleStatus": [ { - "description": "If the Datastream entity was created successfully.", - "statusCode": 201 + "title": "If the Datastream entity was created successfully.", + "statusCode": 201, + "description": "" } ] } @@ -1160,4 +1217,4 @@ } ], "title": "API Doc for the server side API" -} +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b1cacb2..4cdb9f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ rdflib rdflib-jsonld uritemplate httplib2 -redis==2.10.6 +redis graphviz requests --e git+https://github.com/RedisGraph/redisgraph-py@62c591da4ec54d268f3abde3561d378de1763746#egg=redisgraph --e git+https://github.com/HTTP-APIs/hydra-python-core@v0.1#egg=hydra_python_core +-e git+https://github.com/RedisGraph/redisgraph-py#egg=redisgraph +-e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core From 01ea9218dd90280697415b0dcc85ffadc2b7847f Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 16:04:22 -0300 Subject: [PATCH 03/19] Response processing modifications --- hydra_agent/agent.py | 6 +++--- hydra_agent/redis_core/graphutils.py | 17 +++++------------ .../redis_core/graphutils_operations.py | 18 +++++++++--------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 433e77c..53c4c0e 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -1,7 +1,7 @@ import logging -from redis_core.redis_proxy import RedisProxy -from redis_core.graphutils_operations import GraphOperations -from redis_core.graph_init import InitialGraph +from hydra_agent.redis_core.redis_proxy import RedisProxy +from hydra_agent.redis_core.graphutils_operations import GraphOperations +from hydra_agent.redis_core.graph_init import InitialGraph from hydra_python_core import doc_maker from typing import Union, Tuple from requests import Session diff --git a/hydra_agent/redis_core/graphutils.py b/hydra_agent/redis_core/graphutils.py index 6a05976..dc6b9b3 100644 --- a/hydra_agent/redis_core/graphutils.py +++ b/hydra_agent/redis_core/graphutils.py @@ -1,5 +1,5 @@ import logging -from redis_core.redis_proxy import RedisProxy +from hydra_agent.redis_core.redis_proxy import RedisProxy from redisgraph import Node, Edge, Graph from typing import Union, Optional from redis.exceptions import ResponseError @@ -131,21 +131,14 @@ def process_result(self, result: list) -> list: """ response_json_list = [] + if not result.result_set: + return [] + for record in result.result_set[0][:]: new_record = {} if record is None: return - properties = record.properties - #properties = {y.replace("b'", "")[:-1]: properties.get(y).replace("b'", "")[:-1] for y in properties.keys() } - properties = {y.replace("b'", ""): properties.get(y) for y in properties.keys() } - string.encode(encoding='UTF-8',errors='strict') - for j, property_x in enumerate(record.properties): - if property_x is None: - pass - else: - property_name = property_x.decode() - new_record[property_name] = record.properties[j].decode() - + new_record = record.properties if new_record: if 'id' in new_record: new_record['@id'] = new_record.pop('id') diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 7c8cf64..fcfbc96 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -2,8 +2,8 @@ import json import logging from urllib.error import URLError, HTTPError -from redis_core.redis_proxy import RedisProxy -from redis_core.graphutils import GraphUtils +from hydra_agent.redis_core.redis_proxy import RedisProxy +from hydra_agent.redis_core.graphutils import GraphUtils from redisgraph import Graph, Node logging.basicConfig(level=logging.INFO) @@ -41,13 +41,13 @@ def get_processing(self, url: str, resource: dict) -> None: collection_members = self.graph_utils.read( match=":collection", where="id='{}'".format(redis_collection_id), - ret=".members") + ret="") # Checking if it's the first member to be loaded - if collection_members is None: + if not 'members' in collection_members: collection_members = [] else: - collection_members = eval(collection_members[0]['members']) + collection_members = eval(collection_members['members']) collection_members.append({'@id': resource['@id'], '@type': resource['@type']}) @@ -142,13 +142,13 @@ def delete_processing(self, url: str) -> None: collection_members = self.graph_utils.read( match=":collection", where="id='{}'".format(redis_collection_id), - ret=".members") + ret="") # Checking if it's the first member to be loaded - if collection_members is None: - return + if not 'members' in collection_members: + collection_members = [] else: - collection_members = eval(collection_members[0]['members']) + collection_members = eval(collection_members['members']) for member in collection_members: if resource_id in member['@id']: From a9a4119bd677f7190d3cc583cb24536b4559218f Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 16:07:11 -0300 Subject: [PATCH 04/19] Improving error handling and adjusting tests --- hydra_agent/agent.py | 47 ++++++++------------------- hydra_agent/redis_core/redis_proxy.py | 3 +- hydra_agent/tests/test_agent.py | 4 +-- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 53c4c0e..51e36c9 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -49,8 +49,9 @@ def get(self, url: str) -> Union[dict, list]: if response.status_code == 200: self.graph_operations.get_processing(url, response.json()) - - return response.json() + return response.json() + else: + return response.text def put(self, url: str, new_object: dict) -> Tuple[dict, str]: """CREATE resource in the Server/cache it on Redis @@ -64,8 +65,8 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]: url = response.headers['Location'] self.graph_operations.put_processing(url, new_object) return response.json(), url - - return response.json(), "" + else: + return response.text, "" def post(self, url: str, updated_object: dict) -> dict: """UPDATE resource in the Server/cache it on Redis @@ -77,8 +78,9 @@ def post(self, url: str, updated_object: dict) -> dict: if response.status_code == 200: self.graph_operations.post_processing(url, updated_object) - - return response.json() + return response.json() + else: + return response.text def delete(self, url: str) -> dict: """DELETE resource in the Server/delete it on Redis @@ -89,33 +91,10 @@ def delete(self, url: str) -> dict: if response.status_code == 200: self.graph_operations.delete_processing(url) - - return response.json() + return response.json() + else: + return response.text + if __name__ == "__main__": - # GRAPH.QUERY apigraph "MATCH (p:collection {id:'vocab:EntryPoint/DroneCollection'}) RETURN p.members" - # GRAPH.QUERY apigraph "MATCH(p) WHERE(p.id='/serverapi/DroneCollection/d4a8106e-87ab-4f27-ad36-ecae3bca075c') RETURN p" - agent = Agent("http://localhost:8080/serverapi") - logger.info(agent.get("http://localhost:8080/serverapi/DroneCollection/")) - #input(">>>") - - new_object = {"@type": "Drone", "DroneState": "Simplified state", - "name": "Smart Drone", "model": "Hydra Drone", - "MaxSpeed": "999", "Sensor": "Wind"} - print("----PUT-----") - response, new_resource_url = agent.put("http://localhost:8080/serverapi/DroneCollection/", new_object) - print("----GET RESOURCE-----") - logger.info(agent.get(new_resource_url)) - - new_object["name"] = "Updated Name" - del new_object["@id"] - - print("----POST-----") - logger.info(agent.post(new_resource_url, new_object)) - - print("----DELETE-----") - logger.info(agent.delete(new_resource_url)) - - print("----GET COLLECTION-----") - logger.info(agent.get("http://localhost:8080/serverapi/DroneCollection/")) - pass + pass \ No newline at end of file diff --git a/hydra_agent/redis_core/redis_proxy.py b/hydra_agent/redis_core/redis_proxy.py index df7cf2f..1fb7f9a 100644 --- a/hydra_agent/redis_core/redis_proxy.py +++ b/hydra_agent/redis_core/redis_proxy.py @@ -12,7 +12,8 @@ class RedisProxy: def __init__(self): host = os.getenv("REDIS_HOST", "localhost") - self.connection = redis.StrictRedis(host=host, port=6379, db=0) + self.connection = redis.StrictRedis(host=host, port=6379, db=0, + decode_responses=True) def get_connection(self): return self.connection diff --git a/hydra_agent/tests/test_agent.py b/hydra_agent/tests/test_agent.py index 6bfba2d..bb77a30 100644 --- a/hydra_agent/tests/test_agent.py +++ b/hydra_agent/tests/test_agent.py @@ -117,11 +117,11 @@ def test_delete(self, put_session_mock, delete_session_mock, delete_session_mock.return_value.json.return_value = {"msg": "success"} response = self.agent.delete(new_object_url) - get_session_mock.return_value.json.return_value = {"msg": "success"} + get_session_mock.return_value.text = {"msg": "resource doesn't exist"} get_new_object = self.agent.get(new_object_url) # Assert if nothing different was returned by Redis - self.assertEqual(get_new_object, {"msg": "success"}) + self.assertEqual(get_new_object, {"msg": "resource doesn't exist"}) if __name__ == "__main__": unittest.main() From dbc8dcf3df32d14b99f897e7b726170f711fbad7 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 17:37:28 -0300 Subject: [PATCH 05/19] Adjusting RedisGraph installation for travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b8d49ab..304a573 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ python: - "3.6-dev" # 3.6 development branch - "3.7-dev" install: + - pip install -e git+https://github.com/RedisGraph/redisgraph-py#egg=redisgraph - pip install -e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core - pip install -e . From 7a629fb5346c0c278396384ad8d6adb15f487633 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 17:12:52 -0300 Subject: [PATCH 06/19] Setting RedisGraph version for stability --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4cdb9f7..0a54c4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ httplib2 redis graphviz requests --e git+https://github.com/RedisGraph/redisgraph-py#egg=redisgraph +-e git+https://github.com/RedisGraph/redisgraph-py@62c591da4ec54d268f3abde3561d378de1763746#egg=redisgraph -e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core From de79177e9f257c62e3e56c0babc57485532792ed Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Tue, 2 Jul 2019 16:39:07 -0300 Subject: [PATCH 07/19] Fixing duplicate Nodes on commit --- hydra_agent/redis_core/graphutils.py | 7 ++++--- hydra_agent/redis_core/graphutils_operations.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hydra_agent/redis_core/graphutils.py b/hydra_agent/redis_core/graphutils.py index dc6b9b3..3665abf 100644 --- a/hydra_agent/redis_core/graphutils.py +++ b/hydra_agent/redis_core/graphutils.py @@ -120,9 +120,10 @@ def add_edge(self, source_node: Node, predicate: str, edge = Edge(source_node, predicate, dest_node) self.redis_graph.add_edge(edge) - def commit(self) -> None: - """Commit the changes made to the Graph to Redis""" - self.redis_graph.commit() + def flush(self) -> None: + """Commit the changes made to the Graph to Redis and reset/flush + the Nodes and Edges to be added in the next commit""" + self.redis_graph.flush() def process_result(self, result: list) -> list: """ diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index fcfbc96..c1276e9 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -61,7 +61,8 @@ def get_processing(self, url: str, resource: dict) -> None: self.graph_utils.add_node("objects" + resource['@type'], resource['@type'] + resource_id, resource) - self.graph_utils.commit() + # Commits the graph + self.graph_utils.flush() # Creating relation between collection node and member self.graph_utils.create_relation(label_source="collection", From f70fbb33d544ba413bc8bd5c42286a099be57fdd Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Tue, 2 Jul 2019 18:48:20 -0300 Subject: [PATCH 08/19] Changing old mechanism for avoiding duplicates --- hydra_agent/querying_mechanism.py | 2 ++ hydra_agent/redis_core/classes_objects.py | 7 +------ hydra_agent/redis_core/collections_endpoint.py | 7 +------ 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/hydra_agent/querying_mechanism.py b/hydra_agent/querying_mechanism.py index 17876c7..d741f6a 100644 --- a/hydra_agent/querying_mechanism.py +++ b/hydra_agent/querying_mechanism.py @@ -62,6 +62,8 @@ def show_data(self, get_data): if count % 2 != 0: for obj1 in objects: for obj in obj1: + if obj is None: + continue string = obj.decode('utf-8') map_string = map(str.strip, string.split(',')) property_list = list(map_string) diff --git a/hydra_agent/redis_core/classes_objects.py b/hydra_agent/redis_core/classes_objects.py index 824a9de..d743594 100644 --- a/hydra_agent/redis_core/classes_objects.py +++ b/hydra_agent/redis_core/classes_objects.py @@ -179,13 +179,8 @@ def load_from_server( endpoint_property, no_endpoint_property, api_doc) - # delete all the old data that has saved in Redis using redis_graph. - # It will remove duplicate data from Redis. - for key in redis_connection.keys(): - if "fs:" not in key.decode("utf8"): - redis_connection.delete(key) # save the new data. - self.redis_graph.commit() + self.redis_graph.flush() def endpointclasses( self, diff --git a/hydra_agent/redis_core/collections_endpoint.py b/hydra_agent/redis_core/collections_endpoint.py index 554dd3d..2f5b29c 100644 --- a/hydra_agent/redis_core/collections_endpoint.py +++ b/hydra_agent/redis_core/collections_endpoint.py @@ -198,13 +198,8 @@ def load_from_server( url, redis_connection ) - # delete all the old data that has saved in Redis using redis_graph. - # It will remove duplicate data from Redis. - for key in redis_connection.keys(): - if "fs:" not in key.decode("utf8"): - redis_connection.delete(key) # save the new data. - self.redis_graph.commit() + self.redis_graph.flush() # for node in self.redis_graph.nodes.values(): # print("\n",node.alias) From be308e5ab2aada6ebffa9192f70643a4b3624791 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Wed, 3 Jul 2019 12:21:00 -0300 Subject: [PATCH 09/19] Adding api_doc to graph_operations --- hydra_agent/agent.py | 5 +++-- hydra_agent/redis_core/graphutils_operations.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 51e36c9..57a491a 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -19,11 +19,12 @@ def __init__(self, entrypoint_url: str) -> None: self.entrypoint_url = entrypoint_url.strip().rstrip('/') self.redis_proxy = RedisProxy() self.redis_connection = self.redis_proxy.get_connection() - self.graph_operations = GraphOperations(entrypoint_url, - self.redis_proxy) super().__init__() jsonld_api_doc = super().get(self.entrypoint_url + '/vocab').json() self.api_doc = doc_maker.create_doc(jsonld_api_doc) + self.graph_operations = GraphOperations(entrypoint_url, + self.api_doc, + self.redis_proxy) self.initialize_graph() def initialize_graph(self) -> None: diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index c1276e9..6e6925b 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -12,8 +12,10 @@ class GraphOperations(): - def __init__(self, entrypoint_url: str, redis_proxy: RedisProxy): + def __init__(self, entrypoint_url: str, api_doc: dict, + redis_proxy: RedisProxy): self.entrypoint_url = entrypoint_url + self.api_doc = api_doc self.redis_proxy = redis_proxy self.redis_connection = redis_proxy.get_connection() self.vocabulary = 'vocab' From e289ee404f15851fced1ad1b41c987d505b41a75 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Wed, 3 Jul 2019 12:21:45 -0300 Subject: [PATCH 10/19] Creating embedded_resource function for processing --- .../redis_core/graphutils_operations.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 6e6925b..749e758 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -5,6 +5,7 @@ from hydra_agent.redis_core.redis_proxy import RedisProxy from hydra_agent.redis_core.graphutils import GraphUtils from redisgraph import Graph, Node +from requests import Session logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) @@ -21,6 +22,7 @@ def __init__(self, entrypoint_url: str, api_doc: dict, self.vocabulary = 'vocab' self.graph_utils = GraphUtils(redis_proxy) self.redis_graph = Graph("apigraph", self.redis_connection) + self.session = Session() def get_processing(self, url: str, resource: dict) -> None: """Synchronize Redis upon new GET operations @@ -195,5 +197,36 @@ def get_resource(self, url: str) -> dict: return resource + def embedded_resource(self, parent_id: str, parent_type: str, + discovered_url: str) -> str: + """Checks for existance of discovered resource and creates links + for embedded resources inside other resources properties + :parent_id: Resource ID for the parent node that had this reference + :parent_type: Resource Type for the parent node that had this reference + :discovered_url: URL Reference for resource found inside a property + """ + resource = self.get_resource(discovered_url) + if resource is None: + response = self.session.get(discovered_url) + if response.status_code == 200: + resource = response.json() + self.get_processing(discovered_url, resource) + else: + logger.info("Embedded link for resource cannot be fetched") + return + + # Creating relation between collection node and member + response = self.graph_utils.create_relation(label_source="objects" + + parent_type, + where_source="id : \'" + + parent_id + "\'", + relation_type="has_" + + resource['@type'], + label_dest="objects" + + resource['@type'], + where_dest="id : \'" + + resource['@id'] + "\'") + return str(response) + if __name__ == "__main__": pass From 697246ac4bc539e28aacf24f9264c5e76d0d2632 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Wed, 3 Jul 2019 12:22:53 -0300 Subject: [PATCH 11/19] Removing small unnecessary double processing --- hydra_agent/redis_core/graphutils_operations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 749e758..e78a33a 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -30,12 +30,12 @@ def get_processing(self, url: str, resource: dict) -> None: :param resource: Resource object fetched from server. :return: None. """ - url = url.rstrip('/').replace(self.entrypoint_url, "EntryPoint") - url_list = url.split('/') + url_list = url.rstrip('/').replace(self.entrypoint_url, "EntryPoint") + url_list = url_list.split('/') # Updating Redis # First case - When processing a GET for a resource if len(url_list) == 3: - entrypoint, resource_endpoint, resource_id = url.split('/') + entrypoint, resource_endpoint, resource_id = url_list # Building the the collection id, i.e. vocab:Entrypoint/Collection redis_collection_id = self.vocabulary + \ @@ -81,7 +81,7 @@ def get_processing(self, url: str, resource: dict) -> None: return # Second Case - When processing a GET for a Collection elif len(url_list) == 2: - entrypoint, resource_endpoint = url.split('/') + entrypoint, resource_endpoint = url_list redis_collection_id = self.vocabulary + \ ":" + entrypoint + \ "/" + resource_endpoint From 0b0c116998222ac169f677c30a541877199786b8 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Wed, 3 Jul 2019 12:24:14 -0300 Subject: [PATCH 12/19] Call embedded_resource when processing resources --- hydra_agent/redis_core/graphutils_operations.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index e78a33a..197bf4f 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -78,6 +78,21 @@ def get_processing(self, url: str, resource: dict) -> None: resource['@type'], where_dest="id : \'" + resource['@id'] + "\'") + + # Checking for embedded resources in the properties of resource + class_doc = self.api_doc.parsed_classes[resource['@type']]['class'] + supported_properties = class_doc.supportedProperty + for supported_prop in supported_properties: + if (self.vocabulary + ":") in str(supported_prop.prop): + if resource[supported_prop.title]: + discovered_url = self.entrypoint_url.replace( + self.api_doc.entrypoint.api, "").rstrip("/") + discovered_url = discovered_url + \ + resource[supported_prop.title] + self.embedded_resource(resource['@id'], + resource['@type'], + discovered_url) + return # Second Case - When processing a GET for a Collection elif len(url_list) == 2: From c5bdfa84b85d5f24761419cd03c9edfb04f6ce8b Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Wed, 3 Jul 2019 14:47:07 -0300 Subject: [PATCH 13/19] Adjusting Agent tests Failling before beucase it would try to fetch hydrus for internal resouces if doc has a property that has an URL --- hydra_agent/tests/test_agent.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/hydra_agent/tests/test_agent.py b/hydra_agent/tests/test_agent.py index bb77a30..ef3936f 100644 --- a/hydra_agent/tests/test_agent.py +++ b/hydra_agent/tests/test_agent.py @@ -42,9 +42,9 @@ def test_get(self, get_processing_mock, get_session_mock): "api/DroneCollection/1", mock_dict) self.assertEqual(response, mock_dict) - + @patch('hydra_agent.redis_core.graphutils_operations.Session.get') @patch('hydra_agent.agent.Session.put') - def test_put(self, put_session_mock): + def test_put(self, put_session_mock, embedded_get_mock): """Tests put method from the Agent :param put_session_mock: MagicMock object for patching session.put """ @@ -58,15 +58,22 @@ def test_put(self, put_session_mock): put_session_mock.return_value.status_code = 201 put_session_mock.return_value.json.return_value = new_object put_session_mock.return_value.headers = {'Location': new_object_url} + + # Mocking an object to be used for a property that has an embedded link + embedded_get_mock.return_value.status_code = 200 + embedded_get_mock.return_value.json.return_value = new_object + response, new_object_url = self.agent.put(collection_url, new_object) # Assert if object was inserted queried and inserted successfully get_new_object = self.agent.get(new_object_url) self.assertEqual(get_new_object, new_object) + @patch('hydra_agent.redis_core.graphutils_operations.Session.get') @patch('hydra_agent.agent.Session.post') @patch('hydra_agent.agent.Session.put') - def test_post(self, put_session_mock, post_session_mock): + def test_post(self, put_session_mock, post_session_mock, + embedded_get_mock): """Tests post method from the Agent :param put_session_mock: MagicMock object for patching session.put :param post_session_mock: MagicMock object for patching session.post @@ -86,6 +93,11 @@ def test_post(self, put_session_mock, post_session_mock): post_session_mock.return_value.status_code = 200 post_session_mock.return_value.json.return_value = {"msg": "success"} new_object['name'] = "Updated Name" + + # Mocking an object to be used for a property that has an embedded link + embedded_get_mock.return_value.status_code = 200 + embedded_get_mock.return_value.json.return_value = new_object + response = self.agent.post(new_object_url, new_object) # Assert if object was updated successfully as intended From c8f1230e774a899518fa26342d7eded596f863c5 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Thu, 4 Jul 2019 19:45:17 -0300 Subject: [PATCH 14/19] Partial segregation solution --- hydra_agent/agent.py | 12 +++++++- .../redis_core/graphutils_operations.py | 28 +++++++++---------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 57a491a..c161426 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -49,7 +49,17 @@ def get(self, url: str) -> Union[dict, list]: response = super().get(url) if response.status_code == 200: - self.graph_operations.get_processing(url, response.json()) + # Graph operations returns embedded resources if finding any + embedded_resources = \ + self.graph_operations.get_processing(url, response.json()) + # Embedded resources are fetched and then properly linked + for embedded_resource in embedded_resources: + self.get(embedded_resource['embedded_url']) + self.graph_operations.embedded_resource( + embedded_resource['parent_id'], + embedded_resource['parent_type'], + embedded_resource['embedded_url'] + ) return response.json() else: return response.text diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 197bf4f..07a81b3 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -24,11 +24,11 @@ def __init__(self, entrypoint_url: str, api_doc: dict, self.redis_graph = Graph("apigraph", self.redis_connection) self.session = Session() - def get_processing(self, url: str, resource: dict) -> None: + def get_processing(self, url: str, resource: dict) -> list: """Synchronize Redis upon new GET operations :param url: Resource URL to be updated in Redis. :param resource: Resource object fetched from server. - :return: None. + :return: list of embedded resources to be fetched. """ url_list = url.rstrip('/').replace(self.entrypoint_url, "EntryPoint") url_list = url_list.split('/') @@ -82,18 +82,21 @@ def get_processing(self, url: str, resource: dict) -> None: # Checking for embedded resources in the properties of resource class_doc = self.api_doc.parsed_classes[resource['@type']]['class'] supported_properties = class_doc.supportedProperty + embedded_resources = [] for supported_prop in supported_properties: if (self.vocabulary + ":") in str(supported_prop.prop): if resource[supported_prop.title]: + new_resource = {} discovered_url = self.entrypoint_url.replace( self.api_doc.entrypoint.api, "").rstrip("/") discovered_url = discovered_url + \ - resource[supported_prop.title] - self.embedded_resource(resource['@id'], - resource['@type'], - discovered_url) + resource[supported_prop.title] + new_resource['parent_id'] = resource['@id'] + new_resource['parent_type'] = resource['@type'] + new_resource['embedded_url'] = discovered_url + embedded_resources.append(new_resource) - return + return embedded_resources # Second Case - When processing a GET for a Collection elif len(url_list) == 2: entrypoint, resource_endpoint = url_list @@ -222,13 +225,10 @@ def embedded_resource(self, parent_id: str, parent_type: str, """ resource = self.get_resource(discovered_url) if resource is None: - response = self.session.get(discovered_url) - if response.status_code == 200: - resource = response.json() - self.get_processing(discovered_url, resource) - else: - logger.info("Embedded link for resource cannot be fetched") - return + logger.info("\n Embedded link {}".format(discovered_url) + + "cannot be fetched") + return "\n Embedded link {}".format(discovered_url) + \ + "cannot be fetched" # Creating relation between collection node and member response = self.graph_utils.create_relation(label_source="objects" + From e692c1e2f3783087b318a13c91c2515814578e96 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Fri, 5 Jul 2019 16:08:39 -0300 Subject: [PATCH 15/19] Implementing segregation in PUT and POST --- .gitignore | 1 + hydra_agent/agent.py | 27 +++++++++++++++---- .../redis_core/graphutils_operations.py | 21 ++++++++------- requirements.txt | 2 +- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index aa4f257..2dbd562 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ ENV/ share include bin +restart_script # pipenv generated filespip Pipfile diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index c161426..86f47f5 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -55,11 +55,10 @@ def get(self, url: str) -> Union[dict, list]: # Embedded resources are fetched and then properly linked for embedded_resource in embedded_resources: self.get(embedded_resource['embedded_url']) - self.graph_operations.embedded_resource( + self.graph_operations.link_resources( embedded_resource['parent_id'], embedded_resource['parent_type'], - embedded_resource['embedded_url'] - ) + embedded_resource['embedded_url']) return response.json() else: return response.text @@ -74,7 +73,16 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]: if response.status_code == 201: url = response.headers['Location'] - self.graph_operations.put_processing(url, new_object) + # Graph operations returns embedded resources if finding any + embedded_resources = \ + self.graph_operations.put_processing(url, new_object) + # Embedded resources are fetched and then properly linked + for embedded_resource in embedded_resources: + self.get(embedded_resource['embedded_url']) + self.graph_operations.link_resources( + embedded_resource['parent_id'], + embedded_resource['parent_type'], + embedded_resource['embedded_url']) return response.json(), url else: return response.text, "" @@ -88,7 +96,16 @@ def post(self, url: str, updated_object: dict) -> dict: response = super().post(url, json=updated_object) if response.status_code == 200: - self.graph_operations.post_processing(url, updated_object) + # Graph operations returns embedded resources if finding any + embedded_resources = \ + self.graph_operations.post_processing(url, updated_object) + # Embedded resources are fetched and then properly linked + for embedded_resource in embedded_resources: + self.get(embedded_resource['embedded_url']) + self.graph_operations.link_resources( + embedded_resource['parent_id'], + embedded_resource['parent_type'], + embedded_resource['embedded_url']) return response.json() else: return response.text diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 07a81b3..f41a126 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -125,8 +125,9 @@ def put_processing(self, url: str, new_object: dict) -> None: url_list = url.split('/', 3) new_object["@id"] = '/' + url_list[-1] # Simply call sync_get to add the resource to the collection at Redis - self.get_processing(url, new_object) - return + embedded_resources = self.get_processing(url, new_object) + + return embedded_resources def post_processing(self, url: str, updated_object: dict) -> None: """Synchronize Redis upon new POST operations @@ -139,8 +140,8 @@ def post_processing(self, url: str, updated_object: dict) -> None: # Simply call sync_get to add the resource to the collection at Redis self.delete_processing(url) - self.get_processing(url, updated_object) - return + embedded_resources = self.get_processing(url, updated_object) + return embedded_resources def delete_processing(self, url: str) -> None: """Synchronize Redis upon new DELETE operations @@ -215,19 +216,19 @@ def get_resource(self, url: str) -> dict: return resource - def embedded_resource(self, parent_id: str, parent_type: str, - discovered_url: str) -> str: + def link_resources(self, parent_id: str, parent_type: str, + node_url: str) -> str: """Checks for existance of discovered resource and creates links for embedded resources inside other resources properties :parent_id: Resource ID for the parent node that had this reference :parent_type: Resource Type for the parent node that had this reference - :discovered_url: URL Reference for resource found inside a property + :node_url: URL Reference for resource found inside a property """ - resource = self.get_resource(discovered_url) + resource = self.get_resource(node_url) if resource is None: - logger.info("\n Embedded link {}".format(discovered_url) + + logger.info("\n Embedded link {}".format(node_url) + "cannot be fetched") - return "\n Embedded link {}".format(discovered_url) + \ + return "\n Embedded link {}".format(node_url) + \ "cannot be fetched" # Creating relation between collection node and member diff --git a/requirements.txt b/requirements.txt index 0a54c4b..4cdb9f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ httplib2 redis graphviz requests --e git+https://github.com/RedisGraph/redisgraph-py@62c591da4ec54d268f3abde3561d378de1763746#egg=redisgraph +-e git+https://github.com/RedisGraph/redisgraph-py#egg=redisgraph -e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core From 81f16082eb0f255f1dccafdd61a4f14f14d31d5b Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Fri, 5 Jul 2019 18:10:48 -0300 Subject: [PATCH 16/19] Adding missing docstrings --- hydra_agent/agent.py | 3 +++ hydra_agent/redis_core/graphutils.py | 2 +- hydra_agent/redis_core/graphutils_operations.py | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 86f47f5..db7d145 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -11,6 +11,9 @@ class Agent(Session): + """Provides a straightforward GET, PUT, POST, DELETE - + CRUD interface - to query hydrus + """ def __init__(self, entrypoint_url: str) -> None: """Initialize the Agent :param entrypoint_url: Entrypoint URL for the hydrus server diff --git a/hydra_agent/redis_core/graphutils.py b/hydra_agent/redis_core/graphutils.py index 3665abf..c8b3acc 100644 --- a/hydra_agent/redis_core/graphutils.py +++ b/hydra_agent/redis_core/graphutils.py @@ -9,7 +9,7 @@ class GraphUtils: - + """Provides low level functions to interact with Redis Graph""" def __init__(self, redis_proxy: RedisProxy, graph_name="apigraph") -> None: """Initialize Graph Utils module :param redis_proxy: RedisProxy object created from redis_proxy module diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index f41a126..33c7694 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -12,9 +12,16 @@ class GraphOperations(): - + """Responsible to process the requests received by the Agent + inside Redis Graph, making sure it a synchronized cached layer""" def __init__(self, entrypoint_url: str, api_doc: dict, redis_proxy: RedisProxy): + """Initialize GraphOperations + :param entrypoint_url: Entrypoint URL for the hydrus server + :param api_doc: ApiDoc object that contains the documentation + :param redis_proxy: RedisProxy object created from redis_proxy module + :return: None + """ self.entrypoint_url = entrypoint_url self.api_doc = api_doc self.redis_proxy = redis_proxy From 3a8ee47af7417473706012c4ceca5d4b7d30e6c3 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Fri, 5 Jul 2019 18:11:21 -0300 Subject: [PATCH 17/19] Adjusting tests for embedded resources --- hydra_agent/tests/test_agent.py | 41 +++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/hydra_agent/tests/test_agent.py b/hydra_agent/tests/test_agent.py index ef3936f..8d34bbb 100644 --- a/hydra_agent/tests/test_agent.py +++ b/hydra_agent/tests/test_agent.py @@ -19,7 +19,7 @@ def setUp(self, get_session_mock): # Mocking get for ApiDoc to Server, so hydrus doesn't need to be up get_session_mock.return_value.json.return_value = drone_doc - self.agent = Agent("http://localhost:8080/serverapi") + self.agent = Agent("http://localhost:8080/api") @patch('hydra_agent.agent.Session.get') @patch('hydra_agent.agent.GraphOperations.get_processing') @@ -28,40 +28,46 @@ def test_get(self, get_processing_mock, get_session_mock): :param get_processing_mock: MagicMock object to patch graphoperations :param get_mock: MagicMock object for patching session.get """ - mock_dict = {"@type": "Drone", "DroneState": "Simplified state", + mock_dict = {"@type": "Drone", "DroneState": "/api/StateCollection/1", "name": "Smart Drone", "model": "Hydra Drone", "MaxSpeed": "999", "Sensor": "Wind"} # Mock server request to the Server get_session_mock.return_value.status_code = 200 get_session_mock.return_value.json.return_value = mock_dict - response = self.agent.get("http://localhost:8080/serverapi/" + + response = self.agent.get("http://localhost:8080/api/" + "DroneCollection/1") - get_processing_mock.assert_called_with("http://localhost:8080/server" + - "api/DroneCollection/1", + get_processing_mock.assert_called_with("http://localhost:8080/api" + + "/DroneCollection/1", mock_dict) self.assertEqual(response, mock_dict) - @patch('hydra_agent.redis_core.graphutils_operations.Session.get') + @patch('hydra_agent.agent.Session.get') @patch('hydra_agent.agent.Session.put') def test_put(self, put_session_mock, embedded_get_mock): """Tests put method from the Agent :param put_session_mock: MagicMock object for patching session.put """ - new_object = {"@type": "Drone", "DroneState": "Simplified state", + new_object = {"@type": "Drone", "DroneState": "/api/StateCollection/1", "name": "Smart Drone", "model": "Hydra Drone", "MaxSpeed": "999", "Sensor": "Wind"} - collection_url = "http://localhost:8080/serverapi/DroneCollection/" + collection_url = "http://localhost:8080/api/DroneCollection/" new_object_url = collection_url + "1" put_session_mock.return_value.status_code = 201 put_session_mock.return_value.json.return_value = new_object put_session_mock.return_value.headers = {'Location': new_object_url} + state_object = {"@context": "/api/contexts/StateCollection.jsonld", + "@id": "/api/StateCollection/1", "@type": "State", + "Battery": "sensor Ex", "Direction": "speed Ex", + "DroneID": "sensor Ex", "Position": "model Ex", + "SensorStatus": "sensor Ex", "Speed": "2"} + # Mocking an object to be used for a property that has an embedded link embedded_get_mock.return_value.status_code = 200 - embedded_get_mock.return_value.json.return_value = new_object + embedded_get_mock.return_value.json.return_value = state_object response, new_object_url = self.agent.put(collection_url, new_object) @@ -69,7 +75,7 @@ def test_put(self, put_session_mock, embedded_get_mock): get_new_object = self.agent.get(new_object_url) self.assertEqual(get_new_object, new_object) - @patch('hydra_agent.redis_core.graphutils_operations.Session.get') + @patch('hydra_agent.agent.Session.get') @patch('hydra_agent.agent.Session.post') @patch('hydra_agent.agent.Session.put') def test_post(self, put_session_mock, post_session_mock, @@ -78,11 +84,11 @@ def test_post(self, put_session_mock, post_session_mock, :param put_session_mock: MagicMock object for patching session.put :param post_session_mock: MagicMock object for patching session.post """ - new_object = {"@type": "Drone", "DroneState": "Simplified state", + new_object = {"@type": "Drone", "DroneState": "/api/StateCollection/1", "name": "Smart Drone", "model": "Hydra Drone", "MaxSpeed": "999", "Sensor": "Wind"} - collection_url = "http://localhost:8080/serverapi/DroneCollection/" + collection_url = "http://localhost:8080/api/DroneCollection/" new_object_url = collection_url + "2" put_session_mock.return_value.status_code = 201 @@ -94,9 +100,14 @@ def test_post(self, put_session_mock, post_session_mock, post_session_mock.return_value.json.return_value = {"msg": "success"} new_object['name'] = "Updated Name" + state_object = {"@context": "/api/contexts/StateCollection.jsonld", + "@id": "/api/StateCollection/1", "@type": "State", + "Battery": "sensor Ex", "Direction": "speed Ex", + "DroneID": "sensor Ex", "Position": "model Ex", + "SensorStatus": "sensor Ex", "Speed": "2"} # Mocking an object to be used for a property that has an embedded link embedded_get_mock.return_value.status_code = 200 - embedded_get_mock.return_value.json.return_value = new_object + embedded_get_mock.return_value.json.return_value = state_object response = self.agent.post(new_object_url, new_object) @@ -113,11 +124,11 @@ def test_delete(self, put_session_mock, delete_session_mock, :param put_session_mock: MagicMock object for patching session.put :param post_session_mock: MagicMock object for patching session.post """ - new_object = {"@type": "Drone", "DroneState": "Simplified state", + new_object = {"@type": "Drone", "DroneState": "/api/StateCollection/1", "name": "Smart Drone", "model": "Hydra Drone", "MaxSpeed": "999", "Sensor": "Wind"} - collection_url = "http://localhost:8080/serverapi/DroneCollection/" + collection_url = "http://localhost:8080/api/DroneCollection/" new_object_url = collection_url + "3" put_session_mock.return_value.status_code = 201 From 2252465bf2a2e9e72adc5ecf72af8da5705295fe Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Sat, 6 Jul 2019 17:30:01 -0300 Subject: [PATCH 18/19] Typos and pep8 --- README.md | 2 +- hydra_agent/agent.py | 6 +++--- hydra_agent/redis_core/graphutils_operations.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 71a9035..2fedf9a 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ from hydra_agent.agent import Agent agent = Agent("http://localhost:8080/serverapi") agent.get("http://localhost:8080/serverapi/DroneCollection/123-123-123-123") ``` -**Remember that it's important you use the an updated hydrus version, currently the one under the develop branch** +**Remember that it's important you use an hydrus updated version, currently the one under the develop branch** The agent supports GET, PUT, POST or DELETE: - GET - used to READ resources or collections diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index db7d145..741d824 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -52,7 +52,7 @@ def get(self, url: str) -> Union[dict, list]: response = super().get(url) if response.status_code == 200: - # Graph operations returns embedded resources if finding any + # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.get_processing(url, response.json()) # Embedded resources are fetched and then properly linked @@ -76,7 +76,7 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]: if response.status_code == 201: url = response.headers['Location'] - # Graph operations returns embedded resources if finding any + # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.put_processing(url, new_object) # Embedded resources are fetched and then properly linked @@ -99,7 +99,7 @@ def post(self, url: str, updated_object: dict) -> dict: response = super().post(url, json=updated_object) if response.status_code == 200: - # Graph operations returns embedded resources if finding any + # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.post_processing(url, updated_object) # Embedded resources are fetched and then properly linked diff --git a/hydra_agent/redis_core/graphutils_operations.py b/hydra_agent/redis_core/graphutils_operations.py index 33c7694..df9142a 100644 --- a/hydra_agent/redis_core/graphutils_operations.py +++ b/hydra_agent/redis_core/graphutils_operations.py @@ -230,6 +230,7 @@ def link_resources(self, parent_id: str, parent_type: str, :parent_id: Resource ID for the parent node that had this reference :parent_type: Resource Type for the parent node that had this reference :node_url: URL Reference for resource found inside a property + "return: Default Redis response with amount of relations created """ resource = self.get_resource(node_url) if resource is None: From 15d462115317a482f00998054f9f80c7535de476 Mon Sep 17 00:00:00 2001 From: Gustavo Morais Date: Sat, 6 Jul 2019 17:30:19 -0300 Subject: [PATCH 19/19] Embedded helper function --- hydra_agent/agent.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/hydra_agent/agent.py b/hydra_agent/agent.py index 741d824..5cf1fce 100644 --- a/hydra_agent/agent.py +++ b/hydra_agent/agent.py @@ -55,13 +55,7 @@ def get(self, url: str) -> Union[dict, list]: # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.get_processing(url, response.json()) - # Embedded resources are fetched and then properly linked - for embedded_resource in embedded_resources: - self.get(embedded_resource['embedded_url']) - self.graph_operations.link_resources( - embedded_resource['parent_id'], - embedded_resource['parent_type'], - embedded_resource['embedded_url']) + self.process_embedded(embedded_resources) return response.json() else: return response.text @@ -79,13 +73,7 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]: # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.put_processing(url, new_object) - # Embedded resources are fetched and then properly linked - for embedded_resource in embedded_resources: - self.get(embedded_resource['embedded_url']) - self.graph_operations.link_resources( - embedded_resource['parent_id'], - embedded_resource['parent_type'], - embedded_resource['embedded_url']) + self.process_embedded(embedded_resources) return response.json(), url else: return response.text, "" @@ -102,13 +90,7 @@ def post(self, url: str, updated_object: dict) -> dict: # Graph_operations returns the embedded resources if finding any embedded_resources = \ self.graph_operations.post_processing(url, updated_object) - # Embedded resources are fetched and then properly linked - for embedded_resource in embedded_resources: - self.get(embedded_resource['embedded_url']) - self.graph_operations.link_resources( - embedded_resource['parent_id'], - embedded_resource['parent_type'], - embedded_resource['embedded_url']) + self.process_embedded(embedded_resources) return response.json() else: return response.text @@ -125,7 +107,19 @@ def delete(self, url: str) -> dict: return response.json() else: return response.text - + + def process_embedded(self, embedded_resources: list) -> None: + """Helper function to process a list of embedded resources + fetching and linking them to their parent Nodes + :param embedded_resources: List of dicts containing resources + """ + # Embedded resources are fetched and then properly linked + for embedded_resource in embedded_resources: + self.get(embedded_resource['embedded_url']) + self.graph_operations.link_resources( + embedded_resource['parent_id'], + embedded_resource['parent_type'], + embedded_resource['embedded_url']) if __name__ == "__main__": pass \ No newline at end of file