Recursive node creation

.gitignore

@@ -89,6 +89,7 @@ ENV/
 share
 include
 bin
+restart_script
 
 # pipenv generated filespip 
 Pipfile

.travis.yml

@@ -3,7 +3,7 @@ services:
   - docker
 
 before_script:
-  - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph:1.2.2
+  - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph:edge
 
 python:
   - "3.5"
@@ -12,7 +12,8 @@ python:
   - "3.6-dev"  # 3.6 development branch
   - "3.7-dev"
 install:
-  - pip install -e git+https://github.com/HTTP-APIs/hydra-python-core@v0.1#egg=hydra_python_core
+  - pip install -e git+https://github.com/RedisGraph/redisgraph-py#egg=redisgraph
+  - pip install -e git+https://github.com/HTTP-APIs/hydra-python-core#egg=hydra_python_core
   - pip install -e .
 
 script: python -m unittest discover

README.md

@@ -64,7 +64,7 @@ After setup the environment. You can query or run the client.
         docker-compose run client
 
 
-    and provide a valid URL and then you can query in querying format.
+    and provide a valid URL(of a hydrus updated server) and then you can query in querying format.
 
         `>>>url` #here url should be a valid link, for testing you can use http://35.224.198.158:8080/api
         `>>>help` # it will provide the querying format
@@ -148,7 +148,7 @@ from hydra_agent.agent import Agent
 agent = Agent("http://localhost:8080/serverapi")
 agent.get("http://localhost:8080/serverapi/DroneCollection/123-123-123-123")
 ```
-
+**Remember that it's important you use an hydrus updated version, currently the one under the develop branch**
 The agent supports GET, PUT, POST or DELETE:
 
 - GET - used to READ resources or collections
@@ -219,6 +219,11 @@ Get all nodes and filter by label:
 GRAPH.QUERY apigraph "MATCH (p:collection) RETURN p" 
 ```
 
+Get all nodes and filter by label:
+```'
+GRAPH.QUERY apigraph "MATCH (p) WHERE(p.id = '/serverapi/DroneCollection/72b53615-a480-4920-b126-4d1e1e107dc6') RETURN p" 
+```
+
 To read all the edges of the graph
 ```
 GRAPH.QUERY apigraph "MATCH ()-[r]->() RETURN type(r)"

docker-compose.yaml

@@ -14,6 +14,6 @@ services:
         environment:
             - REDIS_HOST=redis_db
     db:
-        image: redislabs/redisgraph
+        image: redislabs/redisgraph:edge
         ports:
             - "6379:6379"

hydra_agent/agent.py

@@ -6,10 +6,14 @@
 from typing import Union, Tuple
 from requests import Session
 
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__file__)
 
 
 class Agent(Session):
+    """Provides a straightforward GET, PUT, POST, DELETE -
+    CRUD interface - to query hydrus
+    """
     def __init__(self, entrypoint_url: str) -> None:
         """Initialize the Agent
         :param entrypoint_url: Entrypoint URL for the hydrus server
@@ -18,11 +22,12 @@ def __init__(self, entrypoint_url: str) -> None:
         self.entrypoint_url = entrypoint_url.strip().rstrip('/')
         self.redis_proxy = RedisProxy()
         self.redis_connection = self.redis_proxy.get_connection()
-        self.graph_operations = GraphOperations(entrypoint_url,
-                                                self.redis_proxy)
         super().__init__()
         jsonld_api_doc = super().get(self.entrypoint_url + '/vocab').json()
         self.api_doc = doc_maker.create_doc(jsonld_api_doc)
+        self.graph_operations = GraphOperations(entrypoint_url,
+                                                self.api_doc,
+                                                self.redis_proxy)
         self.initialize_graph()
 
     def initialize_graph(self) -> None:
@@ -47,9 +52,13 @@ def get(self, url: str) -> Union[dict, list]:
         response = super().get(url)
 
         if response.status_code == 200:
-            self.graph_operations.get_processing(url, response.json())
-
-        return response.json()
+            # Graph_operations returns the embedded resources if finding any
+            embedded_resources = \
+                self.graph_operations.get_processing(url, response.json())
+            self.process_embedded(embedded_resources)
+            return response.json()
+        else:
+            return response.text
 
     def put(self, url: str, new_object: dict) -> Tuple[dict, str]:
         """CREATE resource in the Server/cache it on Redis
@@ -61,10 +70,13 @@ def put(self, url: str, new_object: dict) -> Tuple[dict, str]:
 
         if response.status_code == 201:
             url = response.headers['Location']
-            self.graph_operations.put_processing(url, new_object)
+            # Graph_operations returns the embedded resources if finding any
+            embedded_resources = \
+                self.graph_operations.put_processing(url, new_object)
+            self.process_embedded(embedded_resources)
             return response.json(), url
-
-        return response.json(), ""
+        else:
+            return response.text, ""
 
     def post(self, url: str, updated_object: dict) -> dict:
         """UPDATE resource in the Server/cache it on Redis
@@ -75,9 +87,13 @@ def post(self, url: str, updated_object: dict) -> dict:
         response = super().post(url, json=updated_object)
 
         if response.status_code == 200:
-            self.graph_operations.post_processing(url, updated_object)
-
-        return response.json()
+            # Graph_operations returns the embedded resources if finding any
+            embedded_resources = \
+                self.graph_operations.post_processing(url, updated_object)
+            self.process_embedded(embedded_resources)
+            return response.json()
+        else:
+            return response.text
 
     def delete(self, url: str) -> dict:
         """DELETE resource in the Server/delete it on Redis
@@ -88,8 +104,22 @@ def delete(self, url: str) -> dict:
 
         if response.status_code == 200:
             self.graph_operations.delete_processing(url)
-
-        return response.json()
+            return response.json()
+        else:
+            return response.text
+
+    def process_embedded(self, embedded_resources: list) -> None:
+        """Helper function to process a list of embedded resources
+        fetching and linking them to their parent Nodes
+        :param embedded_resources: List of dicts containing resources
+        """
+        # Embedded resources are fetched and then properly linked
+        for embedded_resource in embedded_resources:
+            self.get(embedded_resource['embedded_url'])
+            self.graph_operations.link_resources(
+                embedded_resource['parent_id'],
+                embedded_resource['parent_type'],
+                embedded_resource['embedded_url'])
 
 if __name__ == "__main__":
-    pass
+    pass

hydra_agent/querying_mechanism.py

@@ -62,6 +62,8 @@ def show_data(self, get_data):
             if count % 2 != 0:
                 for obj1 in objects:
                     for obj in obj1:
+                        if obj is None:
+                            continue
                         string = obj.decode('utf-8')
                         map_string = map(str.strip, string.split(','))
                         property_list = list(map_string)

hydra_agent/redis_core/classes_objects.py

@@ -179,13 +179,8 @@ def load_from_server(
                 endpoint_property,
                 no_endpoint_property,
                 api_doc)
-        # delete all the old data that has saved in Redis using redis_graph.
-        # It will remove duplicate data from Redis.
-        for key in redis_connection.keys():
-            if "fs:" not in key.decode("utf8"):
-                redis_connection.delete(key)
         # save the new data.
-        self.redis_graph.commit()
+        self.redis_graph.flush()
 
     def endpointclasses(
             self,

hydra_agent/redis_core/collections_endpoint.py

@@ -198,13 +198,8 @@ def load_from_server(
             url,
             redis_connection
         )
-        # delete all the old data that has saved in Redis using redis_graph.
-        # It will remove duplicate data from Redis.
-        for key in redis_connection.keys():
-            if "fs:" not in key.decode("utf8"):
-                redis_connection.delete(key)
         # save the new data.
-        self.redis_graph.commit()
+        self.redis_graph.flush()
 #        for node in self.redis_graph.nodes.values():
 #            print("\n",node.alias)
 

hydra_agent/redis_core/graph_init.py

@@ -1,4 +1,3 @@
-import redis
 from redisgraph import Graph, Node
 import urllib.request
 import json

hydra_agent/redis_core/graphutils.py

@@ -4,11 +4,12 @@
 from typing import Union, Optional
 from redis.exceptions import ResponseError
 
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__file__)
 
 
 class GraphUtils:
-
+    """Provides low level functions to interact with Redis Graph"""
     def __init__(self, redis_proxy: RedisProxy, graph_name="apigraph") -> None:
         """Initialize Graph Utils module
         :param redis_proxy: RedisProxy object created from redis_proxy module
@@ -119,9 +120,10 @@ def add_edge(self, source_node: Node, predicate: str,
         edge = Edge(source_node, predicate, dest_node)
         self.redis_graph.add_edge(edge)
 
-    def commit(self) -> None:
-        """Commit the changes made to the Graph to Redis"""
-        self.redis_graph.commit()
+    def flush(self) -> None:
+        """Commit the changes made to the Graph to Redis and reset/flush
+        the Nodes and Edges to be added in the next commit"""
+        self.redis_graph.flush()
 
     def process_result(self, result: list) -> list:
         """
@@ -130,18 +132,14 @@ def process_result(self, result: list) -> list:
         """
         response_json_list = []
 
-        for record in result.result_set[1:]:
+        if not result.result_set:
+            return []
+
+        for record in result.result_set[0][:]:
             new_record = {}
-            for j, property_x in enumerate(record):
-                if property_x is None:
-                    pass
-                else:
-                    property_name = result.result_set[0][j].decode()
-                    try:
-                        node_alias, property_name = property_name.split(".")
-                        new_record[property_name] = property_x.decode()
-                    except ValueError as e:
-                        logger.info("Graph property with no dot/wrong format")
+            if record is None:
+                return
+            new_record = record.properties
             if new_record:
                 if 'id' in new_record:
                     new_record['@id'] = new_record.pop('id')