Skip to content

Commit

Permalink
Splitting backend.neo4j into backend.neo4j and backend.neo4j_spatial
Browse files Browse the repository at this point in the history
  • Loading branch information
@versae
versae committed Jan 20, 2016
1 parent 69782e4 commit 1ed14b6
Show file tree
Hide file tree
Showing 2 changed files with 268 additions and 219 deletions.
254 changes: 35 additions & 219 deletions sylva/engines/gdb/backends/neo4j.py
View file
Expand Up @@ -4,27 +4,18 @@
from django.conf import settings
from django.template.defaultfilters import slugify

from neo4jrestclient.exceptions import StatusException
from pyblueprints.neo4j import (
Neo4jTransactionalIndexableGraph as Neo4jGraphDatabase)
from pyblueprints.neo4j import Neo4jDatabaseConnectionError

from engines.gdb.backends import (GraphDatabaseConnectionError,
GraphDatabaseInitializationError)
from engines.gdb.backends.blueprints import BlueprintsGraphDatabase, VERTEX
from engines.gdb.backends.blueprints import BlueprintsGraphDatabase
from engines.gdb.lookups.neo4j import Q as q_lookup_builder
try:
from engines.gdb.analysis.neo4j import Analysis
except ImportError:
Analysis = None
if settings.ENABLE_SPATIAL:
import geojson
from shapely.geometry import shape
# Any symbol would work
SPATIAL_INDEX_KEY = "$"
SPATIAL_INDEX_VALUE = "$"
SPATIAL_PROPERTY_NAMES = ("bbox", "gtype")


WILDCARD_TYPE = -1
AGGREGATES = {
Expand Down Expand Up @@ -56,9 +47,6 @@ def __init__(self, url, params=None, graph=None):
self._cypher = None
self._spatial = None
self.transaction = getattr(self.gdb.neograph, "transaction", None)
if settings.ENABLE_SPATIAL:
self.sidx = {} # shortcut for spatial indices
self.setup_spatial()

def _get_nidx(self):
if not self._nidx:
Expand Down Expand Up @@ -210,6 +198,13 @@ def get_nodes_by_label(self, label, include_properties=False,
limit=limit, offset=offset,
order_by=order_by)

def _get_filtered_nodes_properties(self, element):
properties = element[1]["data"]
elto_id = properties.pop("_id")
elto_label = properties.pop("_label")
properties.pop("_graph", None)
return (elto_id, properties, elto_label)

def get_filtered_nodes(self, lookups, label=None, include_properties=None,
limit=None, offset=None, order_by=None):
# Using Cypher
Expand Down Expand Up @@ -250,14 +245,7 @@ def get_filtered_nodes(self, lookups, label=None, include_properties=None,
while result and "data" in result:
if include_properties:
for element in result["data"]:
properties = element[1]["data"]
elto_id = properties.pop("_id")
elto_label = properties.pop("_label")
properties.pop("_graph", None)
if settings.ENABLE_SPATIAL:
for to_remove in SPATIAL_PROPERTY_NAMES:
properties.pop(to_remove, None)
yield (elto_id, properties, elto_label)
yield self._get_filtered_nodes_properties(element)
else:
for element in result["data"]:
if len(element) > 1:
Expand All @@ -284,6 +272,31 @@ def get_relationships_by_label(self, label, include_properties=False,
source_id=source_id, target_id=target_id, directed=directed,
limit=limit, offset=offset, order_by=order_by)

def _get_filtered_relationships_properties(element):
properties = element[1]["data"]
properties.pop("_id")
properties.pop("_graph", None)
elto_label = properties.pop("_label")
source_props = element[2]["data"]
source_id = source_props.pop("_id")
source_label = source_props.pop("_label")
source_props.pop("_graph", None)
target_props = element[3]["data"]
target_id = target_props.pop("_id")
target_label = target_props.pop("_label")
target_props.pop("_graph", None)
source = {
"id": source_id,
"properties": source_props,
"label": source_label
}
target = {
"id": target_id,
"properties": target_props,
"label": target_label
}
return (element[0], properties, elto_label, source, target)

def get_filtered_relationships(self, lookups, label=None,
include_properties=None,
source_id=None, target_id=None,
Expand Down Expand Up @@ -340,33 +353,7 @@ def get_filtered_relationships(self, lookups, label=None,
while result and "data" in result and len(result["data"]) > 0:
if include_properties:
for element in result["data"]:
properties = element[1]["data"]
properties.pop("_id")
properties.pop("_graph", None)
elto_label = properties.pop("_label")
source_props = element[2]["data"]
source_id = source_props.pop("_id")
source_label = source_props.pop("_label")
source_props.pop("_graph", None)
target_props = element[3]["data"]
target_id = target_props.pop("_id")
target_label = target_props.pop("_label")
target_props.pop("_graph", None)
if settings.ENABLE_SPATIAL:
for to_remove in SPATIAL_PROPERTY_NAMES:
source_props.pop(to_remove, None)
target_props.pop(to_remove, None)
source = {
"id": source_id,
"properties": source_props,
"label": source_label
}
target = {
"id": target_id,
"properties": target_props,
"label": target_label
}
yield (element[0], properties, elto_label, source, target)
yield self._get_filtered_relationships_properties(element)
else:
for element in result["data"]:
yield (element[0], None, element[1])
Expand Down Expand Up @@ -740,12 +727,6 @@ def _query_generator_patterns(self, patterns_dict, conditions_alias):

def destroy(self):
"""Delete nodes, relationships, and even indices"""
if settings.ENABLE_SPATIAL and self.sidx:
for sidx_key, sidx_dict in self.sidx.items():
index = sidx_dict["index"]
if index in self.gdb.neograph.nodes.indexes.values():
index.delete()
self.sidx = {}
if self.ridx in self.gdb.neograph.relationships.indexes.values():
self.ridx.delete()
if self.nidx in self.gdb.neograph.nodes.indexes.values():
Expand Down Expand Up @@ -783,168 +764,3 @@ def _get_slug_and_prop(self, elem_slug, prop):
match_property = prop_value[0].key
# Finally, we return the values
return elem_slug, match_property

# Spatial
def setup_spatial(self):
# Setup spatial indices if they don't exist already
# The proccess goes as follows
# - post /db/sylva/index/node/
# {"name": "spatial3", "config": {"provider": "spatial", "wkt": "wkt"}}
# - post /db/sylva/index/node/spatial3
# {"value": "0", "key": "0",
# "uri": "http://host:port/db/sylva/node/17600"}
# - post /db/sylva/ext/SpatialPlugin/graphdb/addNodeToLayer
# {"layer": "spatial3", "node": "http://host:port/db/sylva/node/17600"}
spatial_datatypes = [u'p', u'l', u'm']
spatial_properties = self.graph.schema.nodetype_set.filter(
properties__datatype__in=spatial_datatypes
).values("id", "schema__graph__id", "properties__key",
"properties__id")
indices = self.gdb.neograph.nodes.indexes
for spatial_property in spatial_properties:
# Spatial index name is in
# the form: <graph_id>_<nodetype_id>_<property_id>_spatial
spatial_index_name = u"{}_{}_{}_spatial".format(
spatial_property["schema__graph__id"],
spatial_property["id"],
spatial_property["properties__id"])
spatial_index = None
try:
spatial_index = indices.get(spatial_index_name)
except StatusException:
spatial_index = None
finally:
# The spatial index property to index by is
# in the form: _spatial_<property_id>
spatial_index_by = u"_spatial_{}".format(
spatial_property["properties__id"])
if spatial_index is None:
spatial_index = indices.create(
name=spatial_index_name,
provider="spatial",
wkt=spatial_index_by
)
# Keep track of properties indexed in spatial indices
sidx_key = u"{}_{}".format(
spatial_property["id"],
spatial_property["properties__key"])
self.sidx[sidx_key] = {
"index": spatial_index,
"key": spatial_index_by,
}

def _get_spatial(self):
if not self._spatial and settings.ENABLE_SPATIAL:
self._spatial = self.gdb.neograph.extensions.SpatialPlugin
return self._spatial
spatial = property(_get_spatial)

def _index_spatial_property(self, node, key, value, label):
sidx_key = u"{}_{}".format(label, key)
if sidx_key in self.sidx:
geo_value = geojson.loads(value)
is_valid_geojson = geojson.is_valid(geo_value)['valid'] != 'no'
if is_valid_geojson:
# Add node to index
index = self.sidx[sidx_key]["index"]
index_key = self.sidx[sidx_key]["key"]
wkt_value = shape(geo_value).wkt
node[index_key] = wkt_value
index.add(SPATIAL_INDEX_KEY, SPATIAL_INDEX_VALUE, node)
# Add node to layer
self.spatial.addNodeToLayer(layer=index.name, node=node.url)

def _deindex_spatial_property(self, node, key, label):
sidx_key = u"{}_{}".format(label, key)
if sidx_key in self.sidx:
index = self.sidx[sidx_key]["index"]
index.delete(SPATIAL_INDEX_KEY, SPATIAL_INDEX_VALUE, node)

def _reindex_spatial_property(self, node, key, value, label):
self._deindex_spatial_property(node, key, label)
self._index_spatial_property(node, key, value, label)

def _set_element_property(self, element, key, value, element_type=None):
super(GraphDatabase, self)._set_element_property(
element, key, value, element_type)
if settings.ENABLE_SPATIAL and element_type == VERTEX:
label = element.getProperty("%slabel" % self.PRIVATE_PREFIX)
node = self.gdb.neograph.nodes.get(element.getId())
self._reindex_spatial_property(node, key, value, label)

def _delete_element_property(self, element, key, element_type=None):
super(GraphDatabase, self)._delete_element_property(
element, key, element_type)
if settings.ENABLE_SPATIAL and element_type == VERTEX:
label = element.getProperty("%slabel" % self.PRIVATE_PREFIX)
node = self.gdb.neograph.nodes.get(element.getId())
self._deindex_spatial_property(node, key, label)

def _set_element_properties(self, element, properties, element_type=None):
if settings.ENABLE_SPATIAL and element_type == VERTEX:
label = element.getProperty("%slabel" % self.PRIVATE_PREFIX)
node = self.gdb.neograph.nodes.get(element.getId())
for key in self._get_public_keys(element):
self._deindex_spatial_property(node, key, label)
super(GraphDatabase, self)._set_element_properties(
element, properties, element_type)

def _update_element_properties(self, element, properties,
element_type=None):
super(GraphDatabase, self)._update_element_properties(
element, properties, element_type)
if settings.ENABLE_SPATIAL and element_type == VERTEX:
label = element.getProperty("%slabel" % self.PRIVATE_PREFIX)
node = self.gdb.neograph.nodes.get(element.getId())
for key, value in properties.items():
self._reindex_spatial_property(node, key, value, label)

def _delete_element_properties(self, element, element_type=None):
if settings.ENABLE_SPATIAL and element_type == VERTEX:
label = element.getProperty("%slabel" % self.PRIVATE_PREFIX)
node = self.gdb.neograph.nodes.get(element.getId())
for key in self._get_public_keys(element):
self._deindex_spatial_property(node, key, label)
super(GraphDatabase, self)._delete_element_properties(
element, element_type)

def create_node(self, label, properties=None):
node_id = super(GraphDatabase, self).create_node(
label=label, properties=properties)
# We introspect all property values to see if any would have a
# spatial value and an index in which to be indexed
if settings.ENABLE_SPATIAL and self.sidx and properties:
node = self.gdb.neograph.nodes.get(node_id)
for key, value in properties.items():
self._index_spatial_property(node, key, value, label)
return node_id

def get_node_relationships(self, id, incoming=False, outgoing=False,
include_properties=False, label=None):
rels = super(GraphDatabase, self).get_node_relationships(
id, incoming, outgoing, include_properties, label
)
if not settings.ENABLE_SPATIAL:
return rels
# If spatial enabled, avoid returning spatial
# specific relationships by checking their label
spatial_rel_labels = set([
"RTREE_METADATA", "RTREE_ROOT", "RTREE_CHILD",
"RTREE_REFERENCE", "FIRST_NODE", "LAST_NODE", "OTHER", "NEXT",
"OSM", "WAYS", "RELATIONS", "MEMBERS", "MEMBER", "TAGS", "GEOM",
"BBOX", "NODE", "CHANGESET", "USER", "USERS", "OSM_USER",
])
clean_rels = []
for (rel_id, rel_props) in rels:
label = self._get_edge(rel_id).neoelement.type
if not label in spatial_rel_labels:
clean_rels.append((rel_id, rel_props))
return clean_rels

def get_node_properties(self, id):
properties = super(GraphDatabase, self).get_node_properties(id)
if settings.ENABLE_SPATIAL:
for to_remove in SPATIAL_PROPERTY_NAMES:
if to_remove in properties:
del properties[to_remove]
return properties

0 comments on commit 1ed14b6

Please sign in to comment.