Skip to content

Commit

Permalink
Introducing class registry lists
Browse files Browse the repository at this point in the history 
- Supports getting classes from a bundle dependency
  • Loading branch information
@mwatts15
mwatts15 committed Aug 6, 2020
1 parent 194085d commit 3d2fff3
Show file tree
Hide file tree
Showing 6 changed files with 260 additions and 92 deletions.
3 changes: 1 addition & 2 deletions owmeta_core/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,7 @@ def connect(configFile=None,

# Base class names is empty because we won't be adding any objects to the
# context automatically
mapper = Mapper(conf=conf)
conf['mapper'] = mapper
conf['mapper'] = Mapper(conf=conf)
# An "empty" context, that serves as the default when no context is defined

return Connection(conf)
126 changes: 88 additions & 38 deletions owmeta_core/bundle/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
import logging
import re
import shutil
import uuid

from rdflib import plugin
from rdflib.parser import Parser, create_input_source
from rdflib.term import URIRef
from rdflib.graph import ConjunctiveGraph
import six
from textwrap import dedent
import transaction
Expand All @@ -23,6 +25,7 @@
from ..context import (DEFAULT_CONTEXT_KEY, IMPORTS_CONTEXT_KEY,
CLASS_REGISTRY_CONTEXT_KEY, Context)
from ..context_common import CONTEXT_IMPORTS
from ..collections import List
from ..data import Data
from ..file_match import match_files
from ..file_lock import lock_file
Expand Down Expand Up @@ -401,6 +404,12 @@ def resolve(self):
bundle_directory = self._fetch_bundle(self.ident, self.version)
return bundle_directory

@property
def manifest_data(self):
bundle_directory = self.resolve()
with open(p(bundle_directory, BUNDLE_MANIFEST_FILE_NAME)) as mf:
return json.load(mf)

def _get_bundle_directory(self):
# - look up the bundle in the bundle cache
# - generate a config based on the current config load the config
Expand Down Expand Up @@ -1028,6 +1037,8 @@ def __init__(self, source_directory, bundles_directory, graph,
self.class_registry_ctx = class_registry_ctx
self.remotes = list(remotes)
self.remotes_directory = remotes_directory
self._force_include_class_registry = False
self._force_include_imports = False

def install(self, descriptor, progress_reporter=None):
'''
Expand Down Expand Up @@ -1083,9 +1094,9 @@ def _install(self, descriptor, staging_directory, progress_reporter=None):
graphs_directory, files_directory = self._set_up_directories(staging_directory)
self._write_file_hashes(descriptor, files_directory)
self._write_context_data(descriptor, graphs_directory)
self._write_manifest(descriptor, staging_directory)
self._generate_bundle_class_registry_ctx(descriptor, graphs_directory)
self._generate_bundle_imports_ctx(descriptor, graphs_directory)
self._write_manifest(descriptor, staging_directory)
self._initdb(staging_directory)
self._build_indexed_database(staging_directory, progress_reporter)

Expand Down Expand Up @@ -1141,15 +1152,15 @@ def _write_manifest(self, descriptor, staging_directory):
if self.default_ctx:
manifest_data[DEFAULT_CONTEXT_KEY] = self.default_ctx

if self.imports_ctx:
if self.imports_ctx or self._force_include_imports:
# If an imports context was specified, then we'll need to generate an
# imports context with the appropriate imports. We don't use the source
# imports context ID for the bundle's imports context because the bundle
# imports that we actually need are a subset of the total set of imports
manifest_data[IMPORTS_CONTEXT_KEY] = fmt_bundle_imports_ctx_id(descriptor.id,
descriptor.version)

if self.class_registry_ctx:
if self.class_registry_ctx or self._force_include_class_registry:
manifest_data[CLASS_REGISTRY_CONTEXT_KEY] = fmt_bundle_class_registry_ctx_id(descriptor.id,
descriptor.version)

Expand All @@ -1162,40 +1173,6 @@ def _write_manifest(self, descriptor, staging_directory):
with open(p(staging_directory, BUNDLE_MANIFEST_FILE_NAME), 'w') as mf:
json.dump(manifest_data, mf, separators=(',', ':'))

def _generate_bundle_imports_ctx(self, descriptor, graphs_directory):
if not self.imports_ctx:
return
imports_ctxg = self.graph.get_context(self.imports_ctx)
# select all of the imports for all of the contexts in the bundle and serialize
contexts = []
idx_fname = p(graphs_directory, 'index')
with open(idx_fname) as index_file:
for l in index_file:
ctx, _ = l.strip().split('\x00')
contexts.append(URIRef(ctx))
for c in descriptor.empties:
contexts.append(URIRef(c))
if self.class_registry_ctx:
cr_ctxid = URIRef(fmt_bundle_class_registry_ctx_id(descriptor.id, descriptor.version))
contexts.append(cr_ctxid)
ctxid = fmt_bundle_imports_ctx_id(descriptor.id, descriptor.version)
ctxgraph = imports_ctxg.triples_choices((contexts, CONTEXT_IMPORTS, None))
if self.class_registry_ctx:
old_ctxgraph = ctxgraph

def replace_cr_ctxid():
src_cr_ctxid = URIRef(self.class_registry_ctx)
for t in old_ctxgraph:
if t[0] == src_cr_ctxid:
yield (cr_ctxid, t[1], t[2])
elif t[2] == src_cr_ctxid:
yield (t[0], t[1], cr_ctxid)
else:
yield t
ctxgraph = replace_cr_ctxid()

self._write_graph(graphs_directory, ctxid, ctxgraph)

def _generate_bundle_class_registry_ctx(self, descriptor, graphs_directory):
if not self.class_registry_ctx:
return
Expand All @@ -1204,6 +1181,75 @@ def _generate_bundle_class_registry_ctx(self, descriptor, graphs_directory):

self._write_graph(graphs_directory, ctx_id, class_registry_ctxg)

def _declare_class_registry_list(self, descriptor):
if self.imports_ctx is None:
self.imports_ctx = uuid.uuid4().urn
imports_ctx = Context(self.imports_ctx)
ctx_id = fmt_bundle_class_registry_ctx_id(descriptor.id, descriptor.version)
list_id = fmt_bundle_class_registry_ctx_list_id(descriptor.id, descriptor.version)

crctxs = [imports_ctx(Context)(ctx_id).rdf_object]
for d in descriptor.dependencies:
bnd = Bundle(d.id, self.bundles_directory, d.version, remotes=self.remotes,
remotes_directory=self.remotes_directory)
bnd_manifest_data = bnd.manifest_data
bnd_crctx_id = bnd_manifest_data.get(CLASS_REGISTRY_CONTEXT_KEY)

if bnd_crctx_id:
crctxs.append(imports_ctx(Context)(bnd_crctx_id).rdf_object)
graph = None
if len(crctxs) > 1:
graph = ConjunctiveGraph()
self._force_include_class_registry = True
self._force_include_imports = True
imports_ctx(List).from_sequence(crctxs, URIRef(list_id))

imports_ctx.save(graph)

return graph

def _generate_bundle_imports_ctx(self, descriptor, graphs_directory):
if self.imports_ctx:
imports_ctxg = self.graph.get_context(self.imports_ctx)
# select all of the imports for all of the contexts in the bundle and serialize
contexts = []
idx_fname = p(graphs_directory, 'index')
with open(idx_fname) as index_file:
for l in index_file:
ctx, _ = l.strip().split('\x00')
contexts.append(URIRef(ctx))
for c in descriptor.empties:
contexts.append(URIRef(c))
if self.class_registry_ctx:
cr_ctxid = URIRef(fmt_bundle_class_registry_ctx_id(descriptor.id, descriptor.version))
contexts.append(cr_ctxid)
ctxgraph = imports_ctxg.triples_choices((contexts, CONTEXT_IMPORTS, None))
if self.class_registry_ctx or self._force_include_class_registry:
old_ctxgraph = ctxgraph

def replace_cr_ctxid():
src_cr_ctxid = URIRef(self.class_registry_ctx)
for t in old_ctxgraph:
if t[0] == src_cr_ctxid:
yield (cr_ctxid, t[1], t[2])
elif t[2] == src_cr_ctxid:
yield (t[0], t[1], cr_ctxid)
else:
yield t
ctxgraph = replace_cr_ctxid()
else:
ctxgraph = None

crctx = self._declare_class_registry_list(descriptor)
if crctx is not None:
if ctxgraph is not None:
ctxgraph = chain(ctxgraph, crctx)
else:
ctxgraph = crctx
if ctxgraph is not None:
ctxid = fmt_bundle_imports_ctx_id(descriptor.id, descriptor.version)
self._write_graph(graphs_directory, ctxid, ctxgraph)

def _write_graph(self, graphs_directory, ctxid, ctxgraph):
for _ in self._write_graphs(graphs_directory, (ctxid, ctxgraph)):
pass
Expand Down Expand Up @@ -1280,8 +1326,12 @@ def fmt_bundle_class_registry_ctx_id(id, version):
return fmt_bundle_ctx_id('generated_class_registry_ctx', id, version)


def fmt_bundle_class_registry_ctx_list_id(id, version):
return fmt_bundle_ctx_id('generated_class_registry_ctx_list', id, version)


def fmt_bundle_ctx_id(kind, id, version):
return f'http://openworm.org/data/{kind}?bundle_id={urlquote(id)}&bundle_version={version}'
return f'http://data.openworm.org/bundle/{kind}?bundle_id={urlquote(id)}&bundle_version={version}'


class FilesDescriptor(object):
Expand Down
12 changes: 6 additions & 6 deletions owmeta_core/collections.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,15 +186,15 @@ def load_dataobject_sequences(self, seen=None):
yield []
return

if self.identifier in seen:
# Maybe a loop was made on purpose, so no warning, but still worth noting.
L.info('Loop detected: %s in %s', self, seen)
yield _Loop((), self)
return

for m in self.load():
rests = m.rest.get()

if m.identifier in seen:
# Maybe a loop was made on purpose, so no warning, but still worth noting.
L.info('Loop detected: %s in %s', self, seen)
yield _Loop((), m)
return

seen.append(m.identifier)

hit = False
Expand Down
103 changes: 64 additions & 39 deletions owmeta_core/mapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def __init__(self, base_namespace=None, imported=(), name=None, **kwargs):
name = hex(id(self))
self.name = name
self.__class_registry_context = None
self.__class_registry_context_list = None
self._bootstrap_mappings()

@property
Expand All @@ -82,13 +83,32 @@ def class_registry_context(self):
self.__class_registry_context = crctx
return self.__class_registry_context

@property
def class_registry_context_list(self):
from .collections import List
from .context import Context
if self.__class_registry_context_list is None:
res = []
ctx = Context(conf=self.conf).stored
crctx_do = self.class_registry_context.rdf_object
crctx_list = ctx(List)(first=crctx_do)
for seq in crctx_list.load_sequences():
for crctx_do0 in seq:
res.append(Context(crctx_do0.identifier, conf=self.conf))
break
else: # no break
res.append(self.class_registry_context)
self.__class_registry_context_list = res
return self.__class_registry_context_list

def _bootstrap_mappings(self):
from .dataobject import Module
from .collections import List

# Add classes needed for resolving other classes...
# XXX: Smells off...probably don't want to have to do this.
self.process_classes(BaseDataObject, DataObject, PythonClassDescription, Module,
ClassDescription, PythonModule, RegistryEntry)
ClassDescription, PythonModule, RegistryEntry, List)

def add_class(self, cls):
cname = FCN(cls)
Expand Down Expand Up @@ -190,49 +210,54 @@ def load_registry_entries(self):
def resolve_class(self, uri):
# look up the class in the registryCache
c = self.RDFTypeTable.get(uri)

if c is None:
# otherwise, attempt to load into the cache by
# reading the RDF graph.
cr_ctx = self.class_registry_context.stored
re = cr_ctx(RegistryEntry)()
re.rdf_class(uri)
cd = cr_ctx(PythonClassDescription)()
re.class_description(cd)

for cd_l in cd.load():
class_name = cd_l.name()
moddo = cd_l.module()
try:
mod = self.load_module(moddo.name())
except ModuleNotFoundError:
L.warn('Did not find module %s', moddo.name())
continue
c = getattr(mod, class_name, None)
for unstored_cr_ctx in self.class_registry_context_list:
cr_ctx = unstored_cr_ctx.stored
print("OP", cr_ctx.rdf)
for m in cr_ctx.rdf:
print(' '.join(x.n3() for x in m))
re = cr_ctx(RegistryEntry)()
re.rdf_class(uri)
cd = cr_ctx(PythonClassDescription)()
re.class_description(cd)

for cd_l in cd.load():
class_name = cd_l.name()
moddo = cd_l.module()
try:
mod = self.load_module(moddo.name())
except ModuleNotFoundError:
L.warn('Did not find module %s', moddo.name())
continue
c = getattr(mod, class_name, None)
if c is not None:
break
L.warning('Did not find class %s in %s', class_name, mod.__name__)

ymc = getattr(mod, '__yarom_mapped_classes__', None)
if not ymc:
L.warning('No __yarom_mapped_classes__ in %s, so cannot look up %s',
mod.__name__, class_name)
continue

matching_classes = tuple(mc for mc in ymc
if mc.__name__ == class_name)
if not matching_classes:
L.warning('Did not find class %s in %s.__yarom_mapped_classes__',
class_name, mod.__name__)
continue

c = matching_classes[0]
if len(matching_classes) > 1:
L.warning('More than one class has the same name in'
' __yarom_mapped_classes__ for %s, so we are picking'
' the first one as the resolved class among %s',
mod, matching_classes)
break
if c is not None:
break
L.warning('Did not find class %s in %s', class_name, mod.__name__)

ymc = getattr(mod, '__yarom_mapped_classes__', None)
if not ymc:
L.warning('No __yarom_mapped_classes__ in %s, so cannot look up %s',
mod.__name__, class_name)
continue

matching_classes = tuple(mc for mc in ymc
if mc.__name__ == class_name)
if not matching_classes:
L.warning('Did not find class %s in %s.__yarom_mapped_classes__',
class_name, mod.__name__)
continue

c = matching_classes[0]
if len(matching_classes) > 1:
L.warning('More than one class has the same name in'
' __yarom_mapped_classes__ for %s, so we are picking'
' the first one as the resolved class among %s',
mod, matching_classes)
break
return c

def load_class(self, cname_or_mname, cnames=None):
Expand Down
Loading

0 comments on commit 3d2fff3

Please sign in to comment.