Dev extension interface (#11)

* Added WikidataNamespace instance in mtab assistant

* updated WikidataNamespace instance as an instance variable

* Added extension interfaces and moved plugin implementations to extensions module

* Renamed transformations interface and added export interface implementation

* Updated Drepr module in extensions, implemented DreprExport class

* Moved drepr extension under export extension module, renamed drepr script as main.py

* Fixed import error for import_attr function

* Added output format as a parameter for drepr data exports

* Updated output format for drepr extension from String to OutputFormat Enum

* Updated export interface comments

* Updated export interface comments

* Added export config and generalised the usage of IExport implementations

* Updated variable name for local cache for exports

* Added default export in the export config, updated export logic usage key

sand/commands/load.py

@@ -1,18 +1,15 @@
-import glob
-import os
 from pathlib import Path
 
 import click
 from tqdm.auto import tqdm
-from loguru import logger
 from sand.models import (
     Project,
     SemanticModel,
     db as dbconn,
     init_db,
 )
-from sand.plugins.grams_plugin import convert_linked_table
-from sm.dataset import Example, Dataset, FullTable
+from sand.extensions.assistants.grams_plugin import convert_linked_table
+from sm.dataset import Example, Dataset
 from grams.inputs import LinkedTable
 
 

sand/config.py

@@ -2,7 +2,6 @@
 
 from pathlib import Path
 
-
 _ROOT_DIR = Path(os.path.abspath(__file__)).parent.parent
 PACKAGE_DIR = str(Path(os.path.abspath(__file__)).parent)
 FROM_SITEPACKAGES = _ROOT_DIR.name == "site-packages"
@@ -11,7 +10,7 @@
 
 SETTINGS = {
     "entity": {
-        "constructor": "sand.plugins.wikidata.get_entity_db",
+        "constructor": "sand.extensions.wikidata.get_entity_db",
         "uri2id": "sm.namespaces.prelude.WikidataNamespace.get_entity_id",
         "id2uri": "sm.namespaces.prelude.WikidataNamespace.get_entity_abs_uri",
         "args": {
@@ -30,18 +29,18 @@
         "new_entity_template": "http://www.wikidata.org/entity/{id}",
     },
     "ont_classes": {
-        "constructor": "sand.plugins.wikidata.get_ontclass_db",
-        "uri2id": "sand.plugins.wikidata.uri2id",
+        "constructor": "sand.extensions.wikidata.get_ontclass_db",
+        "uri2id": "sand.extensions.wikidata.uri2id",
         "args": {
             "dbfile": "/tmp/wdclasses.db",
             "proxy": True,
         },
         # extra classes
-        "default": "sand.plugins.wikidata.WD_ONT_CLASSES",
+        "default": "sand.extensions.wikidata.WD_ONT_CLASSES",
     },
     "ont_props": {
-        "constructor": "sand.plugins.wikidata.get_ontprop_db",
-        "uri2id": "sand.plugins.wikidata.uri2id",
+        "constructor": "sand.extensions.wikidata.get_ontprop_db",
+        "uri2id": "sand.extensions.wikidata.uri2id",
         "args": {
             "dbfile": "/tmp/wdprops.db",
             "proxy": True,
@@ -59,8 +58,12 @@
     },
     "assistants": {
         # list of assistants' names and their models
-        # "grams": "sand.plugins.grams.GRAMSAssistant",
-        "mtab": "sand.plugins.mtab.MTabAssistant",
+        # "grams": "sand.extensions.grams.GRAMSAssistant",
+        "mtab": "sand.extensions.assistants.mtab.MTabAssistant",
         # "default": "mtab",
     },
+    "exports": {
+        "drepr": "sand.extensions.export.drepr.main.DreprExport",
+        "default": "sand.extensions.export.drepr.main.DreprExport"
+    }
 }

sand/controllers/assistant.py

@@ -17,29 +17,21 @@
 from sand.serializer import batch_serialize_sms, get_label, serialize_graph
 from sand.models.entity import NIL_ENTITY, Entity, EntityAR
 from sand.models.ontology import OntProperty, OntClass, OntPropertyAR, OntClassAR
+from sand.extension_interface.assistant import IAssistant
 from operator import attrgetter
 
 from peewee import Model as PeeweeModel, DoesNotExist, fn
 
 from werkzeug.exceptions import BadRequest, NotFound
 
 
-class Assistant(ABC):
-    @abstractmethod
-    def predict(
-        self, table: Table, rows: List[TableRow]
-    ) -> Tuple[Optional[SemanticModel], Optional[List[TableRow]]]:
-        """Predict semantic model and link entities"""
-        pass
-
-
 assistant_bp = Blueprint("assistant", "assistant")
 
 
 GetAssistantCache = threading.local()
 
 
-def get_assistants() -> Dict[str, Assistant]:
+def get_assistants() -> Dict[str, IAssistant]:
     """Get all assistants"""
     global GetAssistantCache
 
@@ -52,7 +44,7 @@ def get_assistants() -> Dict[str, Assistant]:
 
 
 @assistant_bp.route(f"/{assistant_bp.name}/predict/<table_id>", methods=["GET"])
-def predict(table_id: int):
+def predict_semantic_desc(table_id: int):
 
     table = Table.get_by_id(table_id)
     rows: List[TableRow] = list(

sand/controllers/table.py

@@ -8,18 +8,21 @@
     get_dataclass_deserializer,
     get_deserializer_from_type,
 )
-from rsoup.rsoup import ContentHierarchy
 from sand.models import SemanticModel, Table, TableRow
 from sand.models.ontology import OntClassAR, OntPropertyAR
 from sand.models.table import Link
-from sand.plugins.drepr.relational2rdf import relational2rdf
+from sand.extensions.export.drepr.main import DreprExport
 from sand.serializer import (
     get_label,
 )
+import threading
+from sand.extension_interface.export import IExport
+from sm.misc.funcs import import_func
+from sand.config import SETTINGS
 import sm.outputs.semantic_model as O
 from flask import jsonify, request, make_response
 from peewee import DoesNotExist, fn
-
+from drepr.engine import OutputFormat
 from werkzeug.exceptions import BadRequest, NotFound
 
 
@@ -50,6 +53,20 @@ class UpdateColumnLinksInput:
 deser_update_column_links = get_dataclass_deserializer(UpdateColumnLinksInput, {})
 assert deser_update_column_links is not None
 
+GetExportCache = threading.local()
+
+
+def get_export(name) -> IExport:
+    global GetExportCache
+
+    if not hasattr(GetExportCache, "export"):
+        GetExportCache.export = {}
+        export_config = SETTINGS["exports"]
+        constructor = export_config[name]
+        GetExportCache.export[name] = import_func(constructor)()
+
+    return GetExportCache.export[name]
+
 
 @table_bp.route(f"/{table_bp.name}/<id>/export-models", methods=["GET"])
 def export_sms(id: int):
@@ -89,7 +106,7 @@ def export_sms(id: int):
 
 
 @table_bp.route(f"/{table_bp.name}/<id>/export", methods=["GET"])
-def export_data(id: int):
+def export_table_data(id: int):
     # load table
     table: Table = Table.get_by_id(id)
 
@@ -122,7 +139,7 @@ def export_data(id: int):
     rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table))
 
     # export the data using drepr library
-    content = relational2rdf(table, rows, sm.data)
+    content = get_export('default').export_data(table, rows, sm.data, OutputFormat.TTL)
     resp = make_response(content)
     resp.headers["Content-Type"] = "text/ttl; charset=utf-8"
     if request.args.get("attachment", "false") == "true":
@@ -154,13 +171,13 @@ def update_cell_link(id: int, column: int):
     if "links" not in request_json:
         raise KeyError(f"Field 'links' is required")
     if not isinstance(request_json["links"], list) or not all(
-        isinstance(link, dict) for link in request_json["links"]
+            isinstance(link, dict) for link in request_json["links"]
     ):
         raise KeyError(f"Field 'links' must be a list of dictionaries")
 
     if (
-        len(request_json["links"]) > 0
-        and "candidate_entities" not in request_json["links"][0]
+            len(request_json["links"]) > 0
+            and "candidate_entities" not in request_json["links"][0]
     ):
         # add back the candidate so we can deserialize them
         for link in request_json["links"]:

sand/deserializer.py

@@ -1,7 +1,7 @@
 from gena.deserializer import get_dataclass_deserializer
 import sm.outputs.semantic_model as O
 from sm.outputs.semantic_model import LiteralNodeDataType
-from sand.plugins.wikidata import get_rel_uri
+from sand.extensions.wikidata import get_rel_uri
 
 
 def deserialize_graph(value) -> O.SemanticModel:

sand/extension_interface/assistant.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional, Tuple
+from sm.outputs.semantic_model import SemanticModel
+from sand.models.table import Table, TableRow
+
+
+class IAssistant(ABC):
+    @abstractmethod
+    def predict(
+            self, table: Table, rows: List[TableRow]
+    ) -> Tuple[Optional[SemanticModel], Optional[List[TableRow]]]:
+        """Predict semantic model and link entities"""
+        pass

sand/extension_interface/export.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+import sm.outputs.semantic_model as O
+from sand.models.table import Table, TableRow
+from drepr.models import DRepr
+from drepr.engine import OutputFormat
+
+
+class IExport(ABC):
+    """
+    Export interface class to export relational data to
+    different data formats
+    """
+
+    @abstractmethod
+    def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr:
+        """export data model using DREPR"""
+        pass
+
+    @abstractmethod
+    def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel,
+                    output_format: OutputFormat):
+        """export relational data using DREPR"""
+        pass

sand/extension_interface/search.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+
+
+class ISearch(ABC):
+    """ Search Interface to support searches from multiple
+        KG datastores.
+    """
+    @abstractmethod
+    def find_class_by_name(self):
+        """Search Class using name"""
+        pass
+
+    @abstractmethod
+    def find_entity_by_name(self):
+        """Search Entity using name"""
+        pass
+
+    @abstractmethod
+    def find_props_by_name(self):
+        """Search properties using name"""
+        pass

sand/plugins/mtab.py → sand/extensions/assistants/mtab.py

@@ -6,13 +6,13 @@
 import serde.prelude as serde
 from sm.namespaces.wikidata import WikidataNamespace
 import sm.outputs.semantic_model as O
-from sand.controllers.assistant import Assistant
+from sand.controllers.assistant import IAssistant
 from sand.models.base import init_db
 
 from sand.models.table import CandidateEntity, Link, Table, TableRow
 
 
-class MTabAssistant(Assistant):
+class MTabAssistant(IAssistant):
     def __init__(self):
         self.cache_dir = Path(f"/tmp/mtab")
         self.cache_dir.mkdir(exist_ok=True)