From ae4af89d08a70fed87ea2cd5837a27aca3cc9935 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Sun, 28 May 2023 15:31:12 -0700 Subject: [PATCH 01/14] Added WikidataNamespace instance in mtab assistant --- sand/plugins/mtab.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sand/plugins/mtab.py b/sand/plugins/mtab.py index 4b4d15d..75be52d 100644 --- a/sand/plugins/mtab.py +++ b/sand/plugins/mtab.py @@ -12,6 +12,7 @@ from sand.models.table import CandidateEntity, Link, Table, TableRow +wdns = WikidataNamespace.create() class MTabAssistant(Assistant): def __init__(self): self.cache_dir = Path(f"/tmp/mtab") @@ -142,15 +143,15 @@ def create_sm_from_cta_cpa( # somehow, they may end-up predict multiple classes, we need to select one if qnode_id.find(" ") != -1: qnode_id = qnode_id.split(" ")[0] - curl = WikidataNamespace.get_entity_abs_uri(qnode_id) + curl = wdns.get_entity_abs_uri(qnode_id) try: cnode_label = f"{self.id2label[qnode_id]} ({qnode_id})" except KeyError: - cnode_label = WikidataNamespace.get_entity_rel_uri(qnode_id) + cnode_label = wdns.get_entity_rel_uri(qnode_id) cnode = O.ClassNode( abs_uri=curl, - rel_uri=WikidataNamespace.get_entity_rel_uri(qnode_id), + rel_uri=wdns.get_entity_rel_uri(qnode_id), readable_label=cnode_label, ) sm.add_node(dnode) @@ -196,8 +197,8 @@ def create_sm_from_cta_cpa( O.Edge( source=source.id, target=target.id, - abs_uri=WikidataNamespace.get_prop_abs_uri(prop), - rel_uri=WikidataNamespace.get_prop_rel_uri(prop), + abs_uri=wdns.get_prop_abs_uri(prop), + rel_uri=wdns.get_prop_rel_uri(prop), readable_label=f"{self.id2label[prop]} ({prop})", ) ) From 1abd872a0550336c651b476396a5e6b3aa8f4299 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Sun, 28 May 2023 16:51:16 -0700 Subject: [PATCH 02/14] updated WikidataNamespace instance as an instance variable --- sand/plugins/mtab.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sand/plugins/mtab.py b/sand/plugins/mtab.py index 75be52d..468bc9c 100644 --- a/sand/plugins/mtab.py +++ b/sand/plugins/mtab.py @@ -12,12 +12,11 @@ from sand.models.table import CandidateEntity, Link, Table, TableRow -wdns = WikidataNamespace.create() class MTabAssistant(Assistant): def __init__(self): self.cache_dir = Path(f"/tmp/mtab") self.cache_dir.mkdir(exist_ok=True) - + self.wdns = WikidataNamespace.create() self.id2label = {} def predict(self, table: Table, rows: List[TableRow]): @@ -143,15 +142,15 @@ def create_sm_from_cta_cpa( # somehow, they may end-up predict multiple classes, we need to select one if qnode_id.find(" ") != -1: qnode_id = qnode_id.split(" ")[0] - curl = wdns.get_entity_abs_uri(qnode_id) + curl = self.wdns.get_entity_abs_uri(qnode_id) try: cnode_label = f"{self.id2label[qnode_id]} ({qnode_id})" except KeyError: - cnode_label = wdns.get_entity_rel_uri(qnode_id) + cnode_label = self.wdns.get_entity_rel_uri(qnode_id) cnode = O.ClassNode( abs_uri=curl, - rel_uri=wdns.get_entity_rel_uri(qnode_id), + rel_uri=self.wdns.get_entity_rel_uri(qnode_id), readable_label=cnode_label, ) sm.add_node(dnode) @@ -197,8 +196,8 @@ def create_sm_from_cta_cpa( O.Edge( source=source.id, target=target.id, - abs_uri=wdns.get_prop_abs_uri(prop), - rel_uri=wdns.get_prop_rel_uri(prop), + abs_uri=self.wdns.get_prop_abs_uri(prop), + rel_uri=self.wdns.get_prop_rel_uri(prop), readable_label=f"{self.id2label[prop]} ({prop})", ) ) From 0927b90311eb634b263e768420c4b62b9ae28c7e Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Tue, 30 May 2023 02:25:55 -0700 Subject: [PATCH 03/14] Added extension interfaces and moved plugin implementations to extensions module --- sand/commands/load.py | 7 ++----- sand/config.py | 16 +++++++------- sand/controllers/assistant.py | 12 ++--------- sand/controllers/table.py | 2 +- sand/deserializer.py | 2 +- .../drepr => extension_interface}/__init__.py | 0 sand/extension_interface/assistant.py | 13 ++++++++++++ sand/extension_interface/search.py | 21 +++++++++++++++++++ sand/extension_interface/transformation.py | 6 ++++++ sand/{plugins => extensions}/__init__.py | 0 sand/extensions/assistants/__init__.py | 0 .../assistants}/grams_plugin.py | 0 .../assistants}/mtab.py | 4 ++-- sand/extensions/drepr/__init__.py | 0 .../raw_transformations/global_coordinate.py | 0 .../drepr/raw_transformations/number.py | 0 .../drepr/relational2rdf.py | 6 +++--- .../drepr/resources.py | 0 .../drepr/semanticmodel.py | 0 .../drepr/transformation.py | 0 sand/extensions/search/__init__.py | 0 sand/extensions/transformations/__init__.py | 0 sand/{plugins => extensions}/wikidata.py | 0 23 files changed, 59 insertions(+), 30 deletions(-) rename sand/{plugins/drepr => extension_interface}/__init__.py (100%) create mode 100644 sand/extension_interface/assistant.py create mode 100644 sand/extension_interface/search.py create mode 100644 sand/extension_interface/transformation.py rename sand/{plugins => extensions}/__init__.py (100%) create mode 100644 sand/extensions/assistants/__init__.py rename sand/{plugins => extensions/assistants}/grams_plugin.py (100%) rename sand/{plugins => extensions/assistants}/mtab.py (98%) create mode 100644 sand/extensions/drepr/__init__.py rename sand/{plugins => extensions}/drepr/raw_transformations/global_coordinate.py (100%) rename sand/{plugins => extensions}/drepr/raw_transformations/number.py (100%) rename sand/{plugins => extensions}/drepr/relational2rdf.py (94%) rename sand/{plugins => extensions}/drepr/resources.py (100%) rename sand/{plugins => extensions}/drepr/semanticmodel.py (100%) rename sand/{plugins => extensions}/drepr/transformation.py (100%) create mode 100644 sand/extensions/search/__init__.py create mode 100644 sand/extensions/transformations/__init__.py rename sand/{plugins => extensions}/wikidata.py (100%) diff --git a/sand/commands/load.py b/sand/commands/load.py index e919d5d..ab8328e 100644 --- a/sand/commands/load.py +++ b/sand/commands/load.py @@ -1,18 +1,15 @@ -import glob -import os from pathlib import Path import click from tqdm.auto import tqdm -from loguru import logger from sand.models import ( Project, SemanticModel, db as dbconn, init_db, ) -from sand.plugins.grams_plugin import convert_linked_table -from sm.dataset import Example, Dataset, FullTable +from sand.extensions.assistants.grams_plugin import convert_linked_table +from sm.dataset import Example, Dataset from grams.inputs import LinkedTable diff --git a/sand/config.py b/sand/config.py index 1b3ce91..83cc44b 100644 --- a/sand/config.py +++ b/sand/config.py @@ -11,7 +11,7 @@ SETTINGS = { "entity": { - "constructor": "sand.plugins.wikidata.get_entity_db", + "constructor": "sand.extensions.wikidata.get_entity_db", "uri2id": "sm.namespaces.prelude.WikidataNamespace.get_entity_id", "id2uri": "sm.namespaces.prelude.WikidataNamespace.get_entity_abs_uri", "args": { @@ -30,18 +30,18 @@ "new_entity_template": "http://www.wikidata.org/entity/{id}", }, "ont_classes": { - "constructor": "sand.plugins.wikidata.get_ontclass_db", - "uri2id": "sand.plugins.wikidata.uri2id", + "constructor": "sand.extensions.wikidata.get_ontclass_db", + "uri2id": "sand.extensions.wikidata.uri2id", "args": { "dbfile": "/tmp/wdclasses.db", "proxy": True, }, # extra classes - "default": "sand.plugins.wikidata.WD_ONT_CLASSES", + "default": "sand.extensions.wikidata.WD_ONT_CLASSES", }, "ont_props": { - "constructor": "sand.plugins.wikidata.get_ontprop_db", - "uri2id": "sand.plugins.wikidata.uri2id", + "constructor": "sand.extensions.wikidata.get_ontprop_db", + "uri2id": "sand.extensions.wikidata.uri2id", "args": { "dbfile": "/tmp/wdprops.db", "proxy": True, @@ -59,8 +59,8 @@ }, "assistants": { # list of assistants' names and their models - # "grams": "sand.plugins.grams.GRAMSAssistant", - "mtab": "sand.plugins.mtab.MTabAssistant", + # "grams": "sand.extensions.grams.GRAMSAssistant", + "mtab": "sand.extensions.assistants.mtab.MTabAssistant", # "default": "mtab", }, } diff --git a/sand/controllers/assistant.py b/sand/controllers/assistant.py index c1a519c..5974fa2 100644 --- a/sand/controllers/assistant.py +++ b/sand/controllers/assistant.py @@ -17,6 +17,7 @@ from sand.serializer import batch_serialize_sms, get_label, serialize_graph from sand.models.entity import NIL_ENTITY, Entity, EntityAR from sand.models.ontology import OntProperty, OntClass, OntPropertyAR, OntClassAR +from sand.extension_interface.assistant import IAssistant from operator import attrgetter from peewee import Model as PeeweeModel, DoesNotExist, fn @@ -24,22 +25,13 @@ from werkzeug.exceptions import BadRequest, NotFound -class Assistant(ABC): - @abstractmethod - def predict( - self, table: Table, rows: List[TableRow] - ) -> Tuple[Optional[SemanticModel], Optional[List[TableRow]]]: - """Predict semantic model and link entities""" - pass - - assistant_bp = Blueprint("assistant", "assistant") GetAssistantCache = threading.local() -def get_assistants() -> Dict[str, Assistant]: +def get_assistants() -> Dict[str, IAssistant]: """Get all assistants""" global GetAssistantCache diff --git a/sand/controllers/table.py b/sand/controllers/table.py index e8321d0..189b936 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -12,7 +12,7 @@ from sand.models import SemanticModel, Table, TableRow from sand.models.ontology import OntClassAR, OntPropertyAR from sand.models.table import Link -from sand.plugins.drepr.relational2rdf import relational2rdf +from sand.extensions.drepr.relational2rdf import relational2rdf from sand.serializer import ( get_label, ) diff --git a/sand/deserializer.py b/sand/deserializer.py index 76b7ed8..178f029 100644 --- a/sand/deserializer.py +++ b/sand/deserializer.py @@ -1,7 +1,7 @@ from gena.deserializer import get_dataclass_deserializer import sm.outputs.semantic_model as O from sm.outputs.semantic_model import LiteralNodeDataType -from sand.plugins.wikidata import get_rel_uri +from sand.extensions.wikidata import get_rel_uri def deserialize_graph(value) -> O.SemanticModel: diff --git a/sand/plugins/drepr/__init__.py b/sand/extension_interface/__init__.py similarity index 100% rename from sand/plugins/drepr/__init__.py rename to sand/extension_interface/__init__.py diff --git a/sand/extension_interface/assistant.py b/sand/extension_interface/assistant.py new file mode 100644 index 0000000..ed59790 --- /dev/null +++ b/sand/extension_interface/assistant.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, Tuple +from sm.outputs.semantic_model import SemanticModel +from sand.models.table import Table, TableRow + + +class IAssistant(ABC): + @abstractmethod + def predict( + self, table: Table, rows: List[TableRow] + ) -> Tuple[Optional[SemanticModel], Optional[List[TableRow]]]: + """Predict semantic model and link entities""" + pass diff --git a/sand/extension_interface/search.py b/sand/extension_interface/search.py new file mode 100644 index 0000000..861b1cd --- /dev/null +++ b/sand/extension_interface/search.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod + + +class ISearch(ABC): + """ Search Interface to support searches from multiple + KG datastores. + """ + @abstractmethod + def find_class_by_name(self): + """Search Class using name""" + pass + + @abstractmethod + def find_entity_by_name(self): + """Search Entity using name""" + pass + + @abstractmethod + def find_props_by_name(self): + """Search properties using name""" + pass diff --git a/sand/extension_interface/transformation.py b/sand/extension_interface/transformation.py new file mode 100644 index 0000000..e7ab75e --- /dev/null +++ b/sand/extension_interface/transformation.py @@ -0,0 +1,6 @@ +from abc import ABC + + +class ITransformation(ABC): + pass + diff --git a/sand/plugins/__init__.py b/sand/extensions/__init__.py similarity index 100% rename from sand/plugins/__init__.py rename to sand/extensions/__init__.py diff --git a/sand/extensions/assistants/__init__.py b/sand/extensions/assistants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sand/plugins/grams_plugin.py b/sand/extensions/assistants/grams_plugin.py similarity index 100% rename from sand/plugins/grams_plugin.py rename to sand/extensions/assistants/grams_plugin.py diff --git a/sand/plugins/mtab.py b/sand/extensions/assistants/mtab.py similarity index 98% rename from sand/plugins/mtab.py rename to sand/extensions/assistants/mtab.py index 468bc9c..b7ce6dd 100644 --- a/sand/plugins/mtab.py +++ b/sand/extensions/assistants/mtab.py @@ -6,13 +6,13 @@ import serde.prelude as serde from sm.namespaces.wikidata import WikidataNamespace import sm.outputs.semantic_model as O -from sand.controllers.assistant import Assistant +from sand.controllers.assistant import IAssistant from sand.models.base import init_db from sand.models.table import CandidateEntity, Link, Table, TableRow -class MTabAssistant(Assistant): +class MTabAssistant(IAssistant): def __init__(self): self.cache_dir = Path(f"/tmp/mtab") self.cache_dir.mkdir(exist_ok=True) diff --git a/sand/extensions/drepr/__init__.py b/sand/extensions/drepr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sand/plugins/drepr/raw_transformations/global_coordinate.py b/sand/extensions/drepr/raw_transformations/global_coordinate.py similarity index 100% rename from sand/plugins/drepr/raw_transformations/global_coordinate.py rename to sand/extensions/drepr/raw_transformations/global_coordinate.py diff --git a/sand/plugins/drepr/raw_transformations/number.py b/sand/extensions/drepr/raw_transformations/number.py similarity index 100% rename from sand/plugins/drepr/raw_transformations/number.py rename to sand/extensions/drepr/raw_transformations/number.py diff --git a/sand/plugins/drepr/relational2rdf.py b/sand/extensions/drepr/relational2rdf.py similarity index 94% rename from sand/plugins/drepr/relational2rdf.py rename to sand/extensions/drepr/relational2rdf.py index c6eaabb..ef38e2a 100644 --- a/sand/plugins/drepr/relational2rdf.py +++ b/sand/extensions/drepr/relational2rdf.py @@ -31,9 +31,9 @@ from drepr.engine import execute, FileOutput, OutputFormat, MemoryOutput from uuid import uuid4 from slugify import slugify -from sand.plugins.drepr.resources import get_entity_resource, get_table_resource -from sand.plugins.drepr.semanticmodel import get_drepr_sm, get_entity_data_nodes -from sand.plugins.drepr.transformation import has_transformation, get_transformation +from sand.extensions.drepr.resources import get_entity_resource, get_table_resource +from sand.extensions.drepr.semanticmodel import get_drepr_sm, get_entity_data_nodes +from sand.extensions.drepr.transformation import has_transformation, get_transformation def create_dataset_model(table: Table, sm: O.SemanticModel) -> DRepr: diff --git a/sand/plugins/drepr/resources.py b/sand/extensions/drepr/resources.py similarity index 100% rename from sand/plugins/drepr/resources.py rename to sand/extensions/drepr/resources.py diff --git a/sand/plugins/drepr/semanticmodel.py b/sand/extensions/drepr/semanticmodel.py similarity index 100% rename from sand/plugins/drepr/semanticmodel.py rename to sand/extensions/drepr/semanticmodel.py diff --git a/sand/plugins/drepr/transformation.py b/sand/extensions/drepr/transformation.py similarity index 100% rename from sand/plugins/drepr/transformation.py rename to sand/extensions/drepr/transformation.py diff --git a/sand/extensions/search/__init__.py b/sand/extensions/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sand/extensions/transformations/__init__.py b/sand/extensions/transformations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sand/plugins/wikidata.py b/sand/extensions/wikidata.py similarity index 100% rename from sand/plugins/wikidata.py rename to sand/extensions/wikidata.py From f5c24c8104089ca4980df292a16d9ade300b13d1 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Wed, 31 May 2023 09:51:13 -0700 Subject: [PATCH 04/14] Renamed transformations interface and added export interface implementation --- sand/controllers/assistant.py | 2 +- sand/extension_interface/export.py | 18 ++++++++++++++++++ sand/extension_interface/transformation.py | 6 ------ .../{transformations => export}/__init__.py | 0 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 sand/extension_interface/export.py delete mode 100644 sand/extension_interface/transformation.py rename sand/extensions/{transformations => export}/__init__.py (100%) diff --git a/sand/controllers/assistant.py b/sand/controllers/assistant.py index 5974fa2..03a9e48 100644 --- a/sand/controllers/assistant.py +++ b/sand/controllers/assistant.py @@ -44,7 +44,7 @@ def get_assistants() -> Dict[str, IAssistant]: @assistant_bp.route(f"/{assistant_bp.name}/predict/", methods=["GET"]) -def predict(table_id: int): +def predict_semantic_desc(table_id: int): table = Table.get_by_id(table_id) rows: List[TableRow] = list( diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py new file mode 100644 index 0000000..c45b201 --- /dev/null +++ b/sand/extension_interface/export.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod + + +class IExport(ABC): + """ + Export interface class to export relational data to + different data formats + """ + + @abstractmethod + def export_data_model(self): + """Search Class using name""" + pass + + @abstractmethod + def export_data(self): + """Search Class using name""" + pass diff --git a/sand/extension_interface/transformation.py b/sand/extension_interface/transformation.py deleted file mode 100644 index e7ab75e..0000000 --- a/sand/extension_interface/transformation.py +++ /dev/null @@ -1,6 +0,0 @@ -from abc import ABC - - -class ITransformation(ABC): - pass - diff --git a/sand/extensions/transformations/__init__.py b/sand/extensions/export/__init__.py similarity index 100% rename from sand/extensions/transformations/__init__.py rename to sand/extensions/export/__init__.py From bfb1213b51a56c9d0d804556be1effd0a210d07f Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Thu, 1 Jun 2023 00:29:23 -0700 Subject: [PATCH 05/14] Updated Drepr module in extensions, implemented DreprExport class --- sand/controllers/table.py | 6 +- sand/extension_interface/export.py | 11 +- sand/extensions/drepr/relational2rdf.py | 215 ++++++++++++------------ 3 files changed, 120 insertions(+), 112 deletions(-) diff --git a/sand/controllers/table.py b/sand/controllers/table.py index 189b936..f5dd6b4 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -12,7 +12,7 @@ from sand.models import SemanticModel, Table, TableRow from sand.models.ontology import OntClassAR, OntPropertyAR from sand.models.table import Link -from sand.extensions.drepr.relational2rdf import relational2rdf +from sand.extensions.drepr.relational2rdf import DreprExport from sand.serializer import ( get_label, ) @@ -89,7 +89,7 @@ def export_sms(id: int): @table_bp.route(f"/{table_bp.name}//export", methods=["GET"]) -def export_data(id: int): +def export_table_data(id: int): # load table table: Table = Table.get_by_id(id) @@ -122,7 +122,7 @@ def export_data(id: int): rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table)) # export the data using drepr library - content = relational2rdf(table, rows, sm.data) + content = DreprExport().export_data(table, rows, sm.data) resp = make_response(content) resp.headers["Content-Type"] = "text/ttl; charset=utf-8" if request.args.get("attachment", "false") == "true": diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py index c45b201..47bb332 100644 --- a/sand/extension_interface/export.py +++ b/sand/extension_interface/export.py @@ -1,4 +1,11 @@ from abc import ABC, abstractmethod +from typing import List + + +import sm.outputs.semantic_model as O +from sand.models.table import Table, TableRow +from sand.models.table import Table, TableRow +from drepr.models import DRepr class IExport(ABC): @@ -8,11 +15,11 @@ class IExport(ABC): """ @abstractmethod - def export_data_model(self): + def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: """Search Class using name""" pass @abstractmethod - def export_data(self): + def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel): """Search Class using name""" pass diff --git a/sand/extensions/drepr/relational2rdf.py b/sand/extensions/drepr/relational2rdf.py index ef38e2a..a497992 100644 --- a/sand/extensions/drepr/relational2rdf.py +++ b/sand/extensions/drepr/relational2rdf.py @@ -31,125 +31,126 @@ from drepr.engine import execute, FileOutput, OutputFormat, MemoryOutput from uuid import uuid4 from slugify import slugify +from sand.extension_interface.export import IExport from sand.extensions.drepr.resources import get_entity_resource, get_table_resource from sand.extensions.drepr.semanticmodel import get_drepr_sm, get_entity_data_nodes from sand.extensions.drepr.transformation import has_transformation, get_transformation -def create_dataset_model(table: Table, sm: O.SemanticModel) -> DRepr: - """Create a D-REPR model of the dataset.""" - columns = [slugify(c).replace("-", "_") for c in table.columns] - get_attr_id = lambda ci: f"{ci}__{columns[ci]}" - get_ent_attr_id = lambda ci: f"{ci}__ent__{columns[ci]}" - ent_dnodes = get_entity_data_nodes(sm) +class DreprExport(IExport): + def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: + """Create a D-REPR model of the dataset.""" + columns = [slugify(c).replace("-", "_") for c in table.columns] + get_attr_id = lambda ci: f"{ci}__{columns[ci]}" + get_ent_attr_id = lambda ci: f"{ci}__ent__{columns[ci]}" + ent_dnodes = get_entity_data_nodes(sm) - attrs = [ - Attr( - id=get_attr_id(ci), - resource_id="table", - path=Path( - steps=[ - RangeExpr(start=1, end=table.size + 1, step=1), - IndexExpr(val=ci), - ] - ), - missing_values=[""], - ) - for ci in range(len(table.columns)) - ] - attrs += [ - Attr( - id=get_ent_attr_id(node.col_index), - resource_id="entity", - path=Path( - steps=[ - RangeExpr(start=1, end=table.size + 1, step=1), - IndexExpr(val=node.col_index), - ] - ), - missing_values=[""], - ) - for node in ent_dnodes - ] - - id2props = ChainedMapping( - OntPropertyAR(), - M.import_attr(SETTINGS["ont_props"]["default"]), - ) - dsm = get_drepr_sm(sm, id2props, get_attr_id, get_ent_attr_id) - - datatype_transformations = [] - for node in sm.nodes(): - if not isinstance(node, O.DataNode): - continue - - datatypes: Set[OntPropertyDataType] = { - id2props[OntProperty.uri2id(inedge.abs_uri)].datatype - for inedge in sm.in_edges(node.id) - } - datatype = list(datatypes)[0] if len(datatypes) == 1 else None - if datatype is None or not has_transformation(datatype): - continue - datatype_transformations.append( - Preprocessing( - type=PreprocessingType.pmap, - value=PMap( - resource_id="table", - path=Path( - steps=[ - RangeExpr(start=1, end=table.size + 1, step=1), - IndexExpr(val=node.col_index), - ] - ), - code=get_transformation(datatype), - change_structure=False, + attrs = [ + Attr( + id=get_attr_id(ci), + resource_id="table", + path=Path( + steps=[ + RangeExpr(start=1, end=table.size + 1, step=1), + IndexExpr(val=ci), + ] ), + missing_values=[""], ) - ) - - return DRepr( - resources=[ - Resource(id="table", type=ResourceType.CSV, prop=CSVProp()), - Resource(id="entity", type=ResourceType.CSV, prop=CSVProp()), - ], - preprocessing=datatype_transformations, - attrs=attrs, - aligns=[ - RangeAlignment( - source=get_attr_id(0), - target=get_attr_id(ci), - aligned_steps=[AlignedStep(source_idx=0, target_idx=0)], - ) - for ci in range(1, len(table.columns)) + for ci in range(len(table.columns)) ] - + [ - RangeAlignment( - source=get_attr_id(0), - target=get_ent_attr_id(node.col_index), - aligned_steps=[AlignedStep(source_idx=0, target_idx=0)], + attrs += [ + Attr( + id=get_ent_attr_id(node.col_index), + resource_id="entity", + path=Path( + steps=[ + RangeExpr(start=1, end=table.size + 1, step=1), + IndexExpr(val=node.col_index), + ] + ), + missing_values=[""], ) for node in ent_dnodes - ], - sm=dsm, - ) + ] + + id2props = ChainedMapping( + OntPropertyAR(), + M.import_attr(SETTINGS["ont_props"]["default"]), + ) + dsm = get_drepr_sm(sm, id2props, get_attr_id, get_ent_attr_id) + + datatype_transformations = [] + for node in sm.nodes(): + if not isinstance(node, O.DataNode): + continue + + datatypes: Set[OntPropertyDataType] = { + id2props[OntProperty.uri2id(inedge.abs_uri)].datatype + for inedge in sm.in_edges(node.id) + } + datatype = list(datatypes)[0] if len(datatypes) == 1 else None + if datatype is None or not has_transformation(datatype): + continue + datatype_transformations.append( + Preprocessing( + type=PreprocessingType.pmap, + value=PMap( + resource_id="table", + path=Path( + steps=[ + RangeExpr(start=1, end=table.size + 1, step=1), + IndexExpr(val=node.col_index), + ] + ), + code=get_transformation(datatype), + change_structure=False, + ), + ) + ) + return DRepr( + resources=[ + Resource(id="table", type=ResourceType.CSV, prop=CSVProp()), + Resource(id="entity", type=ResourceType.CSV, prop=CSVProp()), + ], + preprocessing=datatype_transformations, + attrs=attrs, + aligns=[ + RangeAlignment( + source=get_attr_id(0), + target=get_attr_id(ci), + aligned_steps=[AlignedStep(source_idx=0, target_idx=0)], + ) + for ci in range(1, len(table.columns)) + ] + + [ + RangeAlignment( + source=get_attr_id(0), + target=get_ent_attr_id(node.col_index), + aligned_steps=[AlignedStep(source_idx=0, target_idx=0)], + ) + for node in ent_dnodes + ], + sm=dsm, + ) -def relational2rdf(table: Table, rows: List[TableRow], sm: O.SemanticModel): - """Convert a relational table into RDF format""" - if len(table.columns) == 0: - # no column, no data - return "" + def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel): + """Convert a relational table into RDF format""" + if len(table.columns) == 0: + # no column, no data + return "" - ent_columns = {node.col_index for node in get_entity_data_nodes(sm)} - resources = { - "table": get_table_resource(table, rows), - "entity": get_entity_resource(table, rows, ent_columns), - } + ent_columns = {node.col_index for node in get_entity_data_nodes(sm)} + resources = { + "table": get_table_resource(table, rows), + "entity": get_entity_resource(table, rows, ent_columns), + } - content = execute( - ds_model=create_dataset_model(table, sm), - resources=resources, - output=MemoryOutput(OutputFormat.TTL), - debug=False, - ) - return content + content = execute( + ds_model=self.export_data_model(table, sm), + resources=resources, + output=MemoryOutput(OutputFormat.TTL), + debug=False, + ) + return content From 196f63491455e86a773ae236e125bb7f171ea596 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Thu, 1 Jun 2023 00:56:55 -0700 Subject: [PATCH 06/14] Moved drepr extension under export extension module, renamed drepr script as main.py --- sand/controllers/table.py | 3 +-- .../extensions/{ => export}/drepr/__init__.py | 0 .../drepr/main.py} | 19 +++++-------------- .../drepr/raw_transformations/__init__.py | 0 .../raw_transformations/global_coordinate.py | 0 .../drepr/raw_transformations/number.py | 0 .../{ => export}/drepr/resources.py | 0 .../{ => export}/drepr/semanticmodel.py | 0 .../{ => export}/drepr/transformation.py | 0 9 files changed, 6 insertions(+), 16 deletions(-) rename sand/extensions/{ => export}/drepr/__init__.py (100%) rename sand/extensions/{drepr/relational2rdf.py => export/drepr/main.py} (88%) create mode 100644 sand/extensions/export/drepr/raw_transformations/__init__.py rename sand/extensions/{ => export}/drepr/raw_transformations/global_coordinate.py (100%) rename sand/extensions/{ => export}/drepr/raw_transformations/number.py (100%) rename sand/extensions/{ => export}/drepr/resources.py (100%) rename sand/extensions/{ => export}/drepr/semanticmodel.py (100%) rename sand/extensions/{ => export}/drepr/transformation.py (100%) diff --git a/sand/controllers/table.py b/sand/controllers/table.py index f5dd6b4..b4d8ec2 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -8,11 +8,10 @@ get_dataclass_deserializer, get_deserializer_from_type, ) -from rsoup.rsoup import ContentHierarchy from sand.models import SemanticModel, Table, TableRow from sand.models.ontology import OntClassAR, OntPropertyAR from sand.models.table import Link -from sand.extensions.drepr.relational2rdf import DreprExport +from sand.extensions.export.drepr.main import DreprExport from sand.serializer import ( get_label, ) diff --git a/sand/extensions/drepr/__init__.py b/sand/extensions/export/drepr/__init__.py similarity index 100% rename from sand/extensions/drepr/__init__.py rename to sand/extensions/export/drepr/__init__.py diff --git a/sand/extensions/drepr/relational2rdf.py b/sand/extensions/export/drepr/main.py similarity index 88% rename from sand/extensions/drepr/relational2rdf.py rename to sand/extensions/export/drepr/main.py index a497992..45b0bf2 100644 --- a/sand/extensions/drepr/relational2rdf.py +++ b/sand/extensions/export/drepr/main.py @@ -1,9 +1,5 @@ -from dataclasses import dataclass -from io import StringIO -import orjson, csv -from typing import Dict, List, Set, Tuple, cast +from typing import List, Set from sand.config import SETTINGS -from sand.models.entity import NIL_ENTITY, Entity from sand.models.ontology import OntProperty, OntPropertyAR, OntPropertyDataType from sand.models.table import Table, TableRow import sm.outputs.semantic_model as O @@ -19,22 +15,17 @@ IndexExpr, RangeExpr, RangeAlignment, - ValueAlignment, - AlignmentType, AlignedStep, Preprocessing, PreprocessingType, PMap, - ResourceDataString, ) -import drepr.models.sm as drepr_sm -from drepr.engine import execute, FileOutput, OutputFormat, MemoryOutput -from uuid import uuid4 +from drepr.engine import execute, OutputFormat, MemoryOutput from slugify import slugify from sand.extension_interface.export import IExport -from sand.extensions.drepr.resources import get_entity_resource, get_table_resource -from sand.extensions.drepr.semanticmodel import get_drepr_sm, get_entity_data_nodes -from sand.extensions.drepr.transformation import has_transformation, get_transformation +from sand.extensions.export.drepr.resources import get_entity_resource, get_table_resource +from sand.extensions.export.drepr.semanticmodel import get_drepr_sm, get_entity_data_nodes +from sand.extensions.export.drepr.transformation import has_transformation, get_transformation class DreprExport(IExport): diff --git a/sand/extensions/export/drepr/raw_transformations/__init__.py b/sand/extensions/export/drepr/raw_transformations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sand/extensions/drepr/raw_transformations/global_coordinate.py b/sand/extensions/export/drepr/raw_transformations/global_coordinate.py similarity index 100% rename from sand/extensions/drepr/raw_transformations/global_coordinate.py rename to sand/extensions/export/drepr/raw_transformations/global_coordinate.py diff --git a/sand/extensions/drepr/raw_transformations/number.py b/sand/extensions/export/drepr/raw_transformations/number.py similarity index 100% rename from sand/extensions/drepr/raw_transformations/number.py rename to sand/extensions/export/drepr/raw_transformations/number.py diff --git a/sand/extensions/drepr/resources.py b/sand/extensions/export/drepr/resources.py similarity index 100% rename from sand/extensions/drepr/resources.py rename to sand/extensions/export/drepr/resources.py diff --git a/sand/extensions/drepr/semanticmodel.py b/sand/extensions/export/drepr/semanticmodel.py similarity index 100% rename from sand/extensions/drepr/semanticmodel.py rename to sand/extensions/export/drepr/semanticmodel.py diff --git a/sand/extensions/drepr/transformation.py b/sand/extensions/export/drepr/transformation.py similarity index 100% rename from sand/extensions/drepr/transformation.py rename to sand/extensions/export/drepr/transformation.py From bf7879568dafdee915cfe775c3aa2323cc36b59e Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Thu, 1 Jun 2023 23:18:31 -0700 Subject: [PATCH 07/14] Fixed import error for import_attr function --- sand/extensions/export/drepr/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sand/extensions/export/drepr/main.py b/sand/extensions/export/drepr/main.py index 45b0bf2..0cd02e4 100644 --- a/sand/extensions/export/drepr/main.py +++ b/sand/extensions/export/drepr/main.py @@ -3,7 +3,7 @@ from sand.models.ontology import OntProperty, OntPropertyAR, OntPropertyDataType from sand.models.table import Table, TableRow import sm.outputs.semantic_model as O -import sm.misc as M +from sm.misc.funcs import import_attr from hugedict.chained_mapping import ChainedMapping from drepr.models import ( DRepr, @@ -67,7 +67,7 @@ def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: id2props = ChainedMapping( OntPropertyAR(), - M.import_attr(SETTINGS["ont_props"]["default"]), + import_attr(SETTINGS["ont_props"]["default"]), ) dsm = get_drepr_sm(sm, id2props, get_attr_id, get_ent_attr_id) From 164e27a214c7304dfc81167728f53a8340b342f4 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Fri, 2 Jun 2023 01:14:19 -0700 Subject: [PATCH 08/14] Added output format as a parameter for drepr data exports --- sand/controllers/table.py | 2 +- sand/extension_interface/export.py | 6 +++--- sand/extensions/export/drepr/main.py | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sand/controllers/table.py b/sand/controllers/table.py index b4d8ec2..4010e89 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -121,7 +121,7 @@ def export_table_data(id: int): rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table)) # export the data using drepr library - content = DreprExport().export_data(table, rows, sm.data) + content = DreprExport().export_data(table, rows, sm.data, 'TTL') resp = make_response(content) resp.headers["Content-Type"] = "text/ttl; charset=utf-8" if request.args.get("attachment", "false") == "true": diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py index 47bb332..98f573d 100644 --- a/sand/extension_interface/export.py +++ b/sand/extension_interface/export.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod -from typing import List - +from typing import List, AnyStr import sm.outputs.semantic_model as O from sand.models.table import Table, TableRow @@ -20,6 +19,7 @@ def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: pass @abstractmethod - def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel): + def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, + output_format: AnyStr): """Search Class using name""" pass diff --git a/sand/extensions/export/drepr/main.py b/sand/extensions/export/drepr/main.py index 0cd02e4..02aa286 100644 --- a/sand/extensions/export/drepr/main.py +++ b/sand/extensions/export/drepr/main.py @@ -1,4 +1,4 @@ -from typing import List, Set +from typing import List, Set, AnyStr from sand.config import SETTINGS from sand.models.ontology import OntProperty, OntPropertyAR, OntPropertyDataType from sand.models.table import Table, TableRow @@ -126,7 +126,8 @@ def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: sm=dsm, ) - def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel): + def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, + output_format: AnyStr): """Convert a relational table into RDF format""" if len(table.columns) == 0: # no column, no data @@ -141,7 +142,7 @@ def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel): content = execute( ds_model=self.export_data_model(table, sm), resources=resources, - output=MemoryOutput(OutputFormat.TTL), + output=MemoryOutput(OutputFormat.__getitem__(output_format)), debug=False, ) return content From 712e201faf7a28f562ca6c47be88b145a66e78d4 Mon Sep 17 00:00:00 2001 From: P Punith Krishna Date: Fri, 2 Jun 2023 10:26:24 -0700 Subject: [PATCH 09/14] Updated output format for drepr extension from String to OutputFormat Enum --- sand/controllers/table.py | 4 ++-- sand/extension_interface/export.py | 6 +++--- sand/extensions/export/drepr/main.py | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sand/controllers/table.py b/sand/controllers/table.py index 4010e89..7bb624d 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -18,7 +18,7 @@ import sm.outputs.semantic_model as O from flask import jsonify, request, make_response from peewee import DoesNotExist, fn - +from drepr.engine import OutputFormat from werkzeug.exceptions import BadRequest, NotFound @@ -121,7 +121,7 @@ def export_table_data(id: int): rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table)) # export the data using drepr library - content = DreprExport().export_data(table, rows, sm.data, 'TTL') + content = DreprExport().export_data(table, rows, sm.data, OutputFormat.TTL) resp = make_response(content) resp.headers["Content-Type"] = "text/ttl; charset=utf-8" if request.args.get("attachment", "false") == "true": diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py index 98f573d..5b7e864 100644 --- a/sand/extension_interface/export.py +++ b/sand/extension_interface/export.py @@ -1,10 +1,10 @@ from abc import ABC, abstractmethod -from typing import List, AnyStr +from typing import List import sm.outputs.semantic_model as O from sand.models.table import Table, TableRow -from sand.models.table import Table, TableRow from drepr.models import DRepr +from drepr.engine import OutputFormat class IExport(ABC): @@ -20,6 +20,6 @@ def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: @abstractmethod def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, - output_format: AnyStr): + output_format: OutputFormat): """Search Class using name""" pass diff --git a/sand/extensions/export/drepr/main.py b/sand/extensions/export/drepr/main.py index 02aa286..30768ef 100644 --- a/sand/extensions/export/drepr/main.py +++ b/sand/extensions/export/drepr/main.py @@ -1,4 +1,5 @@ -from typing import List, Set, AnyStr +from typing import List, Set +from enum import Enum from sand.config import SETTINGS from sand.models.ontology import OntProperty, OntPropertyAR, OntPropertyDataType from sand.models.table import Table, TableRow @@ -127,7 +128,7 @@ def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: ) def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, - output_format: AnyStr): + output_format: OutputFormat): """Convert a relational table into RDF format""" if len(table.columns) == 0: # no column, no data @@ -142,7 +143,7 @@ def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, content = execute( ds_model=self.export_data_model(table, sm), resources=resources, - output=MemoryOutput(OutputFormat.__getitem__(output_format)), + output=MemoryOutput(output_format), debug=False, ) return content From 570eef5aa2c702f921693e7844cbe337db963936 Mon Sep 17 00:00:00 2001 From: P PUNITH KRISHNA <32421081+punith300i@users.noreply.github.com> Date: Fri, 2 Jun 2023 11:45:19 -0700 Subject: [PATCH 10/14] Updated export interface comments --- sand/extension_interface/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py index 5b7e864..ab99029 100644 --- a/sand/extension_interface/export.py +++ b/sand/extension_interface/export.py @@ -15,11 +15,11 @@ class IExport(ABC): @abstractmethod def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: - """Search Class using name""" + """Class to export data model""" pass @abstractmethod def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, output_format: OutputFormat): - """Search Class using name""" + """Class to export data""" pass From ef7527dcd2df2ce2762c3f74e683df5de3cbfcd2 Mon Sep 17 00:00:00 2001 From: P PUNITH KRISHNA <32421081+punith300i@users.noreply.github.com> Date: Fri, 2 Jun 2023 11:49:02 -0700 Subject: [PATCH 11/14] Updated export interface comments --- sand/extension_interface/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sand/extension_interface/export.py b/sand/extension_interface/export.py index ab99029..de932a3 100644 --- a/sand/extension_interface/export.py +++ b/sand/extension_interface/export.py @@ -15,11 +15,11 @@ class IExport(ABC): @abstractmethod def export_data_model(self, table: Table, sm: O.SemanticModel) -> DRepr: - """Class to export data model""" + """export data model using DREPR""" pass @abstractmethod def export_data(self, table: Table, rows: List[TableRow], sm: O.SemanticModel, output_format: OutputFormat): - """Class to export data""" + """export relational data using DREPR""" pass From 28f804c4a9af97e131921273f305ac1b0cfc8cb2 Mon Sep 17 00:00:00 2001 From: P PUNITH KRISHNA <32421081+punith300i@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:20:34 -0700 Subject: [PATCH 12/14] Added export config and generalised the usage of IExport implementations --- sand/config.py | 4 +++- sand/controllers/table.py | 26 ++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/sand/config.py b/sand/config.py index 83cc44b..0f7abef 100644 --- a/sand/config.py +++ b/sand/config.py @@ -2,7 +2,6 @@ from pathlib import Path - _ROOT_DIR = Path(os.path.abspath(__file__)).parent.parent PACKAGE_DIR = str(Path(os.path.abspath(__file__)).parent) FROM_SITEPACKAGES = _ROOT_DIR.name == "site-packages" @@ -63,4 +62,7 @@ "mtab": "sand.extensions.assistants.mtab.MTabAssistant", # "default": "mtab", }, + "exports": { + "drepr": "sand.extensions.export.drepr.main.DreprExport" + } } diff --git a/sand/controllers/table.py b/sand/controllers/table.py index 7bb624d..de59d7e 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -15,6 +15,10 @@ from sand.serializer import ( get_label, ) +import threading +from sand.extension_interface.export import IExport +from sm.misc.funcs import import_func +from sand.config import SETTINGS import sm.outputs.semantic_model as O from flask import jsonify, request, make_response from peewee import DoesNotExist, fn @@ -49,6 +53,20 @@ class UpdateColumnLinksInput: deser_update_column_links = get_dataclass_deserializer(UpdateColumnLinksInput, {}) assert deser_update_column_links is not None +GetAssistantCache = threading.local() + + +def get_exports(name) -> IExport: + global GetAssistantCache + + if not hasattr(GetAssistantCache, "exports"): + GetAssistantCache.exports = {} + export_config = SETTINGS["exports"] + constructor = export_config[name] + GetAssistantCache.exports[name] = import_func(constructor)() + + return GetAssistantCache.exports[name] + @table_bp.route(f"/{table_bp.name}//export-models", methods=["GET"]) def export_sms(id: int): @@ -121,7 +139,7 @@ def export_table_data(id: int): rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table)) # export the data using drepr library - content = DreprExport().export_data(table, rows, sm.data, OutputFormat.TTL) + content = get_exports('drepr').export_data(table, rows, sm.data, OutputFormat.TTL) resp = make_response(content) resp.headers["Content-Type"] = "text/ttl; charset=utf-8" if request.args.get("attachment", "false") == "true": @@ -153,13 +171,13 @@ def update_cell_link(id: int, column: int): if "links" not in request_json: raise KeyError(f"Field 'links' is required") if not isinstance(request_json["links"], list) or not all( - isinstance(link, dict) for link in request_json["links"] + isinstance(link, dict) for link in request_json["links"] ): raise KeyError(f"Field 'links' must be a list of dictionaries") if ( - len(request_json["links"]) > 0 - and "candidate_entities" not in request_json["links"][0] + len(request_json["links"]) > 0 + and "candidate_entities" not in request_json["links"][0] ): # add back the candidate so we can deserialize them for link in request_json["links"]: From 36ffa26db58cc8179ab39ebca2b133d21bcc8c71 Mon Sep 17 00:00:00 2001 From: P PUNITH KRISHNA <32421081+punith300i@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:23:13 -0700 Subject: [PATCH 13/14] Updated variable name for local cache for exports --- sand/controllers/table.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sand/controllers/table.py b/sand/controllers/table.py index de59d7e..e233f4c 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -53,19 +53,19 @@ class UpdateColumnLinksInput: deser_update_column_links = get_dataclass_deserializer(UpdateColumnLinksInput, {}) assert deser_update_column_links is not None -GetAssistantCache = threading.local() +GetExportCache = threading.local() def get_exports(name) -> IExport: - global GetAssistantCache + global GetExportCache - if not hasattr(GetAssistantCache, "exports"): - GetAssistantCache.exports = {} + if not hasattr(GetExportCache, "exports"): + GetExportCache.exports = {} export_config = SETTINGS["exports"] constructor = export_config[name] - GetAssistantCache.exports[name] = import_func(constructor)() + GetExportCache.exports[name] = import_func(constructor)() - return GetAssistantCache.exports[name] + return GetExportCache.exports[name] @table_bp.route(f"/{table_bp.name}//export-models", methods=["GET"]) From 79bdb74517c002050e92ea022c04fbdb9962f0e4 Mon Sep 17 00:00:00 2001 From: P PUNITH KRISHNA <32421081+punith300i@users.noreply.github.com> Date: Fri, 2 Jun 2023 18:13:45 -0700 Subject: [PATCH 14/14] Added default export in the export config, updated export logic usage key --- sand/config.py | 3 ++- sand/controllers/table.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sand/config.py b/sand/config.py index 0f7abef..4fb8e88 100644 --- a/sand/config.py +++ b/sand/config.py @@ -63,6 +63,7 @@ # "default": "mtab", }, "exports": { - "drepr": "sand.extensions.export.drepr.main.DreprExport" + "drepr": "sand.extensions.export.drepr.main.DreprExport", + "default": "sand.extensions.export.drepr.main.DreprExport" } } diff --git a/sand/controllers/table.py b/sand/controllers/table.py index e233f4c..d8d21dc 100644 --- a/sand/controllers/table.py +++ b/sand/controllers/table.py @@ -56,16 +56,16 @@ class UpdateColumnLinksInput: GetExportCache = threading.local() -def get_exports(name) -> IExport: +def get_export(name) -> IExport: global GetExportCache - if not hasattr(GetExportCache, "exports"): - GetExportCache.exports = {} + if not hasattr(GetExportCache, "export"): + GetExportCache.export = {} export_config = SETTINGS["exports"] constructor = export_config[name] - GetExportCache.exports[name] = import_func(constructor)() + GetExportCache.export[name] = import_func(constructor)() - return GetExportCache.exports[name] + return GetExportCache.export[name] @table_bp.route(f"/{table_bp.name}//export-models", methods=["GET"]) @@ -139,7 +139,7 @@ def export_table_data(id: int): rows: List[TableRow] = list(TableRow.select().where(TableRow.table == table)) # export the data using drepr library - content = get_exports('drepr').export_data(table, rows, sm.data, OutputFormat.TTL) + content = get_export('default').export_data(table, rows, sm.data, OutputFormat.TTL) resp = make_response(content) resp.headers["Content-Type"] = "text/ttl; charset=utf-8" if request.args.get("attachment", "false") == "true":