Skip to content

Commit

Permalink
migrate away from using global/static variables
Browse files Browse the repository at this point in the history
  • Loading branch information
Binh Vu committed Nov 6, 2023
1 parent a3b88ec commit 0adb16d
Show file tree
Hide file tree
Showing 24 changed files with 1,236 additions and 1,068 deletions.
3 changes: 3 additions & 0 deletions sand/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from gena.config import GenaConfig

GenaConfig.SERIALIZE_FOREIGN_KEY_FIELD_NAME = "db_field"
10 changes: 7 additions & 3 deletions sand/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from loguru import logger
from peewee import fn
from sand.commands.load import load_dataset
from sand.config import APP_CONFIG, AppConfig
from sand.config import AppConfig
from sand.models import Project, SemanticModel, Table, TableRow
from sand.models import db as dbconn
from sand.models import init_db
Expand Down Expand Up @@ -47,15 +47,19 @@ def start(
init_db(db)

if config is not None:
APP_CONFIG.update(AppConfig.from_yaml(config))
cfg = AppConfig.from_yaml(config)
else:
cfg = AppConfig.default()

if certfile is None or keyfile is None:
ssl_options = None
else:
ssl_options = {"certfile": certfile, "keyfile": keyfile}
assert not wsgi

from sand.app import app
from sand.app import App

app = App(cfg).get_flask_app()

if wsgi:
app.run(host="0.0.0.0", port=port)
Expand Down
153 changes: 93 additions & 60 deletions sand/app.py
Original file line number Diff line number Diff line change
@@ -1,75 +1,108 @@
import os
from itertools import chain

from flask import jsonify
from gena import generate_api, generate_app, generate_readonly_api_4dict
from sand.config import APP_CONFIG
from sand.controllers.assistant import assistant_bp
from sand.config import AppConfig
from sand.controllers.assistant import AssistantController
from sand.controllers.project import project_bp
from sand.controllers.search import search_bp
from sand.controllers.settings import setting_bp
from sand.controllers.table import table_bp, table_row_bp
from sand.controllers.search import SearchController
from sand.controllers.settings import SettingController
from sand.controllers.table import TableController, table_row_bp
from sand.controllers.transform import transform_bp
from sand.deserializer import deserialize_graph
from sand.models import EntityAR, SemanticModel
from sand.models.entity import Entity
from sand.models.ontology import OntClass, OntClassAR, OntProperty, OntPropertyAR
from sand.serializer import (
batch_serialize_sms,
serialize_class,
serialize_entity,
serialize_property,
)
from sand.serde import AppSerde
from sm.misc.funcs import import_func
from werkzeug.exceptions import HTTPException

uri_to_id = APP_CONFIG.get_kgns().uri_to_id

app = generate_app(
[
table_bp,
project_bp,
assistant_bp,
table_row_bp,
setting_bp,
search_bp,
transform_bp,
generate_api(
SemanticModel,
deserializers={"data": deserialize_graph},
batch_serialize=batch_serialize_sms,
),
generate_readonly_api_4dict(
"entities",
serialize=serialize_entity,
id2ent=EntityAR(),
),
generate_readonly_api_4dict(
"classes",
serialize=serialize_class,
id2ent=OntClassAR(),
unique_field_funcs={"uri": uri_to_id},
),
generate_readonly_api_4dict(
"properties",
serialize=serialize_property,
id2ent=OntPropertyAR(),
unique_field_funcs={"uri": uri_to_id},
),
],
os.path.dirname(__file__),
)
class App:
def __init__(self, cfg: AppConfig):
self.cfg = cfg

# knowledge graph namespace
self.kgns = cfg.get_kgns()

@app.errorhandler(HTTPException)
def handle_exception(e):
"""Return JSON instead of HTML for HTTP errors."""
return (
jsonify(
{
"status": "error",
"message": str(e),
}
),
e.code,
)
# KG databases
self.entity_ar = EntityAR(cfg)
self.ontclass_ar = OntClassAR(cfg)
self.ontprop_ar = OntPropertyAR(cfg)

# for serialization and deserialization
self.serde = AppSerde(self)

app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 * 1024 # maximum upload of 16 MB
# mapping from uri to the default resources
self.uri2resource = {}
self.id2resource = {}
for func in [cfg.entity.default, cfg.clazz.default, cfg.property.default]:
for obj in import_func(func)(cfg).values():
obj: Entity | OntClass | OntProperty
assert obj.uri not in self.uri2resource
self.uri2resource[obj.uri] = obj
self.id2resource[obj.id] = obj

def uri_to_id(self, uri: str):
"""Convert an URI to the corresponding ID"""
if uri in self.uri2resource:
return self.uri2resource[uri].id
return self.kgns.uri_to_id(uri)

def id_to_uri(self, id: str):
"""Convert an ID to the corresponding URI"""
if id in self.id2resource:
return self.id2resource[id].uri
return self.kgns.id_to_uri(id)

def get_flask_app(self):
app = generate_app(
[
TableController(self).get_blueprint(),
project_bp,
AssistantController(self).get_blueprint(),
table_row_bp,
SettingController(self).get_blueprint(),
SearchController(self).get_blueprint(),
transform_bp,
generate_api(
SemanticModel,
deserializers={"data": self.serde.deserialize_graph},
batch_serialize=self.serde.batch_serialize_sms,
),
generate_readonly_api_4dict(
"entities",
serialize=self.serde.serialize_entity,
id2ent=self.entity_ar,
),
generate_readonly_api_4dict(
"classes",
serialize=self.serde.serialize_class,
id2ent=self.ontclass_ar,
unique_field_funcs={"uri": self.uri_to_id},
),
generate_readonly_api_4dict(
"properties",
serialize=self.serde.serialize_property,
id2ent=self.ontprop_ar,
unique_field_funcs={"uri": self.uri_to_id},
),
],
os.path.dirname(__file__),
)

@app.errorhandler(HTTPException)
def handle_exception(e):
"""Return JSON instead of HTML for HTTP errors."""
return (
jsonify(
{
"status": "error",
"message": str(e),
}
),
e.code,
)

app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 * 1024 # maximum upload of 16 MB
return app
9 changes: 4 additions & 5 deletions sand/config.default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ entity:
args:
dbfile: /tmp/entities.db
proxy: true
default: sand.models.entity.DEFAULT_ENTITY
default: sand.models.entity.get_default_entities
instanceof:
"http://www.wikidata.org": P31
nil:
Expand All @@ -16,22 +16,21 @@ class:
args:
dbfile: /tmp/classes.db
proxy: true
default: sand.extensions.wikidata.WD_ONT_CLASSES
default: sand.extensions.wikidata.get_default_classes
property:
constructor: sand.extensions.wikidata.get_ontprop_db
args:
dbfile: /tmp/properties.db
proxy: true
default: sand.models.ontology.DEFAULT_ONT_PROPS
default: sand.models.ontology.get_default_properties
semantic_model:
identifiers:
- http://www.w3.org/2000/01/rdf-schema#label
statements:
- http://wikiba.se/ontology#Statement
search:
entity: sand.extensions.search.wikidata_search.extended_wikidata_search
class: sand.extensions.search.wikidata_search.extended_wikidata_search
property: sand.extensions.search.wikidata_search.extended_wikidata_search
ontology: sand.extensions.search.wikidata_search.extended_wikidata_search
export:
default: drepr
drepr: sand.extensions.export.drepr.main.DreprExport
Expand Down
13 changes: 6 additions & 7 deletions sand/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ def from_yaml(infile: Path | str) -> AppConfig:
),
search=SearchConfig(
entity=obj["search"]["entity"],
clazz=obj["search"]["class"],
property=obj["search"]["property"],
ontology=obj["search"]["ontology"],
),
assistant=FnConfig(
default=obj["assistant"].pop("default"),
Expand All @@ -75,6 +74,10 @@ def from_yaml(infile: Path | str) -> AppConfig:
export=FnConfig(default=obj["export"].pop("default"), funcs=obj["export"]),
)

@staticmethod
def default() -> AppConfig:
return AppConfig.from_yaml(os.path.join(PACKAGE_DIR, "config.default.yml"))

def update(self, obj: AppConfig):
for k, v in obj.__dict__.items():
if k.startswith("_"):
Expand All @@ -95,8 +98,7 @@ def _parse_args(obj: dict, cwd: Path):
@dataclass
class SearchConfig:
entity: str
clazz: str
property: str
ontology: str


@dataclass
Expand Down Expand Up @@ -137,6 +139,3 @@ class OntResourceConfig:
class SemanticModelConfig:
identifiers: list[str]
statements: list[str]


APP_CONFIG = AppConfig.from_yaml(os.path.join(PACKAGE_DIR, "config.default.yml"))
Loading

0 comments on commit 0adb16d

Please sign in to comment.