From b24ec62fbdd4f23ed76c66c30166e31cd19af3c0 Mon Sep 17 00:00:00 2001 From: Yasamato Date: Wed, 14 Apr 2021 12:49:32 +0200 Subject: [PATCH] Correct api endpoints --- api/app.py | 7 +- api/init.py | 172 +++++++++++++++++++++++++------------------------ api/models.py | 87 ++++++++++++++++--------- api/queries.py | 14 ++-- start.sh | 17 ++--- 5 files changed, 170 insertions(+), 127 deletions(-) diff --git a/api/app.py b/api/app.py index 5b474af4..fd7a1c97 100644 --- a/api/app.py +++ b/api/app.py @@ -41,7 +41,10 @@ def health(): app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False # TODO: test on separate mariadb or postgress database # app.config["SQLALCHEMY_DATABASE_URI"] = "mysql+pymysql://user:pass@some_mariadb/dbname?charset=utf8mb4" - app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///config/data.db" + # internally it will be mapped to "sqlite:///your/path//data.sqlite3" + # or "sqlite:///your/path//app/api/data.sqlite3".... why? + # TODO: move away from sqlite + app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///data.sqlite3" with app.app_context(): from queries import bp as queries_bp @@ -56,5 +59,5 @@ def health(): app.discord = DiscordOAuth2Session(app) db.init_app(app) - # db.create_all() + db.create_all() return app diff --git a/api/init.py b/api/init.py index 503af70a..c056e28b 100644 --- a/api/init.py +++ b/api/init.py @@ -2,9 +2,11 @@ # It will generate the missing data.db with the default value and structure import json +import logging import os -from app import db +from models import Column, TableColumn, Tab, Table, Data +from app import create_app, db anime_type = [ "englishAnimeSites", @@ -121,12 +123,13 @@ def insert_db(table, entry): hasSimKLSupport=(entry["hasSimKLSupport"] if "hasSimKLSupport" in entry else "") ) - con = db.get_db() - con["table_" + table].insert(insert_data) + t = Table.query.filter_by(name=table).first() + d = Data(data=json.dumps(insert_data), table_id=t.id) + db.session.add(d) + db.session.commit() def transfer_table(data, table, old_name): - db.create_table("table_" + table) for entry in data[old_name]: insert_db(table, entry) print("Migrated " + old_name + " -> table_" + table + " to DB.") @@ -138,83 +141,84 @@ def transfer_table(data, table, old_name): with open(os.path.join('../static', 'tables.json')) as json_file: tables_data = json.load(json_file) - con = db.get_db() - - for key in columns_data["keys"].keys(): - column = columns_data["keys"][key] - con["columns"].insert(dict( - key=key, - name=column["name"], - description=column["description"], - type=column["type"] - )) - print("columns table generated") - - for t in columns_data["types"].keys(): - con["table_types"].insert(dict( - name=t - )) - order = 0 - for column in columns_data["types"][t]: - con["type_" + t].insert(dict( - key=column["key"], - hidden=column["hidden"], - order=order - )) - order += 1 - print("table_types table generated") - - for tab in tables_data: - con["tabs"].insert(dict( - key=tab["tab"], - name=tab["name"] - )) - - order = 0 - for table in tab["tables"]: - con["tables"].insert(dict( - key=table["id"], - title=table["title"], - type=table["type"] - )) - - con["tab_" + tab["tab"]].insert(dict( - key=table["id"], - order=order - )) - order += 1 - print("tables table generated") - print("tabs table generated") - - with open(os.path.join("../static", "data.json"), encoding="utf8") as json_file: - old_data = json.load(json_file) - - # streaming sites - transfer_table(old_data, "englishAnimeSites", "englishAnimeSites") - transfer_table(old_data, "foreignAnimeSites", "foreignAnimeSites") - transfer_table(old_data, "downloadSites", "animeDownloadSites") - - # manga/scans - transfer_table(old_data, "englishMangaAggregators", "englishMangaSites") - transfer_table(old_data, "englishMangaScans", "englishMangaScans") - transfer_table(old_data, "foreignMangaAggregators", "foreignMangaSites") - transfer_table(old_data, "foreignMangaScans", "foreignMangaScans") - - # novel - transfer_table(old_data, "lightNovels", "lightNovels") - transfer_table(old_data, "visualNovels", "visualNovels") - - # applications - transfer_table(old_data, "iosApplications", "iOSApplications") - transfer_table(old_data, "androidApplications", "androidApplications") - transfer_table(old_data, "windowsApplications", "windowsApplications") - transfer_table(old_data, "macOSApplications", "macOSApplications") - transfer_table(old_data, "browserExtensions", "browserExtensions") - - # hentai - transfer_table(old_data, "hentaiAnimeSites", "hentaiAnime") - transfer_table(old_data, "hentaiDoujinshiSites", "hentaiDoujinshi") - transfer_table(old_data, "hentaiDownloadSites", "hentaiDownload") - transfer_table(old_data, "hentaiApplications", "hentaiApplications") - - print("Initialization process complete.") + with create_app().app_context(): + for key in columns_data["keys"].keys(): + c = columns_data["keys"][key] + col = Column( + name=c["name"], + column_type=c["type"], + description=c["description"] + ) + db.session.add(col) + db.session.commit() + print("column table generated") + for c in Column.query.all(): + print(c.to_dict()) + + for tab in tables_data: + db.session.add(Tab(name=tab["tab"], description=tab["name"])) + db.session.commit() + order = 0 + gen_tab = Tab.query.filter_by(name=tab["tab"]).first() + for t in tab["tables"]: + db.session.add(Table(name=t["id"], tab_id=gen_tab.id, description=t["title"], order=order)) + order += 1 + db.session.commit() + print("tables table generated") + print("tabs table generated") + for c in Table.query.all(): + print(c.to_dict()) + for c in Tab.query.all(): + print(c.to_dict()) + + for tab in tables_data: + for t in tab["tables"]: + order = 0 + for c in columns_data["types"][t["type"]]: + col = Column.query.filter_by(name=columns_data["keys"][c["key"]]["name"]).first() + table = Table.query.filter_by(name=t["id"]).first() + tc = TableColumn( + table_id=table.id, + column_id=col.id, + order=order, + hidden=c["hidden"] + ) + db.session.add(tc) + order += 1 + db.session.commit() + print("tablecolumn table generated") + for c in TableColumn.query.all(): + print(c.to_dict()) + + with open(os.path.join("../static", "data.json"), encoding="utf8") as json_file: + old_data = json.load(json_file) + + # streaming sites + transfer_table(old_data, "englishAnimeSites", "englishAnimeSites") + transfer_table(old_data, "foreignAnimeSites", "foreignAnimeSites") + transfer_table(old_data, "downloadSites", "animeDownloadSites") + + # manga/scans + transfer_table(old_data, "englishMangaAggregators", "englishMangaSites") + transfer_table(old_data, "englishMangaScans", "englishMangaScans") + transfer_table(old_data, "foreignMangaAggregators", "foreignMangaSites") + transfer_table(old_data, "foreignMangaScans", "foreignMangaScans") + + # novel + transfer_table(old_data, "lightNovels", "lightNovels") + transfer_table(old_data, "visualNovels", "visualNovels") + + # applications + transfer_table(old_data, "iosApplications", "iOSApplications") + transfer_table(old_data, "androidApplications", "androidApplications") + transfer_table(old_data, "windowsApplications", "windowsApplications") + transfer_table(old_data, "macOSApplications", "macOSApplications") + transfer_table(old_data, "browserExtensions", "browserExtensions") + + # hentai + transfer_table(old_data, "hentaiAnimeSites", "hentaiAnime") + transfer_table(old_data, "hentaiDoujinshiSites", "hentaiDoujinshi") + transfer_table(old_data, "hentaiDownloadSites", "hentaiDownload") + transfer_table(old_data, "hentaiApplications", "hentaiApplications") + + print("Initialization process complete.") diff --git a/api/models.py b/api/models.py index 962219c3..d1c92c35 100644 --- a/api/models.py +++ b/api/models.py @@ -1,25 +1,24 @@ from app import db -# TODO: add manipulation of hidden and order -table_columns = db.Table( - 'table_columns', - db.Column('table_id', db.Integer, db.ForeignKey('table.id'), primary_key=True), - db.Column('column_id', db.Integer, db.ForeignKey('column.id'), primary_key=True), - db.Column('hidden', db.Boolean), - db.Column('order', db.Integer) -) - -class Data(db.Model): +class Tab(db.Model): id = db.Column(db.Integer, primary_key=True) - data = db.Column(db.String, nullable=False) - table = db.relationship('Table', backref=db.backref('data', lazy=True)) + name = db.Column(db.String, nullable=False) + description = db.Column(db.String) + tables = db.relationship( + 'Table', + backref="tab", + lazy=True, + uselist=True, + # primaryjoin="foreign(Tab.id) == remote(Table.tab_id)" + ) def to_dict(self): return { "id": self.id, - "data": self.data, - "table": self.table + "name": self.name, + "description": self.description, + "tables": [t.to_dict() for t in self.tables] } @@ -27,39 +26,50 @@ class Table(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String, nullable=False) description = db.Column(db.String) - tab = db.relationship('Tab', backref=db.backref('tables', lazy=True)) - columns = db.relationship('Column', secondary=table_columns, lazy='subquery', - backref=db.backref('tables', lazy=True)) + tab_id = db.Column(db.Integer, db.ForeignKey('tab.id')) + order = db.Column(db.Integer) + hidden = db.Column(db.Boolean, default=False) + columns = db.relationship( + 'TableColumn', + backref="table", + lazy=True, + uselist=True + ) + data = db.relationship( + 'Data', + backref="table", + lazy=True, + uselist=True + ) def to_dict(self): return { "id": self.id, + "tab_id": self.tab_id, "name": self.name, "description": self.description, - "tab": self.tab, - "columns": self.columns, - "data": self.data + "columns": [t.to_dict() for t in self.columns], + "data": [t.to_dict() for t in self.data] } -class Tab(db.Model): +class Data(db.Model): id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String, nullable=False) - description = db.Column(db.String) + data = db.Column(db.String, nullable=False) + table_id = db.Column(db.Integer, db.ForeignKey('table.id')) def to_dict(self): return { "id": self.id, - "name": self.name, - "description": self.description, - "tables": self.tables + "data": self.data, + "table_id": self.table_id } class Column(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String, nullable=False) - column_type = db.Column(db.String) + column_type = db.Column(db.String, default="text") description = db.Column(db.String) def to_dict(self): @@ -67,6 +77,25 @@ def to_dict(self): "id": self.id, "name": self.name, "description": self.description, - "column_type": self.column_type, - "tables": self.tables + "column_type": self.column_type } + + +# TODO: add manipulation of hidden and order +class TableColumn(db.Model): + table_id = db.Column(db.Integer, db.ForeignKey('table.id'), primary_key=True) + column_id = db.Column(db.Integer, db.ForeignKey('column.id'), primary_key=True) + column = db.relationship( + 'Column', + primaryjoin="foreign(TableColumn.column_id) == remote(Column.id)" + ) + order = db.Column(db.Integer) + hidden = db.Column(db.Boolean, default=False) + + def to_dict(self): + return { + "table_id": self.table_id, + "column_id": self.column_id, + "order": self.order, + "hidden": self.hidden + } | self.column.to_dict() diff --git a/api/queries.py b/api/queries.py index d706f514..c9bfe329 100644 --- a/api/queries.py +++ b/api/queries.py @@ -21,7 +21,9 @@ def resolve_tables(): @bp.route("/api/tables/", methods=["GET"]) def resolve_table(table_id): try: - table = Table.query.get(table_id).first_or_404(description=f"table {table_id} not found") + table = Table.query.get(table_id) + if table is None: + return f"table {table_id} not found", 404 return jsonify(table.to_dict()) except Exception as e: return str(e), 500 @@ -41,7 +43,9 @@ def resolve_tabs(): @bp.route("/api/tabs/", methods=["GET"]) def resolve_tab(tab_id): try: - tab = Tab.query.get(tab_id).first_or_404(description=f"tab {tab_id} not found") + tab = Tab.query.get(tab_id) + if tab is None: + return f"tab {tab_id} not found", 404 return jsonify(tab.to_dict()) except Exception as e: return str(e), 500 @@ -53,7 +57,7 @@ def resolve_tab(tab_id): @bp.route("/api/columns", methods=["GET"]) def resolve_columns(): try: - return jsonify([tab.to_dict() for tab in Tab.query.all()]) + return jsonify([tab.to_dict() for tab in Column.query.all()]) except Exception as e: return str(e), 500 @@ -61,7 +65,9 @@ def resolve_columns(): @bp.route("/api/columns/", methods=["GET"]) def resolve_column(column_id): try: - column = Column.query.get(column_id).first_or_404(description=f"column {column_id} not found") + column = Column.query.get(column_id) + if column is None: + return f"column {column_id} not found", 404 return jsonify(column.to_dict()) except Exception as e: return str(e), 500 diff --git a/start.sh b/start.sh index b1b93b17..8ca708b1 100644 --- a/start.sh +++ b/start.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash service nginx restart -cd /app/api - -# migrate if db does not exists -if [ ! -f /config/data.db ]; then - echo "Could not find existing db, trying to run migration script" - python init.py -fi - # generate a new one time flask_secret if [ ! -f /srv/.flask_secret ]; then flask_secret=$(makepasswd) @@ -21,5 +13,14 @@ sponsored_anime=$(makepasswd --minchars=5 --maxchars=20 --string="abcdefghijklmn sed -i "s/sponsoredAnime/${sponsored_anime}/g" /app/static/js/sponsored.js sed -i "s/sponsoredAnime/${sponsored_anime}/g" /app/index.html +mkdir -p /config +cd /app/api || return + +# migrate if db does not exists +if [ ! -f /config/data.db ]; then + echo "Could not find existing db, trying to run migration script" + python init.py +fi + # start the web api gunicorn --workers 3 -b unix:/tmp/gunicorn.sock 'app:create_app()' \ No newline at end of file