Merge pull request #45 from averkij/t/master/wiki

T/master/wiki

be/align_processor.py

@@ -133,7 +133,7 @@ def process_batch(self, lines_from_batch, lines_to_batch, line_ids_from, line_id
         # use_proxy_to = self.lines_proxy_to != None #and len(self.lines_proxy_to)>=len(lines_to)
         #print("use_proxy_to", use_proxy_to, len(lines_proxy_to), len(lines_to))
 
-        logging.info(f"Aligning started for {self.db_path}.")
+        logging.info(f"Alignment started for {self.db_path}.")
         try:
             print("batch:", batch_number)
             logging.info(f"Batch {batch_number}. Calculating vectors.")

be/constants.py

@@ -17,6 +17,7 @@
 TYPE_TO = "to"
 
 USER_DB_NAME = "user.db"
+MAIN_DB_NAME = "main.db"
 
 PROC_INIT = 0
 PROC_IN_PROGRESS = 1

be/helper.py

@@ -1,5 +1,6 @@
 """Misc helper functions"""
 
+import datetime
 import glob
 import json
 import logging
@@ -89,6 +90,18 @@ def create_folders(username, lang):
                                   con.DONE_FOLDER, lang)).mkdir(parents=True, exist_ok=True)
 
 
+def init_main_db():
+    """Init main database"""
+    pathlib.Path(os.path.join(con.UPLOAD_FOLDER)
+                 ).mkdir(parents=True, exist_ok=True)
+    main_db_path = os.path.join(con.UPLOAD_FOLDER, con.MAIN_DB_NAME)
+    if not os.path.isfile(main_db_path):
+        logging.info(f"creating main db: {main_db_path}")
+        with sqlite3.connect(main_db_path) as db:
+            db.execute(
+                'create table global_alignments(id integer primary key, username text, lang_from text, lang_to text, guid text, name varchar, state integer, insert_ts text, deleted integer)')
+
+
 def init_document_db(db_path):
     """Init document database (alignment) with tables structure"""
     if os.path.isfile(db_path):
@@ -404,10 +417,14 @@ def alignment_guid_exists(username, guid):
         return bool(cur.fetchone())
 
 
-def register_alignment(username, guid, guid_from, guid_to, name, total_batches):
-    """Register new alignment in database"""
+def register_alignment(username, lang_from, lang_to, guid, guid_from, guid_to, name, total_batches):
+    """Register new alignment in user.db and main.db"""
+    main_db_path = os.path.join(con.UPLOAD_FOLDER, con.MAIN_DB_NAME)
     db_path = os.path.join(con.UPLOAD_FOLDER, username, con.USER_DB_NAME)
     if not alignment_exists(username,  guid_from, guid_to):
+        with sqlite3.connect(main_db_path) as main_db:
+            main_db.execute('insert into global_alignments(guid, username, lang_from, lang_to, name, state, insert_ts, deleted) values (:guid, :username, :lang_from, :lang_to, :name, 2, :insert_ts, 0) ', {
+                "guid": guid, "username": username, "lang_from": lang_from, "lang_to": lang_to, "name": name, "insert_ts": datetime.datetime.utcnow().strftime('%Y-%m-%d_%H:%M:%S')})
         with sqlite3.connect(db_path) as db:
             db.execute('insert into alignments(guid, guid_from, guid_to, name, state, curr_batches, total_batches) values (:guid, :guid_from, :guid_to, :name, 2, 0, :total_batches) ', {
                        "guid": guid, "guid_from": guid_from, "guid_to": guid_to, "name": name, "total_batches": total_batches})
@@ -443,8 +460,13 @@ def update_batch_progress(db_path, batch_id):
             "insert or ignore into batches (batch_id, insert_ts) values (?, datetime('now'))", (batch_id,))
 
 
-def delete_alignment(user_db_path, guid):
+def delete_alignment(username, guid):
     """Mark alignment as deleted"""
+    main_db_path = os.path.join(con.UPLOAD_FOLDER, con.MAIN_DB_NAME)
+    user_db_path = os.path.join(con.UPLOAD_FOLDER, username, con.USER_DB_NAME)
+    with sqlite3.connect(main_db_path) as db:
+        db.execute('update global_alignments set deleted = 1 where guid=:guid', {
+                   "guid": guid})
     with sqlite3.connect(user_db_path) as db:
         db.execute('update alignments set deleted = 1 where guid=:guid', {
                    "guid": guid})

be/main.py

@@ -28,6 +28,7 @@
 @app.route("/items/<username>/init", methods=["GET"])
 def init_userspace(username):
     """Prepare user workspace"""
+    helper.init_main_db()
     helper.init_user_db(username)
     return ('', 200)
 
@@ -172,7 +173,7 @@ def create_alignment(username):
     if config.TEST_RESTRICTION_MAX_BATCHES > 0:
         total_batches = min(config.TEST_RESTRICTION_MAX_BATCHES, total_batches)
 
-    helper.register_alignment(username, align_guid,
+    helper.register_alignment(username, lang_from, lang_to, align_guid,
                               id_from, id_to, name, total_batches)
 
     return ('', 200)
@@ -186,8 +187,7 @@ def delete_alignment(username):
     if not helper.alignment_guid_exists(username, align_guid):
         return ('', 400)
 
-    user_db_path = os.path.join(con.UPLOAD_FOLDER, username, con.USER_DB_NAME)
-    helper.delete_alignment(user_db_path, align_guid)
+    helper.delete_alignment(username, align_guid)
 
     return ('', 200)
 
@@ -257,7 +257,7 @@ def start_alignment(username):
         user_db_path, align_guid, con.PROC_IN_PROGRESS)
 
     # parallel processing
-    logging.info(f"{username}: aligning started")
+    logging.info(f"{username}: align started")
     res_img = os.path.join(con.STATIC_FOLDER, con.IMG_FOLDER,
                            username, f"{align_guid}.png")
     res_img_best = os.path.join(
@@ -322,7 +322,7 @@ def align_next_batch(username):
         user_db_path, align_guid, con.PROC_IN_PROGRESS)
 
     # parallel processing
-    logging.info(f"{username}: aligning started")
+    logging.info(f"{username}: align started")
     res_img = os.path.join(con.STATIC_FOLDER, con.IMG_FOLDER,
                            username, f"{align_guid}.png")
     res_img_best = os.path.join(