From c60e039cf06b7e046c82e7953f0c4bd3e5685773 Mon Sep 17 00:00:00 2001
From: mageirakos <up1053601@upnet.gr>
Date: Thu, 20 Aug 2020 10:08:59 +0300
Subject: [PATCH] moved dir creation from setup to build_donkeybot; relates #18

---
 scripts/build_donkeybot.py | 46 +++++++++++++++++++++++++-------------
 setup.py                   |  4 ----
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/scripts/build_donkeybot.py b/scripts/build_donkeybot.py
index 3c961d8..4626e9a 100644
--- a/scripts/build_donkeybot.py
+++ b/scripts/build_donkeybot.py
@@ -9,8 +9,13 @@
 # 4) Creates rucio documentation and questions indexes for the SearchEngine
 # 5) Saves all of the above under '/data/data_storage.db'
 
+#bot modules
+import bot.config as config
+
+# general python
 import subprocess
 import argparse
+import os
 
 
 def main():
@@ -30,25 +35,36 @@ def main():
     args = parser.parse_args()
     api_token = args.token
 
-    # fetch and store data
-    subprocess.run(
-        f"python -m scripts.fetch_issues -r rucio/rucio -t {api_token}", shell=True,
-    )
-    subprocess.run(
-        f"python -m scripts.fetch_rucio_docs -t {api_token}", shell=True,
-    )
-    # parse and store data
-    subprocess.run(
-        f"python -m scripts.parse_all", shell=True,
-    )
-    # detect questions in data_storage
-    subprocess.run(
-        f"python -m scripts.detect_all_questions", shell=True,
-    )
+    # # fetch and store data
+    # subprocess.run(
+    #     f"python -m scripts.fetch_issues -r rucio/rucio -t {api_token}", shell=True,
+    # )
+    # subprocess.run(
+    #     f"python -m scripts.fetch_rucio_docs -t {api_token}", shell=True,
+    # )
+    # # parse and store data
+    # subprocess.run(
+    #     f"python -m scripts.parse_all", shell=True,
+    # )
+    # # detect questions in data_storage
+    # subprocess.run(
+    #     f"python -m scripts.detect_all_questions", shell=True,
+    # )
     # create search engine for documents and questions
     subprocess.run(
         f"python -m scripts.create_se_indexes", shell=True,
     )
+    # download BERT models for Question Answering
+    try:
+        os.makedirs(config.DATA_DIR+"models/distilbert-base-cased-distilled-squad")
+        os.makedirs(config.DATA_DIR+"models/bert-large-cased-whole-word-masking-finetuned-squad")
+        os.makedirs(config.DATA_DIR+"models/bert-large-uncased-whole-word-masking-finetuned-squad")
+    except FileExistsError as _e:
+        print(_e)
+        print('moving on...')
+
+    print('Done!')
+
 
 
 if __name__ == "__main__":
diff --git a/setup.py b/setup.py
index 3d18f18..04476b1 100644
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,6 @@
 from setuptools import setup, find_packages
 import os
 
-try:
-    os.mkdir("data")
-except FileExistsError as _e:
-    pass
 
 with open("README.md", "r") as fh:
     long_description = fh.read()