Separate fetch from KoreanChatbot (#53, #59)

ko-nlp · Sep 8, 2020 · cb49aca · cb49aca
1 parent cfc6b5a
commit cb49aca
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/Korpora/korpora_chatbot_data.py b/Korpora/korpora_chatbot_data.py
@@ -63,7 +63,6 @@ def __init__(self, root_dir=None, force_download=False):
         if root_dir is None:
             root_dir = default_korpora_path
         local_path = os.path.join(os.path.abspath(root_dir), KOREAN_CHATBOT_CORPUS_INFORMATION[0]['destination'])
-        fetch(KOREAN_CHATBOT_CORPUS_INFORMATION[0]['url'], local_path, 'korean_chatbot_data', force_download)
         f = open(local_path, 'r', encoding='utf-8')
         questions, answers, labels = self.cleaning(csv.reader(f, delimiter=','))
         self.train = KoreanChatbotData(description, questions, answers, labels)
@@ -86,3 +85,11 @@ def get_all_pairs(self):
 
     def get_all_labels(self):
         return self.train.get_all_labels()
+
+
+def fetch_chatbot(root_dir, force_download):
+    for information in KOREAN_CHATBOT_CORPUS_INFORMATION:
+        url = information['url']
+        destination = information['destination']
+        local_path = os.path.join(os.path.abspath(root_dir), destination)
+        fetch(url, local_path, 'korean_chatbot_data', force_download, information['method'])