Implement download.py script (SQuAD fetch) #9

cdqa-suite · Feb 14, 2019 · d8493e5 · d8493e5
1 parent d3d8397
commit d8493e5
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/reading-comprehension/pipeline/download.py b/reading-comprehension/pipeline/download.py
@@ -0,0 +1,11 @@
+import wget
+
+squad_urls = [
+    'https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json',
+    'https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json',
+]
+
+for squad_url in squad_urls:
+    wget.download(url=squad_url, out='data')
+
+wget.download(url='https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py')
diff --git a/requirements.txt b/requirements.txt
@@ -4,4 +4,5 @@ matplotlib
 pytorch-pretrained-bert
 beautifulsoup4
 tqdm
+wget
 prettytable