Merge pull request #48 from aauss/proxy

 Add command-line flags to set proxy for importers

epitator/importers/import_disease_ontology.py

@@ -7,6 +7,7 @@
 from __future__ import print_function
 import rdflib
 import re
+from six.moves.urllib.error import URLError
 from ..get_database_connection import get_database_connection
 from ..utils import batched
 
@@ -42,7 +43,12 @@ def import_disease_ontology(drop_previous=False, root_uri=None):
     )""")
     print("Loading disease ontology...")
     disease_ontology = rdflib.Graph()
-    disease_ontology.parse(DISEASE_ONTOLOGY_URL, format="xml")
+    try:
+        disease_ontology.parse(DISEASE_ONTOLOGY_URL, format="xml")
+    except URLError as e:
+        print(e)
+        print("You might be operating behind a proxy. Try adopting your proxy settings.")
+        return
 
     # Store disease ontology version
     version_query = disease_ontology.query("""

epitator/importers/import_geonames.py

@@ -8,6 +8,7 @@
 from six import BytesIO
 from zipfile import ZipFile
 from six.moves.urllib import request
+from six.moves.urllib.error import URLError
 from ..get_database_connection import get_database_connection
 from ..utils import parse_number, batched, normalize_text
 
@@ -39,7 +40,12 @@
 
 def read_geonames_csv():
     print("Downloading geoname data from: " + GEONAMES_ZIP_URL)
-    url = request.urlopen(GEONAMES_ZIP_URL)
+    try:
+        url = request.urlopen(GEONAMES_ZIP_URL)
+    except URLError as e:
+        print(e)
+        print("You might be operating behind a proxy. Try adopting your proxy settings.")
+        return
     zipfile = ZipFile(BytesIO(url.read()))
     print("Download complete")
     # Loading geonames data may cause errors without this line:

epitator/importers/import_species.py

@@ -9,8 +9,10 @@
 from six import BytesIO
 from zipfile import ZipFile
 from six.moves.urllib import request
+from six.moves.urllib_error import URLError
 from tempfile import NamedTemporaryFile
 import os
+import sys
 from ..get_database_connection import get_database_connection
 from ..utils import batched
 
@@ -22,7 +24,11 @@
 # https://www.itis.gov/pdf/ITIS_ConceptualModelEntityDefinition.pdf
 def download_itis_database():
     print("Downloading ITIS data from: " + ITIS_URL)
-    url = request.urlopen(ITIS_URL)
+    try:
+        url = request.urlopen(ITIS_URL)
+    except URLError:
+        print("You might be operating behind a proxy. Try adopting your proxy settings.")
+        sys.exit(1)
     zipfile = ZipFile(BytesIO(url.read(int(url.headers['content-length']))))
     print("Download complete")
     named_temp_file = NamedTemporaryFile()

epitator/importers/import_wikidata.py

@@ -8,6 +8,7 @@
 from ..get_database_connection import get_database_connection
 from six.moves.urllib.request import urlopen
 from six.moves.urllib.parse import urlencode
+from six.moves.urllib.error import URLError
 import json
 import datetime
 
@@ -31,15 +32,20 @@ def import_wikidata(drop_previous=False):
         return
     cur.execute("INSERT INTO metadata VALUES ('wikidata_retrieval_date', ?)",
                 (datetime.date.today().isoformat(),))
-    response = urlopen("https://query.wikidata.org/sparql", str.encode(urlencode({
-        "format": "json",
-        "query": """
-        SELECT ?item ?itemLabel WHERE {
-          ?item wdt:P31 wd:Q9190427.
-          SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
-        }
-        """
-    })))
+    try:
+        response = urlopen("https://query.wikidata.org/sparql", str.encode(urlencode({
+            "format": "json",
+            "query": """
+            SELECT ?item ?itemLabel WHERE {
+              ?item wdt:P31 wd:Q9190427.
+              SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
+            }
+            """
+        })))
+    except URLError as e:
+        print(e)
+        print("You might be operating behind a proxy. Try adopting your proxy settings.")
+        return
     results = json.loads(response.read())['results']['bindings']
     print("Importing synonyms...")
     cur.executemany("INSERT INTO entities VALUES (?, ?, 'disease', 'Wikidata')", [