Add test and option to disable online retrieve rules

Snakefile

@@ -48,7 +48,10 @@ wildcard_constraints:
 
 include: "rules/common.smk"
 include: "rules/collect.smk"
-include: "rules/retrieve.smk"
+if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True:
+    include: "rules/retrieve.smk"
+else:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
 include: "rules/build_electricity.smk"
 include: "rules/build_sector.smk"
 include: "rules/solve_electricity.smk"

config/config.default.yaml

@@ -67,6 +67,7 @@ snapshots:
   inclusive: 'left' # include start, not end
 
 enable:
+  retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
   retrieve_sector_databundle: true

rules/common.smk

@@ -23,6 +23,22 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+# Check if the workflow has access to the internet by trying to access the HEAD of specified url
+def has_internet_access(url="www.zenodo.org") -> bool:
+    import http.client as http_client
+
+    # based on answer and comments from
+    # https://stackoverflow.com/a/29854274/11318472
+    conn = http_client.HTTPConnection(url, timeout=5)  # need access to zenodo anyway
+    try:
+        conn.request("HEAD", "/")
+        return True
+    except:
+        return False
+    finally:
+        conn.close()
+
+
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]