-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #673 from nu-radio/add_fallback_antennaserver
Unify download for antenna models / proposal tables / shower library with additional fallback servers
- Loading branch information
Showing
6 changed files
with
111 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
import requests | ||
import os | ||
import filelock | ||
import logging | ||
import shutil | ||
from glob import glob | ||
|
||
logger = logging.getLogger('NuRadioReco.dataservers') | ||
|
||
dataservers = ["https://rnog-data.zeuthen.desy.de", "https://rno-g.uchicago.edu/data/desy-mirror"] | ||
|
||
def get_available_dataservers_by_responsetime(dataservers=dataservers): | ||
""" requests a small index file from the list of dataservers and returns a list of responsive ones ordered by elapsed time """ | ||
response_times = [] | ||
available_dataservers = [] | ||
|
||
for dataserver in dataservers: | ||
# get the index of the shower_library directory, because it is short | ||
testdir = f"{dataserver}/shower_library/" | ||
try: | ||
response = requests.get(testdir, timeout=5) | ||
response.raise_for_status() | ||
except: | ||
continue | ||
response_times.append(response.elapsed) | ||
available_dataservers.append(dataserver) | ||
ranked_dataservers = [x for _, x in sorted(zip(response_times, available_dataservers))] | ||
return ranked_dataservers | ||
|
||
|
||
def download_from_dataserver(remote_path, target_path, unpack_tarball=True, dataservers=dataservers, try_ordered=False): | ||
""" download remote_path to target_path from the list of NuRadio dataservers """ | ||
|
||
folder = os.path.dirname(target_path) | ||
if not os.path.exists(folder): | ||
os.makedirs(folder) | ||
|
||
lockfile = target_path+".lock" | ||
lock = filelock.FileLock(lockfile) | ||
|
||
logger.warning(f"Assuring no other process is downloading. Will wait until {lockfile} is unlocked.") | ||
with lock: | ||
if os.path.isfile(target_path): | ||
logger.warning(f"{target_path} already exists. Maybe download was already completed by another instance?") | ||
return | ||
elif unpack_tarball and (len(glob(os.path.dirname(target_path) + "/*.dat")) > 0): #just check if any .dat files present (similar to NuRadioProposal.py) | ||
logger.warning(f"{os.path.dirname(target_path)} contains .dat files. Maybe download was already completed by another instance?") | ||
return | ||
|
||
if try_ordered: | ||
dataservers = get_available_dataservers_by_responsetime(dataservers) | ||
|
||
requests_status = requests.codes["not_found"] | ||
for dataserver in dataservers: | ||
URL = f'{dataserver}/{remote_path}' | ||
|
||
logger.warning( | ||
"downloading file {} from {}. This can take a while...".format(target_path, URL)) | ||
|
||
try: | ||
r = requests.get(URL) | ||
r.raise_for_status() | ||
requests_status = r.status_code | ||
break | ||
except requests.exceptions.HTTPError as errh: | ||
logger.warning(f"HTTP Error for {dataserver}. Does the file {remote_path} exist on the server?") | ||
pass | ||
except requests.exceptions.ConnectionError as errc: | ||
logger.warning(f"Error Connecting to {dataserver}. Maybe you don't have internet... or the server is down?") | ||
pass | ||
except requests.exceptions.Timeout as errt: | ||
logger.warning(f"Timeout Error for {dataserver}.") | ||
pass | ||
except requests.exceptions.RequestException as err: | ||
logger.warning(f"An unusual error for {dataserver} occurred:", err) | ||
pass | ||
|
||
logger.warning("problem downloading file {} from {}. Let's see if there is another server.".format(target_path, URL)) | ||
|
||
if requests_status != requests.codes["ok"]: | ||
logger.error(f"error in download of file {target_path}. Tried all servers in {dataservers} without success.") | ||
raise IOError | ||
|
||
with open(target_path, "wb") as code: | ||
code.write(r.content) | ||
logger.warning("...download finished.") | ||
|
||
if unpack_tarball and target_path.endswith(".tar.gz"): | ||
target_dir = os.path.dirname(target_path) | ||
logger.warning(f"...unpacking archive to {target_dir}") | ||
shutil.unpack_archive(target_path, target_dir) | ||
os.remove(target_path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters