Skip to content

Commit

Permalink
handle tar downloads from RDA (fixes #114)
Browse files Browse the repository at this point in the history
  • Loading branch information
letmaik committed Mar 9, 2019
1 parent 4d1dd4e commit 9376e0a
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions gis4wrf/core/downloaders/met.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
import time
import requests
from pathlib import Path
import glob
import os
import shutil
from datetime import datetime

from .util import download_file_with_progress, requests_retry_session
Expand Down Expand Up @@ -159,6 +162,11 @@ def rda_download_dataset(request_id: str, auth: tuple, path: Path) -> Iterable[T
for file_progress in download_file_with_progress(url, path_tmp / file_name, session=session):
dataset_progress = (i + file_progress) / len(urls)
yield dataset_progress, file_progress, url

# Downloaded files may be tar archives, not always though.
for tar_path in glob.glob(str(path_tmp / '*.tar')):
shutil.unpack_archive(tar_path, path_tmp)
os.remove(tar_path)

path_tmp.rename(path)

Expand Down

0 comments on commit 9376e0a

Please sign in to comment.