Skip to content

Commit

Permalink
fix: changed assert to actual exceptions
Browse files Browse the repository at this point in the history
  • Loading branch information
newgene committed May 4, 2023
1 parent d648aed commit 5a0e1f1
Showing 1 changed file with 11 additions and 11 deletions.
22 changes: 11 additions & 11 deletions biothings/hub/dataload/dumper.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from concurrent.futures import ProcessPoolExecutor
from copy import deepcopy
from datetime import datetime, timezone
from ftplib import FTP
from functools import partial
from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Union
from urllib import parse as urlparse
Expand All @@ -27,6 +28,7 @@
docker_avail = False

import orjson
import requests

from biothings import config as btconfig
from biothings.hub import DUMPER_CATEGORY, UPLOADER_CATEGORY, renderer as job_renderer
Expand Down Expand Up @@ -573,9 +575,6 @@ def prepare_local_folders(self, localfile):
pass


from ftplib import FTP


class FTPDumper(BaseDumper):
FTP_HOST = ""
CWD_DIR = ""
Expand Down Expand Up @@ -778,9 +777,6 @@ def download(self, urlremotefile, localfile, headers={}): # noqa: B006
return ftpdumper.download(remotefile, localfile)


import requests


class HTTPDumper(BaseDumper):
"""Dumper using HTTP protocol and "requests" library"""

Expand Down Expand Up @@ -1321,7 +1317,6 @@ def download(self, remotefile, localfile):


class DumperManager(BaseSourceManager):

SOURCE_CLASS = BaseDumper

def get_source_ids(self):
Expand Down Expand Up @@ -1887,8 +1882,10 @@ def source_config(self):

def prepare_client(self):
self.logger.info("Preparing docker client...")
assert type(self.__class__.SRC_URLS) is list, "SRC_URLS should be a list"
assert self.__class__.SRC_URLS, "SRC_URLS list is empty"
if not isinstance(self.__class__.SRC_URLS, list):
raise DumperException("SRC_URLS should be a list")
if not self.__class__.SRC_URLS:
raise DumperException("SRC_URLS list is empty")
if not self._state["client"]:
docker_connection = btconfig.DOCKER_CONFIG.get(self.source_config["connection_name"])
self.DOCKER_CLIENT_URL = docker_connection.get("client_url")
Expand Down Expand Up @@ -1934,6 +1931,7 @@ def set_release(self):
_release = remote_version.decode().strip()
else:
self.logger.error("Failed to run get_version_cmd. Fallback to timestamp as the release.")
# if _release is None, use timestamp as the release
self.release = _release or datetime.now().strftime("%Y%m%d%H%M%S")

def release_client(self):
Expand Down Expand Up @@ -2008,8 +2006,10 @@ def get_remote_file(self):
return self._data_path

async def create_todump_list(self, force=False, job_manager=None, **kwargs):
assert type(self.__class__.SRC_URLS) is list, "SRC_URLS should be a list"
assert self.__class__.SRC_URLS, "SRC_URLS list is empty"
if not isinstance(self.__class__.SRC_URLS, list):
raise DumperException("SRC_URLS should be a list")
if not self.__class__.SRC_URLS:
raise DumperException("SRC_URLS list is empty")
self.prepare_remote_container()
# unprepare unpicklable objects so we can use multiprocessing
self.unprepare()
Expand Down

0 comments on commit 5a0e1f1

Please sign in to comment.