SYNPY-907 clean up imports

.pep8speaks.yml

@@ -0,0 +1,12 @@
+scanner:
+    diff_only: True  # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned.
+    linter: pycodestyle  # Other option is flake8
+
+pycodestyle:  # Same as scanner.linter value. Other option is flake8
+    max-line-length: 120  # Default is 79 in PEP 8
+    ignore:  # Errors and warnings to ignore
+        # - W504  # line break after binary operator
+        # - E402  # module level import not at top of file
+        # - E731  # do not assign a lambda expression, use a def
+
+no_blank_comment: True  # If True, no comment is made on PR without any errors.

synapseclient/__main__.py

@@ -58,17 +58,18 @@
 import argparse
 import os
 import sys
-import synapseclient
-import synapseutils
-from . import Activity
 import signal
 import json
-from synapseclient.core.exceptions import *
-from .wiki import Wiki
 import getpass
 import csv
 import re
 
+import synapseclient
+import synapseutils
+from . import Activity
+from .wiki import Wiki
+from synapseclient.core.exceptions import *
+
 
 def query(args, syn):
     try:

synapseclient/client.py

@@ -28,19 +28,13 @@
 
 """
 import configparser
-
 import collections
 import os
 import errno
 import sys
 import re
 import time
 import hashlib
-
-from urllib.parse import urlparse
-from urllib.parse import quote
-from urllib.request import urlretrieve
-
 import webbrowser
 import shutil
 import zipfile
@@ -49,21 +43,12 @@
 import warnings
 import getpass
 import json
-from collections import OrderedDict
 import logging
 import deprecated.sphinx
+import urllib.parse as urllib_urlparse
+
 
 import synapseclient
-from synapseclient.core import cache, exceptions
-from synapseclient.core.constants import config_file_constants
-from synapseclient.core.constants import concrete_types
-from synapseclient.core.credentials import UserLoginArgs, get_default_credential_chain
-from synapseclient.core.credentials import cached_sessions
-from synapseclient.core.logging_setup import DEFAULT_LOGGER_NAME, DEBUG_LOGGER_NAME
-from synapseclient.core.exceptions import *
-from synapseclient.core.version_check import version_check
-from synapseclient.core.utils import id_of, get_properties, MB, memoize, _is_json, _extract_synapse_id_from_query, find_data_file_handle,\
-    _extract_zip_file_to_directory, _is_integer, require_param
 from .annotations import from_synapse_annotations, to_synapse_annotations
 from .activity import Activity
 from .entity import Entity, File, Versionable, split_entity_namespaces, is_versionable, is_container, is_synapse_entity
@@ -72,6 +57,16 @@
 from .table import Schema, SchemaBase, Column, TableQueryResult, CsvFileTable, TableAbstractBaseClass
 from .team import UserProfile, Team, TeamMember, UserGroupHeader
 from .wiki import Wiki, WikiAttachment
+from synapseclient.core import cache, exceptions
+from synapseclient.core.constants import config_file_constants
+from synapseclient.core.constants import concrete_types
+from synapseclient.core.credentials import UserLoginArgs, get_default_credential_chain
+from synapseclient.core.credentials import cached_sessions
+from synapseclient.core.logging_setup import DEFAULT_LOGGER_NAME, DEBUG_LOGGER_NAME
+from synapseclient.core.exceptions import *
+from synapseclient.core.version_check import version_check
+from synapseclient.core.utils import id_of, get_properties, MB, memoize, _is_json, _extract_synapse_id_from_query, \
+    find_data_file_handle, _extract_zip_file_to_directory, _is_integer, require_param
 from synapseclient.core.retry import _with_retry
 from synapseclient.core.upload.multipart_upload import multipart_upload_file, multipart_upload_string
 from synapseclient.core.remote_file_storage_wrappers import S3ClientWrapper, SFTPWrapper
@@ -511,7 +506,7 @@ def _findPrincipals(self, query_string):
              {u'displayName': ... }]
 
         """
-        uri = '/userGroupHeaders?prefix=%s' % quote(query_string)
+        uri = '/userGroupHeaders?prefix=%s' % urllib_urlparse.quote(query_string)
         return [UserGroupHeader(**result) for result in self._GET_paginated(uri)]
 
     def onweb(self, entity, subpageId=None):
@@ -1294,7 +1289,7 @@ def _uploadToFileHandleService(self, filename, synapseStore=True, mimetype=None,
         if filename is None:
             raise ValueError('No filename given')
         elif utils.is_url(filename):
-            if synapseStore and urlparse(filename).scheme != 'sftp':
+            if synapseStore and urllib_urlparse.urlparse(filename).scheme != 'sftp':
                 raise NotImplementedError('Automatic storing of external files is not supported.'
                                           ' Please try downloading the file locally first before storing it or set'
                                           ' synapseStore=False')
@@ -1753,7 +1748,7 @@ def _download_from_URL(self, url, destination, fileHandleId=None, expected_md5=N
         delete_on_md5_mismatch = True
         while redirect_count < REDIRECT_LIMIT:
             redirect_count += 1
-            scheme = urlparse(url).scheme
+            scheme = urllib_urlparse.urlparse(url).scheme
             if scheme == 'file':
                 delete_on_md5_mismatch = False
                 destination = utils.file_url_to_path(url, verify_exists=True)
@@ -1765,7 +1760,7 @@ def _download_from_URL(self, url, destination, fileHandleId=None, expected_md5=N
                 destination = SFTPWrapper.download_file(url, destination, username, password)
                 break
             elif scheme == 'ftp':
-                urlretrieve(url, destination)
+                urllib_urlparse.urlretrieve(url, destination)
                 break
             elif scheme == 'http' or scheme == 'https':
                 # if a partial download exists with the temporary name,
@@ -1937,7 +1932,7 @@ def _getUserCredentials(self, url, username=None, password=None):
         """
         # Get authentication information from configFile
 
-        parsedURL = urlparse(url)
+        parsedURL = urllib_urlparse.urlparse(url)
         baseURL = parsedURL.scheme+'://'+parsedURL.hostname
 
         config = self.getConfigFile(self.configPath)
@@ -2090,7 +2085,7 @@ def getEvaluationByName(self, name):
 
         See: :py:mod:`synapseclient.evaluation`
         """
-        uri = Evaluation.getByNameURI(quote(name))
+        uri = Evaluation.getByNameURI(urllib_urlparse.quote(name))
         return Evaluation(**self.restGET(uri))
 
     def getEvaluationByContentSource(self, entity):
@@ -3059,7 +3054,7 @@ def downloadTableColumns(self, table, columns, **kwargs):
         self.logger.info("Downloading %d files, %d cached locally" % (len(file_handle_associations),
                                                                       len(file_handle_to_path_map)))
 
-        permanent_failures = OrderedDict()
+        permanent_failures = collections.OrderedDict()
 
         attempts = 0
         while len(file_handle_associations) > 0 and attempts < MAX_DOWNLOAD_TRIES:
@@ -3130,7 +3125,7 @@ def _build_table_download_file_handle_list(self, table, columns):
         col_indices = [i for i, h in enumerate(table.headers) if h.name in columns]
         # see: http://docs.synapse.org/rest/org/sagebionetworks/repo/model/file/BulkFileDownloadRequest.html
         file_handle_associations = []
-        file_handle_to_path_map = OrderedDict()
+        file_handle_to_path_map = collections.OrderedDict()
         seen_file_handle_ids = set()  # ensure not sending duplicate requests for the same FileHandle IDs
         for row in table:
             for col_index in col_indices:
@@ -3370,7 +3365,7 @@ def _build_uri_and_headers(self, uri, endpoint=None, headers=None):
 
         # Check to see if the URI is incomplete (i.e. a Synapse URL)
         # In that case, append a Synapse endpoint to the URI
-        parsedURL = urlparse(uri)
+        parsedURL = urllib_urlparse.urlparse(uri)
         if parsedURL.netloc == '':
             uri = endpoint + uri
 

synapseclient/core/cache.py

@@ -17,7 +17,8 @@
 import os
 import re
 import shutil
-from math import floor
+import math
+
 from synapseclient.core.lock import Lock
 from synapseclient.core.exceptions import *
 
@@ -53,7 +54,7 @@ def compare_timestamps(modified_time, cached_time):
     if cached_time is None or modified_time is None:
         return False
     if cached_time.endswith(".000Z"):
-        return cached_time == epoch_time_to_iso(floor(modified_time))
+        return cached_time == epoch_time_to_iso(math.floor(modified_time))
     else:
         return cached_time == epoch_time_to_iso(modified_time)
 
@@ -217,7 +218,7 @@ def add(self, file_handle_id, path):
 
             path = utils.normalize_path(path)
             # write .000 milliseconds for backward compatibility
-            cache_map[path] = epoch_time_to_iso(floor(_get_modified_time(path)))
+            cache_map[path] = epoch_time_to_iso(math.floor(_get_modified_time(path)))
             self._write_cache_map(cache_dir, cache_map)
 
         return cache_map

synapseclient/core/credentials/cached_sessions.py

@@ -1,8 +1,8 @@
 import keyring
 import os
 import json
-from keyring.errors import PasswordDeleteError
-from synapseclient.core.cache import CACHE_ROOT_DIR as _DEFAULT_CACHE_ROOT_DIR
+import keyring.errors as keyring_errors
+from synapseclient.core.cache import CACHE_ROOT_DIR
 from synapseclient.core.utils import equal_paths
 
 SYNAPSE_CACHED_SESSION_APLICATION_NAME = "SYNAPSE.ORG_CLIENT"
@@ -24,7 +24,7 @@ def get_api_key(username):
 def remove_api_key(username):
         try:
             keyring.delete_password(SYNAPSE_CACHED_SESSION_APLICATION_NAME, username)
-        except PasswordDeleteError:
+        except keyring_errors.PasswordDeleteError:
             # The API key does not exist, but that is fine
             pass
 
@@ -68,7 +68,7 @@ def migrate_old_session_file_credentials_if_necessary(syn):
 
     # only migrate if the download cache is in the default location (i.e. user did not set its location)
     # we don't want to migrate credentials if they were a part of a cache shared by multiple people
-    if equal_paths(syn.cache.cache_root_dir, os.path.expanduser(_DEFAULT_CACHE_ROOT_DIR)):
+    if equal_paths(syn.cache.cache_root_dir, os.path.expanduser(CACHE_ROOT_DIR)):
         # iterate through the old file and place in new credential storage
         old_session_dict = _read_session_cache(old_session_file_path)
         for key, value in old_session_dict.items():

synapseclient/core/credentials/cred_data.py

@@ -1,12 +1,11 @@
-from collections import namedtuple
-
-import synapseclient.core.utils
 import time
 import base64
 import hmac
 import hashlib
+import collections
+import urllib.parse as urllib_parse
 
-from urllib.parse import urlparse
+import synapseclient.core.utils
 
 
 # TODO: inherit requests.AuthBase so that this object can be simply passed to requests library
@@ -35,7 +34,7 @@ def get_signed_headers(self, url):
         :return:
         """
         sig_timestamp = time.strftime(synapseclient.core.utils.ISO_FORMAT, time.gmtime())
-        url = urlparse(url).path
+        url = urllib_parse.urlparse(url).path
         sig_data = self.username + url + sig_timestamp
         signature = base64.b64encode(hmac.new(self._api_key,
                                               sig_data.encode('utf-8'),
@@ -51,6 +50,7 @@ def __repr__(self):
 
 # a class that just contains args passed form synapse client login
 # TODO remove deprecated sessionToken
-UserLoginArgs = namedtuple('UserLoginArgs', ['username', 'password', 'api_key', 'skip_cache', 'session_token'])
+UserLoginArgs = collections.namedtuple('UserLoginArgs',
+                                       ['username', 'password', 'api_key', 'skip_cache', 'session_token'])
 # make the namedtuple's arguments optional instead of positional. All values default to None
 UserLoginArgs.__new__.__defaults__ = (None,) * len(UserLoginArgs._fields)

synapseclient/core/credentials/credential_provider.py

@@ -1,16 +1,17 @@
-from abc import ABCMeta, abstractmethod
+import abc
+import deprecated.sphinx
+
 from .cred_data import SynapseCredentials
 from . import cached_sessions
-import deprecated.sphinx
 
 
-class SynapseCredentialsProvider(metaclass=ABCMeta):
+class SynapseCredentialsProvider(metaclass=abc.ABCMeta):
     """
     A credential provider is responsible for retrieving synapse authentication information (e.g. username/password or
     username/api key) from a source(e.g. login args, config file, cached credentials in keyring), and use them to return
     a ``SynapseCredentials` instance.
     """
-    @abstractmethod
+    @abc.abstractmethod
     def _get_auth_info(self, syn, user_login_args):
         """
         Subclasses must implement this to decide how to obtain username, password, and api_key.

synapseclient/core/exceptions.py

@@ -13,7 +13,8 @@
 """
 
 import requests
-import synapseclient.core.utils as utils
+
+from synapseclient.core import utils
 
 
 class SynapseError(Exception):

synapseclient/core/lock.py

@@ -3,12 +3,13 @@
 import shutil
 import sys
 import time
-from datetime import timedelta
+import datetime
+
 from synapseclient.core.exceptions import *
 from synapseclient.core.dozer import doze
 
-LOCK_DEFAULT_MAX_AGE = timedelta(seconds=10)
-DEFAULT_BLOCKING_TIMEOUT = timedelta(seconds=70)
+LOCK_DEFAULT_MAX_AGE = datetime.timedelta(seconds=10)
+DEFAULT_BLOCKING_TIMEOUT = datetime.timedelta(seconds=70)
 CACHE_UNLOCK_WAIT_TIME = 0.5
 
 

synapseclient/core/logging_setup.py

@@ -2,26 +2,13 @@
 # logging setup
 ################
 
-# import os
-# import errno
+
 import logging
 import logging.config as logging_config
-logging.captureWarnings(True)
-logging.getLogger("requests").setLevel(logging.WARNING)
 
-# _errlog_dir = os.path.expanduser("~/.synapseCache/logs/")
-# _errlog_filename = 'synapseclient_errors.log'
-# _errlog_path = os.path.join(_errlog_dir, _errlog_filename)
 
-
-# create the log file if not exist
-# try:
-#     os.makedirs(_errlog_dir)
-# except OSError as e:
-#     if e.errno != errno.EEXIST:
-#         raise
-# with open(_errlog_path, 'a'):
-#     pass
+logging.captureWarnings(True)
+logging.getLogger("requests").setLevel(logging.WARNING)
 
 DEBUG_LOGGER_NAME = 'synapseclient_debug'
 DEFAULT_LOGGER_NAME = 'synapseclient_default'

synapseclient/core/models/custom_json.py

@@ -1,22 +1,22 @@
 """
 When imported, monkey-patches the 'json' module's encoder with a custom json encoding function.
 """
+import json
+import datetime
 
-from json import JSONEncoder
-from datetime import datetime as Datetime
 from synapseclient.core.utils import datetime_to_iso
 
 
 # monkey-patching JSONEncoder from
 # https://stackoverflow.com/questions/18478287/making-object-json-serializable-with-regular-encoder
 def _json_encoder(self, obj):
-    if isinstance(obj, Datetime):
+    if isinstance(obj, datetime.datetime):
         # backend takes date string format of "yy-M-d H:m:s.SSS" with the time zone being UTC
         return datetime_to_iso(obj, sep=" ").replace("Z", '')
 
     else:
         return getattr(obj.__class__, "to_json", _json_encoder.default)(obj)
 
 
-_json_encoder.default = JSONEncoder().default  # Save unmodified default.
-JSONEncoder.default = _json_encoder  # replacement
+_json_encoder.default = json.JSONEncoder().default  # Save unmodified default.
+json.JSONEncoder.default = _json_encoder  # replacement

synapseclient/core/pool_provider.py

@@ -15,6 +15,7 @@
 
 import multiprocessing
 import multiprocessing.dummy
+
 from . import config