Merge branch 'master' into roo-viz-part2

onecodex/auth.py

@@ -68,17 +68,14 @@ def _login(server=None, creds_file=None, api_key=None, silent=False):
         with open(creds_file, 'r') as fp:
             try:
                 creds = json.load(fp)
-
-                # this is just so the exception is triggered if api_key and e-mail aren't there
-                creds_email = creds['email']
-                creds_api_key = creds['api_key']
-            except (KeyError, ValueError):
+            except ValueError:
                 click.echo('Your ~/.onecodex credentials file appears to be corrupted. '  # noqa
                            'Please delete it and re-authorize.', err=True)
                 sys.exit(1)
 
         # check for updates if logged in more than one day ago
-        last_update = creds['updated_at'] if creds.get('updated_at') else creds['saved_at']
+        last_update = creds.get('updated_at', creds.get('saved_at'))
+        last_update = last_update if last_update else datetime.datetime.now().strftime(DATE_FORMAT)
         diff = datetime.datetime.now() - datetime.datetime.strptime(last_update,
                                                                     DATE_FORMAT)
         if diff.days >= 1:
@@ -102,11 +99,11 @@ def _login(server=None, creds_file=None, api_key=None, silent=False):
 
         # finally, give the user back what they want (whether silent or not)
         if silent:
-            return creds_api_key
+            return creds.get('api_key', None)
 
-        click.echo('Credentials file already exists ({})'.format(collapse_user(creds_file)),
+        click.echo('Credentials file already exists ({}). Logout first.'.format(collapse_user(creds_file)),
                    err=True)
-        return creds_email
+        return creds.get('email', None)
 
     # creds_file was not found and we're not silent, so prompt user to login
     email, api_key = login_uname_pwd(server, api_key=api_key)
@@ -174,22 +171,40 @@ def _logout(creds_file=None):
 
 
 def login_required(fn):
-    """
-    Decorator for the CLI for requiring login before proceeding.
+    """Requires login before proceeding, but does not prompt the user to login. Decorator should
+    be used only on Click CLI commands.
+
+    Notes
+    -----
+    Different means of authentication will be attempted in this order:
+        1. An API key present in the Click context object from a previous successful authentication.
+        2. A bearer token (ONE_CODEX_BEARER_TOKEN) in the environment.
+        3. An API key (ONE_CODEX_API_KEY) in the environment.
+        4. An API key in the credentials file (~/.onecodex).
     """
 
     @wraps(fn)
     def login_wrapper(ctx, *args, **kwargs):
-        if 'API_KEY' in ctx.obj and ctx.obj['API_KEY'] is not None:
-            ctx.obj['API'] = Api(api_key=ctx.obj['API_KEY'], telemetry=ctx.obj['TELEMETRY'])
+        api_kwargs = {'telemetry': ctx.obj['TELEMETRY']}
+
+        api_key_prior_login = ctx.obj.get('API_KEY')
+        bearer_token_env = os.environ.get('ONE_CODEX_BEARER_TOKEN')
+        api_key_env = os.environ.get('ONE_CODEX_API_KEY')
+        api_key_creds_file = _login(silent=True)
+
+        if api_key_prior_login is not None:
+            api_kwargs['api_key'] = api_key_prior_login
+        elif bearer_token_env is not None:
+            api_kwargs['bearer_token'] = bearer_token_env
+        elif api_key_env is not None:
+            api_kwargs['api_key'] = api_key_env
+        elif api_key_creds_file is not None:
+            api_kwargs['api_key'] = api_key_creds_file
         else:
-            # try and find it
-            api_key = _login(silent=True)
-            if api_key is not None:
-                ctx.obj['API'] = Api(api_key=api_key, telemetry=ctx.obj['TELEMETRY'])
-            else:
-                click.echo('The command you specified requires authentication. Please login first.\n', err=True)
-                ctx.exit()
+            click.echo('The command you specified requires authentication. Please login first.\n', err=True)
+            ctx.exit()
+
+        ctx.obj['API'] = Api(**api_kwargs)
 
         return fn(ctx, *args, **kwargs)
 

onecodex/cli.py

@@ -13,8 +13,7 @@
 
 from onecodex.api import Api
 from onecodex.auth import _login, _logout, _remove_creds, login_required
-from onecodex.exceptions import (ValidationWarning,
-                                 ValidationError, UploadException)
+from onecodex.exceptions import (ValidationWarning, ValidationError)
 from onecodex.metadata_upload import validate_appendables
 from onecodex.scripts import filter_reads
 from onecodex.utils import (cli_resource_fetcher, download_file_helper,
@@ -162,6 +161,8 @@ def samples(ctx, samples):
 @click.option('--max-threads', default=4,
               help=OPTION_HELP['max_threads'], metavar='<int:threads>')
 @click.argument('files', nargs=-1, required=False, type=click.Path(exists=True))
+@click.option('--coerce-ascii', is_flag=True, default=False,
+              help='automatically rename unicode filenames to ASCII')
 @click.option('--forward', type=click.Path(exists=True),
               help=OPTION_HELP['forward'])
 @click.option('--reverse', type=click.Path(exists=True),
@@ -180,7 +181,7 @@ def samples(ctx, samples):
 @telemetry
 @login_required
 def upload(ctx, files, max_threads, clean, no_interleave, prompt, validate,
-           forward, reverse, tags, metadata, project_id):
+           forward, reverse, tags, metadata, project_id, coerce_ascii):
     """Upload a FASTA or FASTQ (optionally gzip'd) to One Codex"""
 
     appendables = {}
@@ -201,12 +202,12 @@ def upload(ctx, files, max_threads, clean, no_interleave, prompt, validate,
 
     if (forward or reverse) and not (forward and reverse):
         click.echo('You must specify both forward and reverse files', err=True)
-        sys.exit(1)
+        ctx.exit(1)
     if forward and reverse:
         if len(files) > 0:
             click.echo('You may not pass a FILES argument when using the '
                        ' --forward and --reverse options.', err=True)
-            sys.exit(1)
+            ctx.exit(1)
         files = [(forward, reverse)]
         no_interleave = True
     if len(files) == 0:
@@ -262,6 +263,7 @@ def upload(ctx, files, max_threads, clean, no_interleave, prompt, validate,
         'metadata': appendables['valid_metadata'],
         'tags': appendables['valid_tags'],
         'project': project_id,
+        'coerce_ascii': coerce_ascii,
     }
 
     try:
@@ -274,12 +276,12 @@ def upload(ctx, files, max_threads, clean, no_interleave, prompt, validate,
         sys.stderr.write('\nERROR: {}. {}'.format(
             e, 'Running with the --clean flag will suppress this error.'
         ))
-        sys.exit(1)
-    except (ValidationError, UploadException, Exception) as e:
+        ctx.exit(1)
+    except ValidationError as e:
         # TODO: Some day improve specific other exception error messages, e.g., gzip CRC IOError
         sys.stderr.write('\nERROR: {}'.format(e))
         sys.stderr.write('\nPlease feel free to contact us for help at help@onecodex.com\n\n')
-        sys.exit(1)
+        ctx.exit(1)
 
 
 @onecodex.command('login')
@@ -299,7 +301,7 @@ def login(ctx):
         if ocx._client.Account.instances()['email'] != email:
             click.echo('Your login credentials do not match the provided email!', err=True)
             _remove_creds()
-            sys.exit(1)
+            ctx.exit(1)
 
 
 @onecodex.command('logout')

onecodex/lib/inline_validator.py

@@ -10,6 +10,7 @@
 from onecodex.exceptions import ValidationError, ValidationWarning
 
 GZIP_COMPRESSION_LEVEL = 5
+MIN_LENGTH_FOR_ANALYSIS = 70
 
 
 # buffer code from
@@ -168,7 +169,7 @@ def _set_total_size(self):
         else:
             try:
                 self.total_size = os.fstat(self.file_obj.fileno()).st_size
-                if self.total_size < 70:
+                if self.total_size < MIN_LENGTH_FOR_ANALYSIS:
                     raise ValidationError('{} is too small to be analyzed: {} bytes'.format(
                         self.name, self.total_size
                     ))
@@ -231,6 +232,7 @@ def _validate_record(self, rec):
 
     def __iter__(self):
         eof = False
+        waiting_for_data = False
         while not eof:
             new_data = self.file_obj.read(self.buffer_read_size)
             # if we're at the end of the file
@@ -249,7 +251,17 @@ def __iter__(self):
             while True:
                 match = self.seq_reader.match(self.unchecked_buffer, end)
                 if match is None:
+                    if (waiting_for_data or eof) and len(self.unchecked_buffer) - end > 0:
+                        raise ValidationError(
+                            'Your FASTA/Q file terminates abruptly or is otherwise malformed.'
+                        )
+
+                    if not eof:
+                        waiting_for_data = True
+
                     break
+                else:
+                    waiting_for_data = False
                 rec = match.groupdict()
                 seq_id, seq, seq_id2, qual = self._validate_record(rec)
                 if self.as_raw:

onecodex/lib/upload.py

@@ -12,6 +12,7 @@
 import requests
 from requests_toolbelt import MultipartEncoder
 from threading import BoundedSemaphore, Thread
+from unidecode import unidecode
 import warnings
 
 from onecodex.exceptions import OneCodexException, UploadException, process_api_error
@@ -76,7 +77,7 @@ def _wrap_files(filename, logger=None, validate=True):
 
 
 def upload(files, session, samples_resource, server_url, threads=DEFAULT_UPLOAD_THREADS,
-           validate=True, log_to=None, metadata=None, tags=None, project=None):
+           validate=True, log_to=None, metadata=None, tags=None, project=None, coerce_ascii=None):
     """
     Uploads several files to the One Codex server, auto-detecting sizes and using the appropriate
     downstream upload functions. Also, wraps the files with a streaming validator to ensure they
@@ -92,6 +93,21 @@ def upload(files, session, samples_resource, server_url, threads=DEFAULT_UPLOAD_
         filenames.append(normalized_filename)
         file_sizes.append(file_size)
 
+    # if filename cannot be represented as ascii, raise and suggest renaming
+    for idx, fname in enumerate(filenames):
+        ascii_fname = unidecode(fname)
+
+        if fname != ascii_fname:
+            if coerce_ascii:
+                if log_to is not None:
+                    log_to.write('Renaming {} to {}, must be ASCII\n'.format(
+                        fname.encode('utf-8'), ascii_fname
+                    ))
+                    log_to.flush()
+                filenames[idx] = ascii_fname
+            else:
+                raise OneCodexException('Filenames must be ascii. Try using --coerce-ascii')
+
     # set up the logging
     bar_length = 20
     if log_to is not None:

onecodex/models/sample.py

@@ -95,7 +95,7 @@ def save(self):
 
     @classmethod
     def upload(cls, filename, threads=None, validate=True, metadata=None, tags=None,
-               project=None, metadata_snake_case=True):
+               project=None, metadata_snake_case=True, coerce_ascii=False):
         """
         Uploads a series of files to the One Codex server. These files are automatically
         validated during upload.
@@ -140,7 +140,8 @@ def upload(cls, filename, threads=None, validate=True, metadata=None, tags=None,
                 project = project_search[0]
 
         samples = upload(filename, res._client.session, res, res._client._root_url + '/', threads=threads,
-                         validate=validate, log_to=sys.stderr, metadata=metadata, tags=tags, project=project)
+                         validate=validate, log_to=sys.stderr, metadata=metadata, tags=tags, project=project,
+                         coerce_ascii=coerce_ascii)
         return samples
         # FIXME: pass the auth into this so we can authenticate the callback?
 

onecodex/utils.py

@@ -333,12 +333,6 @@ def telemetry_wrapper(*args, **kwargs):
 
         try:
             return fn(*args, **kwargs)
-        except SystemExit as e:
-            if client:
-                client.captureException()
-                client.context.clear()
-                sys.stdout = StringIO()  # See: https://github.com/getsentry/raven-python/issues/904
-            sys.exit(e.code)  # make sure we still exit with the proper code
         except Exception as e:
             if client:
                 client.captureException()

onecodex/version.py

@@ -1 +1 @@
-__version__ = '0.4.0'
+__version__ = '0.4.0'

requirements.txt

@@ -8,6 +8,7 @@ raven>=6.1.0
 requests>=2.9
 requests_toolbelt>=0.7.0
 six>=1.10.0
+unidecode==1.0.23
 
 # extensions
 altair>=2.3.0

setup.py

@@ -15,6 +15,27 @@
 
 """
 from setuptools import setup, find_packages
+from setuptools.command.install import install
+
+
+class PostInstallCommand(install):
+    def run(self):
+        install.run(self)
+
+        # try to enable bash completion, if possible
+        import os
+
+        paths_to_try = ['/etc/bash_completion.d', '/usr/local/etc/bash_completion.d']
+
+        for path in paths_to_try:
+            if os.access(path, os.W_OK):
+                try:
+                    with open(os.path.join(path, 'onecodex'), 'w') as f:
+                        f.write('eval "$(_ONECODEX_COMPLETE=source onecodex)"')
+                    print('Enabled bash auto-completion for onecodex')
+                    return
+                except Exception:
+                    print('Unable to enable bash auto-completion for onecodex')
 
 
 with open('onecodex/version.py') as import_file:
@@ -30,22 +51,23 @@
     version=__version__,  # noqa
     packages=find_packages(exclude=['*test*']),
     install_requires=[
-        'boto3>=1.4.2', 
+        'boto3>=1.4.2',
         'click>=6.6',
         'jsonschema>=2.4'
-        'python-dateutil>=2.5.3', 
+        'python-dateutil>=2.5.3',
         'pytz>=2014.1',
         'raven>=6.1.0',
         'requests>=2.9',
         'requests_toolbelt>=0.7.0',
         'six>=1.10.0',
+        'unidecode==1.0.23',
     ],
     include_package_data=True,
     zip_safe=False,
     extras_require={
         'all': [
-            'altair>=2.3.0',
-            'networkx>=1.11,<2.0', 
+            'altair==2.3.0',
+            'networkx>=1.11,<2.0',
             'numpy>=1.11.0',
             'pandas>=0.23.0',
             'scikit-bio>=0.4.2,<0.5.0',
@@ -68,6 +90,7 @@
     dependency_links=[],
     author='Kyle McChesney & Nick Greenfield & Roderick Bovee',
     author_email='opensource@onecodex.com',
+    cmdclass={'install': PostInstallCommand},
     description='One Codex API client and Python library',
     long_description=README,
     long_description_content_type='text/markdown',

tests/conftest.py

@@ -357,12 +357,17 @@ def ocx_schemas():
 
 @pytest.fixture(scope='session')
 def ocx_w_raven():
-    patched_env = {
+    patched_env = os.environ.copy()
+    patch = {
         'ONE_CODEX_API_BASE': 'http://localhost:3000',
         'ONE_CODEX_API_KEY': '1eab4217d30d42849dbde0cd1bb94e39',
         'ONE_CODEX_SENTRY_DSN': 'https://key:pass@sentry.example.com/1',
+        'ONE_CODEX_NO_TELEMETRY': None,
     }
-    with mock.patch.dict(os.environ, patched_env):
+
+    patched_env.update(patch)
+
+    with mock.patch.object(os, 'environ', patched_env):
         with mock_requests(SCHEMA_ROUTES):
             return Api(cache_schema=False, telemetry=True)
 

tests/data/files/François.fq

@@ -0,0 +1,4 @@
+@test
+ATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGC
++
+BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB

tests/data/files/Málaga.fasta

@@ -0,0 +1,2 @@
+>test
+ATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCTAT

tests/data/files/Röö.fastq

@@ -0,0 +1,4 @@
+@test
+ATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGC
++
+BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB