Revert "Revert "[WIP] Mega flake8 linting""

This reverts commit 37f1976.

.ci/pep8_sources.txt

@@ -1,26 +1,30 @@
 cron/parse_builds.py
-lib/galaxy/auth
-lib/galaxy/config.py
-lib/galaxy/datatypes/{data,proteomics}.py
-lib/galaxy/datatypes/converters/{bedgraph_to_array_tree_converter,wiggle_to_array_tree_converter}.py
-lib/galaxy/exceptions/error_codes.py
-lib/galaxy/jobs/{__init__,error_level,manager,stock_rules}.py
-lib/galaxy/main.py
-lib/galaxy/managers/{api_keys,citations,collections,collections_util,datasets,folders,pages,tags,workflows}.py
-lib/galaxy/model/orm/{engine_factory,now}.py
+lib/galaxy/{__init__,app,config,main,queue_worker,queues,version}.py
+lib/galaxy/actions/
+lib/galaxy/auth/
+lib/galaxy/dataset_collections/
+lib/galaxy/datatypes/{assembly,binary,checkers,chrominfo,coverage,data,proteomics}.py
+lib/galaxy/datatypes/converters/{bedgraph_to_array_tree_converter,bgzip,wiggle_to_array_tree_converter}.py
+lib/galaxy/eggs/
+lib/galaxy/exceptions/
+lib/galaxy/external_services/
+lib/galaxy/forms/
+lib/galaxy/jobs/
+lib/galaxy/managers/
+lib/galaxy/model/orm/{__init__,scripts,engine_factory,now,scripts}.py
 lib/galaxy/model/{base,custom_types,search,util}.py
-lib/galaxy/model/tool_shed_install/__init__.py
-lib/galaxy/objectstore/{__init__,pulsar,rods,s3}.py
-lib/galaxy/openid/__init__.py
-lib/galaxy/queues.py
-lib/galaxy/queue_worker.py
-lib/galaxy/tags
-lib/galaxy/tools
+lib/galaxy/model/migrate/check.py
+lib/galaxy/model/migrate/versions/{0001_initial_tables,0002_metadata_file_table,0003_security_and_libraries,0004_indexes_and_defaults,0005_cleanup_datasets_fix}.py
+lib/galaxy/model/tool_shed_install/{__init__,mapping}.py
+lib/galaxy/model/tool_shed_install/migrate/check.py
+lib/galaxy/objectstore/
+lib/galaxy/openid/
+lib/galaxy/quota/
+lib/galaxy/tags/
+lib/galaxy/tools/
 lib/galaxy/util/{__init__,json,permutations,plugin_config,properties,simplegraph,sockets,sleeper,streamball,submodules,xml_macros}.py
-lib/galaxy/visualization/data_providers/{__init__,genome,registry}.py
-lib/galaxy/web/{proxy,security}
-lib/galaxy/web/base/{__init__,interactive_environments}.py
-lib/galaxy/web/formatting.py
+lib/galaxy/visualization/
+lib/galaxy/web/
 lib/galaxy/webapps/__init__.py
 lib/galaxy/webapps/galaxy/__init__.py
 lib/galaxy/webapps/galaxy/api/{__init__,authenticate,dataset_collections,datatypes,folder_contents,folders,lda_datasets,job_files,libraries,library_contents,provenance,remote_files,search,users}.py
@@ -32,9 +36,8 @@ lib/galaxy/webapps/tool_shed/controllers/__init__.py
 lib/galaxy/webapps/tool_shed/framework/__init__.py
 lib/galaxy/webapps/tool_shed/framework/middleware/__init__.py
 lib/galaxy/webapps/tool_shed/util/{__init__,shed_statistics}.py
-lib/galaxy/work
-lib/galaxy/workflow
-lib/galaxy/version.py
+lib/galaxy/work/
+lib/galaxy/workflow/
 lib/pulsar
 lib/tool_shed/__init__.py
 lib/tool_shed/galaxy_install/{install_manager,repair_repository_manager}.py

lib/galaxy/__init__.py

@@ -4,13 +4,12 @@
 
 __import__( "pkg_resources" ).declare_namespace( __name__ )
 
-import re
-import os
 import sys
 import platform
 
 import pkg_resources
 
+
 # patch get_platform() for better ABI recognition
 def _get_build_platform():
     plat = pkg_resources._get_build_platform()
@@ -32,11 +31,11 @@ def _get_build_platform():
         if plat_split[-1] == 'universal' and platform.processor() != 'powerpc':
             plat_split[-1] = 'intel'
         plat = '-'.join( plat_split )
-    elif sys.platform == "linux2" and sys.maxint < 2**31 and plat.endswith( '-x86_64' ):
+    elif sys.platform == "linux2" and sys.maxint < 2 ** 31 and plat.endswith( '-x86_64' ):
         # 32 bit Python on 64 bit Linux
         plat = plat.replace( '-x86_64', '-i686' )
     if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')):
-        if sys.maxunicode > 2**16:
+        if sys.maxunicode > 2 ** 16:
             plat += '-ucs4'
         else:
             plat += '-ucs2'
@@ -48,16 +47,17 @@ def _get_build_platform():
     pkg_resources.get_build_platform = _get_build_platform
     pkg_resources.get_platform = _get_build_platform
 
+
 # patch to insert eggs at the beginning of sys.path instead of at the end
-def _insert_on(self, path, loc = None):
+def _insert_on(self, path, loc=None):
     """Insert self.location in path before its nearest parent directory"""
 
     loc = loc or self.location
     if not loc:
         return
 
     nloc = pkg_resources._normalize_cached(loc)
-    npath= [(p and pkg_resources._normalize_cached(p) or p) for p in path]
+    npath = [(p and pkg_resources._normalize_cached(p) or p) for p in path]
 
     if path is sys.path:
         self.check_version_conflict()
@@ -91,5 +91,3 @@ class NullHandler( logging.Handler ):
         def emit( self, record ):
             pass
     logging.NullHandler = NullHandler
-
-import galaxy.eggs

lib/galaxy/dataset_collections/subcollections.py

@@ -11,7 +11,7 @@ def _split_dataset_collection( dataset_collection, collection_type ):
     this_collection_type = dataset_collection.collection_type
     if not this_collection_type.endswith( collection_type ) or this_collection_type == collection_type:
         raise exceptions.MessageException( "Cannot split collection in desired fashion." )
-        
+
     split_elements = []
     for element in dataset_collection.elements:
         child_collection = element.child_collection

lib/galaxy/dataset_collections/types/__init__.py

@@ -2,8 +2,6 @@
 from abc import ABCMeta
 from abc import abstractmethod
 
-from galaxy import model
-
 import logging
 log = logging.getLogger( __name__ )
 

lib/galaxy/datatypes/assembly.py

@@ -13,7 +13,6 @@
 from galaxy.datatypes.images import Html
 from galaxy.datatypes.metadata import MetadataElement
 
-
 log = logging.getLogger(__name__)
 
 

lib/galaxy/datatypes/binary.py

@@ -323,13 +323,13 @@ def set_meta( self, dataset, overwrite=True, **kwd ):
         if exit_code == -6:
             # SIGABRT, most likely samtools 1.0+ which does not accept the index name parameter.
             dataset_symlink = os.path.join( os.path.dirname( index_file.file_name ),
-                    '__dataset_%d_%s' % ( dataset.id, os.path.basename( index_file.file_name ) ) )
+                                            '__dataset_%d_%s' % ( dataset.id, os.path.basename( index_file.file_name ) ) )
             os.symlink( dataset.file_name, dataset_symlink )
             try:
                 command = [ 'samtools', 'index', dataset_symlink ]
                 exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) )
                 shutil.move( dataset_symlink + '.bai', index_file.file_name )
-            except Exception, e:
+            except Exception as e:
                 open( stderr_name, 'ab+' ).write( 'Galaxy attempted to build the BAM index with samtools 1.0+ but failed: %s\n' % e)
             finally:
                 os.unlink( dataset_symlink )
@@ -494,7 +494,7 @@ def set_meta( self, dataset, overwrite=True, **kwd ):
         # Usage: bcftools index <in.bcf>
 
         dataset_symlink = os.path.join( os.path.dirname( index_file.file_name ),
-                    '__dataset_%d_%s' % ( dataset.id, os.path.basename( index_file.file_name ) ) )
+                                        '__dataset_%d_%s' % ( dataset.id, os.path.basename( index_file.file_name ) ) )
         os.symlink( dataset.file_name, dataset_symlink )
 
         stderr_name = tempfile.NamedTemporaryFile( prefix="bcf_index_stderr" ).name
@@ -711,18 +711,18 @@ def set_meta( self, dataset, overwrite=True, **kwd ):
                     cur = conn.cursor().execute(col_query)
                     cols = [col[0] for col in cur.description]
                     columns[table] = cols
-                except Exception, exc:
+                except Exception as exc:
                     log.warn( '%s, set_meta Exception: %s', self, exc )
             for table in tables:
                 try:
                     row_query = "SELECT count(*) FROM %s" % table
                     rowcounts[table] = c.execute(row_query).fetchone()[0]
-                except Exception, exc:
+                except Exception as exc:
                     log.warn( '%s, set_meta Exception: %s', self, exc )
             dataset.metadata.tables = tables
             dataset.metadata.table_columns = columns
             dataset.metadata.table_row_count = rowcounts
-        except Exception, exc:
+        except Exception as exc:
             log.warn( '%s, set_meta Exception: %s', self, exc )
 
     def sniff( self, filename ):
@@ -793,13 +793,13 @@ def set_meta( self, dataset, overwrite=True, **kwd ):
             for version, in result:
                 dataset.metadata.gemini_version = version
             # TODO: Can/should we detect even more attributes, such as use of PED file, what was input annotation type, etc.
-        except Exception, e:
+        except Exception as e:
             log.warn( '%s, set_meta Exception: %s', self, e )
 
     def sniff( self, filename ):
         if super( GeminiSQLite, self ).sniff( filename ):
             gemini_table_names = [ "gene_detailed", "gene_summary", "resources", "sample_genotype_counts", "sample_genotypes", "samples",
-                                  "variant_impacts", "variants", "version" ]
+                                   "variant_impacts", "variants", "version" ]
             try:
                 conn = sqlite.connect( filename )
                 c = conn.cursor()
@@ -810,7 +810,7 @@ def sniff( self, filename ):
                     if table_name not in result:
                         return False
                 return True
-            except Exception, e:
+            except Exception as e:
                 log.warn( '%s, sniff Exception: %s', self, e )
         return False
 

lib/galaxy/datatypes/converters/bed_to_gff_converter.py

@@ -4,6 +4,7 @@
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
+
 def __main__():
     input_name = sys.argv[1]
     output_name = sys.argv[2]
@@ -67,7 +68,7 @@ def __main__():
     out.close()
     info_msg = "%i lines converted to GFF version 2.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
-        info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." %( skipped_lines, first_skipped_line )
+        info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
     print info_msg
 
 if __name__ == "__main__": __main__()

lib/galaxy/datatypes/converters/bgzip.py

@@ -6,7 +6,6 @@
 usage: %prog in_file out_file
 """
 
-from galaxy import eggs
 import pkg_resources
 pkg_resources.require( "pysam" )
 import ctabix
@@ -29,11 +28,12 @@ def main():
     sort_params = None
 
     if options.chrom_col and options.start_col and options.end_col:
-        sort_params = ["sort",
-                    "-k%(i)s,%(i)s" % { 'i': options.chrom_col },
-                    "-k%(i)i,%(i)in" % { 'i': options.start_col },
-                    "-k%(i)i,%(i)in" % { 'i': options.end_col }
-                    ]
+        sort_params = [
+            "sort",
+            "-k%(i)s,%(i)s" % { 'i': options.chrom_col },
+            "-k%(i)i,%(i)in" % { 'i': options.start_col },
+            "-k%(i)i,%(i)in" % { 'i': options.end_col }
+        ]
     elif options.preset == "bed":
         sort_params = ["sort", "-k1,1", "-k2,2n", "-k3,3n"]
     elif options.preset == "vcf":

lib/galaxy/datatypes/converters/fasta_to_len.py

@@ -9,6 +9,7 @@
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
+
 def compute_fasta_length( fasta_file, out_file, keep_first_char, keep_first_word=False ):
 
     infile = fasta_file

lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py

@@ -15,6 +15,7 @@
 
 seq_hash = {}
 
+
 def __main__():
     infile = sys.argv[1]
     outfile = sys.argv[2]

lib/galaxy/datatypes/converters/fastq_to_fqtoc.py

@@ -24,8 +24,8 @@ def main():
     out_file = open(sys.argv[2], 'w')
 
     current_line = 0
-    sequences=1000000
-    lines_per_chunk = 4*sequences
+    sequences = 1000000
+    lines_per_chunk = 4 * sequences
     chunk_begin = 0
 
     in_file = open(input_fname)

lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py

@@ -19,18 +19,20 @@
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
+
 def stop_err( msg ):
     sys.stderr.write( "%s" % msg )
     sys.exit()
 
+
 def __main__():
     infile_name = sys.argv[1]
     outfile = open( sys.argv[2], 'w' )
     fastq_block_lines = 0
     seq_title_startswith = ''
 
     for i, line in enumerate( file( infile_name ) ):
-        line = line.rstrip() # eliminate trailing space and new line characters
+        line = line.rstrip()  # eliminate trailing space and new line characters
         if not line or line.startswith( '#' ):
             continue
         fastq_block_lines = ( fastq_block_lines + 1 ) % 4
@@ -40,7 +42,7 @@ def __main__():
             if not seq_title_startswith:
                 seq_title_startswith = line_startswith
             if seq_title_startswith != line_startswith:
-                stop_err( 'Invalid fastqsolexa format at line %d: %s.' %( i + 1, line ) )
+                stop_err( 'Invalid fastqsolexa format at line %d: %s.' % ( i + 1, line ) )
             read_title = line[ 1: ]
             outfile.write( '>%s\n' % line[1:] )
         elif fastq_block_lines == 2:

lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py

@@ -18,10 +18,12 @@
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
+
 def stop_err( msg ):
     sys.stderr.write( "%s" % msg )
     sys.exit()
 
+
 def __main__():
     infile_name = sys.argv[1]
     outfile_score = open( sys.argv[2], 'w' )
@@ -73,7 +75,7 @@ def __main__():
             except:
                 fastq_integer = False
 
-            if fastq_integer: # digits
+            if fastq_integer:  # digits
                 qual = line
             else:
                 # ascii

lib/galaxy/datatypes/converters/gff_to_bed_converter.py

@@ -3,6 +3,7 @@
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
+
 def __main__():
     input_name = sys.argv[1]
     output_name = sys.argv[2]
@@ -24,7 +25,7 @@ def __main__():
                 #
                 # Replace any spaces in the name with underscores so UCSC will not complain
                 name = elems[2].replace(" ", "_")
-                out.write( "%s\t%s\t%s\t%s\t0\t%s\n" %( elems[0], start, elems[4], name, strand ) )
+                out.write( "%s\t%s\t%s\t%s\t0\t%s\n" % ( elems[0], start, elems[4], name, strand ) )
             except:
                 skipped_lines += 1
                 if not first_skipped_line:
@@ -36,7 +37,7 @@ def __main__():
     out.close()
     info_msg = "%i lines converted to BED.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
-        info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." %( skipped_lines, first_skipped_line )
+        info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
     print info_msg
 
 if __name__ == "__main__": __main__()

lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py

@@ -15,6 +15,7 @@
 from galaxy.datatypes.util.gff_util import *
 from bx.interval_index_file import Indexes
 
+
 def main():
     # Arguments
     input_fname, out_fname = sys.argv[1:]