Merge pull request #12774 from nsoranzo/pyupgrade_fixes

New pyupgrade cleanups

contrib/galaxy_config_merger.py

@@ -69,7 +69,7 @@ def main():
                         config_sample.set(section, name, value)
                 else:
                     if not config_sample.get(section, name) == value:
-                        logging.info("- diff - section [{}] option '{}' has different value ('{}':'{}'). It will be modified.".format(section, name, config_sample.get(section, name), value))
+                        logging.info(f"- diff - section [{section}] option '{name}' has different value ('{config_sample.get(section, name)}':'{value}'). It will be modified.")
                         config_sample.set(section, name, value)
     logging.info("---------- DIFFERENCE ANALYSIS END   ----------")
 

lib/galaxy/config/__init__.py

@@ -975,7 +975,7 @@ def reload_sanitize_allowlist(self, explicit=True):
                 log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", self.sanitize_allowlist_file)
         else:
             with open(self.sanitize_allowlist_file) as f:
-                self.sanitize_allowlist = sorted([line.strip() for line in f.readlines() if not line.startswith('#')])
+                self.sanitize_allowlist = sorted(line.strip() for line in f.readlines() if not line.startswith('#'))
 
     def ensure_tempdir(self):
         self._ensure_directory(self.new_file_path)

lib/galaxy/containers/docker_model.py

@@ -44,7 +44,7 @@ def __ne__(self, other):
         return not self.__eq__(other)
 
     def __hash__(self):
-        return hash(tuple(sorted([repr(x) for x in self._members])))
+        return hash(tuple(sorted(repr(x) for x in self._members)))
 
     def __str__(self):
         return ', '.join(str(x) for x in self._members) or 'None'

lib/galaxy/datatypes/binary.py

@@ -1600,7 +1600,7 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None
 
         repr_ = self.get_repr(dataset.file_name)
 
-        return "<pre>{}</pre><pre>{}</pre>".format(repr_, rval)
+        return f"<pre>{repr_}</pre><pre>{rval}</pre>"
 
 
 class HexrdMaterials(H5):

lib/galaxy/datatypes/images.py

@@ -359,7 +359,7 @@ def set_peek(self, dataset, is_multi_byte=False):
                     "nobutton": "false",
                     "urlpause": "100",
                     "debug": "false",
-                    "posturl": "history_add_to?%s" % "&".join("{}={}".format(x[0], quote_plus(str(x[1]))) for x in [('copy_access_from', dataset.id), ('history_id', dataset.history_id), ('ext', 'maf'), ('name', f'GMAJ Output on data {dataset.hid}'), ('info', 'Added by GMAJ'), ('dbkey', dataset.dbkey)])
+                    "posturl": "history_add_to?%s" % "&".join(f"{x[0]}={quote_plus(str(x[1]))}" for x in [('copy_access_from', dataset.id), ('history_id', dataset.history_id), ('ext', 'maf'), ('name', f'GMAJ Output on data {dataset.hid}'), ('info', 'Added by GMAJ'), ('dbkey', dataset.dbkey)])
                 }
                 class_name = "edu.psu.bx.gmaj.MajApplet.class"
                 archive = "/static/gmaj/gmaj.jar"

lib/galaxy/datatypes/tabular.py

@@ -103,7 +103,7 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None
             else:
                 trans.response.set_content_type("text/html")
                 return trans.stream_template_mako("/dataset/large_file.mako",
-                                                  truncated_data=open(dataset.file_name, mode='r').read(max_peek_size),
+                                                  truncated_data=open(dataset.file_name).read(max_peek_size),
                                                   data=dataset)
         else:
             column_names = 'null'

lib/galaxy/files/__init__.py

@@ -250,13 +250,13 @@ def preferences(self):
     def role_names(self) -> Set[str]:
         """The set of role names of this user."""
         user = self.trans.user
-        return set(ura.role.name for ura in user.roles) if user else set()
+        return {ura.role.name for ura in user.roles} if user else set()
 
     @property
     def group_names(self) -> Set[str]:
         """The set of group names to which this user belongs."""
         user = self.trans.user
-        return set(ugr.group.name for ugr in user.groups) if user else set()
+        return {ugr.group.name for ugr in user.groups} if user else set()
 
     @property
     def is_admin(self):

lib/galaxy/jobs/__init__.py

@@ -539,12 +539,12 @@ def _configure_from_dict(self, job_config_dict):
                 self.limits.__dict__[limit_type] = types.get(limit_type, str)(limit_value)
 
         if self.limits.walltime is not None:
-            h, m, s = [int(v) for v in self.limits.walltime.split(':')]
+            h, m, s = (int(v) for v in self.limits.walltime.split(':'))
             self.limits.walltime_delta = datetime.timedelta(0, s, 0, 0, m, h)
 
         if "raw" in self.limits.total_walltime:
-            h, m, s = [int(v) for v in
-                       self.limits.total_walltime["raw"].split(':')]
+            h, m, s = (int(v) for v in
+                       self.limits.total_walltime["raw"].split(':'))
             self.limits.total_walltime["delta"] = datetime.timedelta(
                 0, s, 0, 0, m, h
             )

lib/galaxy/jobs/runners/kubernetes.py

@@ -110,7 +110,7 @@ def setup_volumes(self):
             volume_claims = dict(volume.split(":") for volume in self.runner_params['k8s_persistent_volume_claims'].split(','))
         else:
             volume_claims = {}
-        mountable_volumes = list(set([claim_name if "/" not in claim_name else claim_name.split("/")[0] for claim_name in volume_claims]))
+        mountable_volumes = list({claim_name if "/" not in claim_name else claim_name.split("/")[0] for claim_name in volume_claims})
         mountable_volumes = [{'name': claim_name, 'persistentVolumeClaim': {'claimName': claim_name}} for claim_name in mountable_volumes]
         self.runner_params['k8s_mountable_volumes'] = mountable_volumes
         volume_mounts = [{'name': claim_name, 'mountPath': mount_path} for claim_name, mount_path in volume_claims.items()]
@@ -366,7 +366,7 @@ def __get_k8s_service_spec(self, ajs):
                 }
             },
             "spec": {
-                "ports": [{"name": "job-{}-{}".format(self.__force_label_conformity(ajs.job_wrapper.get_id_tag()), p),
+                "ports": [{"name": f"job-{self.__force_label_conformity(ajs.job_wrapper.get_id_tag())}-{p}",
                            "port": int(p),
                            "protocol": "TCP",
                            "targetPort": int(p)} for p in guest_ports],
@@ -426,7 +426,7 @@ def __get_k8s_ingress_spec(self, ajs):
             }
         }
         if self.runner_params.get("k8s_interactivetools_use_ssl"):
-            domains = list(set([e["domain"] for e in entry_points]))
+            domains = list({e["domain"] for e in entry_points})
             k8s_spec_template["spec"]["tls"] = [{"hosts": [domain],
                                                  "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains]
         if self.runner_params.get("k8s_interactivetools_ingress_annotations"):
@@ -809,7 +809,7 @@ def __job_pending_due_to_unschedulable_pod(self, job_state):
 
     def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job):
         k8s_job_prefix = self.__produce_k8s_job_prefix()
-        k8s_job_name = "{}-{}".format(k8s_job_prefix, self.__force_label_conformity(job_wrapper.get_id_tag()))
+        k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}"
         log.debug(f'Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}')
         job_failed = (k8s_job.obj['status']['failed'] > 0
                       if 'failed' in k8s_job.obj['status'] else False)

lib/galaxy/jobs/runners/pbs.py

@@ -385,7 +385,7 @@ def check_watched_items(self):
                 pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING)
             if status.job_state == "R" and status.get('resources_used', False):
                 # resources_used may not be in the status for new jobs
-                h, m, s = [int(i) for i in status.resources_used.walltime.split(':')]
+                h, m, s = (int(i) for i in status.resources_used.walltime.split(':'))
                 runtime = timedelta(0, s, 0, 0, m, h)
                 if pbs_job_state.check_limits(runtime=runtime):
                     self.work_queue.put((self.fail_job, pbs_job_state))

lib/galaxy/jobs/runners/pulsar.py

@@ -376,7 +376,7 @@ def queue_job(self, job_wrapper):
             remote_pulsar_app_config = dest_params.get("pulsar_app_config", {}).copy()
             if "pulsar_app_config_path" in dest_params:
                 pulsar_app_config_path = dest_params["pulsar_app_config_path"]
-                with open(pulsar_app_config_path, "r") as fh:
+                with open(pulsar_app_config_path) as fh:
                     remote_pulsar_app_config.update(yaml.safe_load(fh))
             job_directory_files = []
             config_files = job_wrapper.extra_filenames

lib/galaxy/managers/collections.py

@@ -305,7 +305,7 @@ def delete(self, trans, instance_type, id, recursive=False, purge=False):
                 try:
                     self.hda_manager.error_unless_owner(dataset, user=trans.get_user(), current_history=trans.history)
                 except HistoryDatasetAssociationNoHistoryException:
-                    log.info("Cannot delete HistoryDatasetAssociation {}, HistoryDatasetAssociation has no associated History, cannot verify owner".format(dataset.id))
+                    log.info(f"Cannot delete HistoryDatasetAssociation {dataset.id}, HistoryDatasetAssociation has no associated History, cannot verify owner")
                     continue
                 if not dataset.deleted:
                     dataset.deleted = True

lib/galaxy/managers/collections_util.py

@@ -262,7 +262,7 @@ def gen_rank_fuzzy_counts(collection_type, fuzzy_count=None):
         return [None for rt in rank_collection_types]
     else:
         # This is a list...
-        paired_count = sum([1 if rt == "paired" else 0 for rt in rank_collection_types])
+        paired_count = sum(1 if rt == "paired" else 0 for rt in rank_collection_types)
         list_count = len(rank_collection_types) - paired_count
         paired_fuzzy_count_mult = 1 if paired_count == 0 else 2 << (paired_count - 1)
         list_fuzzy_count_mult = math.floor((fuzzy_count * 1.0) / paired_fuzzy_count_mult)

lib/galaxy/managers/genomes.py

@@ -57,7 +57,7 @@ def get_indexes(self, id: str, index_type: str) -> Any:
         ext = index_extensions[index_type]
         index_filename = self._get_index_filename(id, tbl_entries, ext, index_type)
         try:
-            with open(index_filename, mode='r') as f:
+            with open(index_filename) as f:
                 return f.read()
         except OSError:
             raise ReferenceDataError(f'Failed to load index file for {id}')

lib/galaxy/managers/remote_files.py

@@ -1,4 +1,3 @@
-
 import hashlib
 import logging
 from operator import itemgetter

lib/galaxy/managers/tool_data.py

@@ -1,4 +1,3 @@
-
 from pathlib import Path
 from typing import (
     Dict,

lib/galaxy/model/__init__.py

@@ -2488,7 +2488,7 @@ def add_datasets(self, sa_session, datasets, parent_id=None, genome_build=None,
         if optimize:
             self.__add_datasets_optimized(datasets, genome_build=genome_build)
             if quota and self.user:
-                disk_usage = sum([d.get_total_size() for d in datasets if is_hda(d)])
+                disk_usage = sum(d.get_total_size() for d in datasets if is_hda(d))
                 self.user.adjust_total_disk_usage(disk_usage)
             sa_session.add_all(datasets)
             if flush:
@@ -3305,7 +3305,7 @@ def set_total_size(self):
         if rel_path is not None:
             if self.object_store.exists(self, extra_dir=rel_path, dir_only=True):
                 for root, _, files in os.walk(self.extra_files_path):
-                    self.total_size += sum([os.path.getsize(os.path.join(root, file)) for file in files if os.path.exists(os.path.join(root, file))])
+                    self.total_size += sum(os.path.getsize(os.path.join(root, file)) for file in files if os.path.exists(os.path.join(root, file)))
         return self.total_size
 
     def has_data(self):

lib/galaxy/model/dataset_collections/structure.py

@@ -119,7 +119,7 @@ def can_match(self, other_structure):
         return True
 
     def __len__(self):
-        return sum([len(c[1]) for c in self.children])
+        return sum(len(c[1]) for c in self.children)
 
     def multiply(self, other_structure):
         if other_structure.is_leaf:

lib/galaxy/model/migrate/versions/0173_add_job_id_to_dataset.py

@@ -1,7 +1,6 @@
 """
 Migration script for adding job_id column to dataset table.
 """
-from __future__ import print_function
 
 import logging
 

lib/galaxy/selenium/context.py

@@ -64,7 +64,7 @@ def _screenshot_path(self, label, extension=".png"):
 
 def init(config=None, clazz=GalaxySeleniumContextImpl) -> GalaxySeleniumContext:
     if os.path.exists("galaxy_selenium_context.yml"):
-        with open("galaxy_selenium_context.yml", "r") as f:
+        with open("galaxy_selenium_context.yml") as f:
             as_dict = yaml.safe_load(f)
         context = clazz(as_dict)
     else:

lib/galaxy/tool_util/biotools/source.py

@@ -26,7 +26,7 @@ def get_biotools_metadata(self, biotools_reference: str) -> Optional[BiotoolsEnt
         path = os.path.join(self._content_directory, "data", biotools_reference, f"{biotools_reference}.biotools.json")
         if not os.path.exists(path):
             return None
-        with open(path, "r") as f:
+        with open(path) as f:
             content_json = json.load(f)
             return BiotoolsEntry.from_json(content_json)
 

lib/galaxy/tool_util/deps/container_resolvers/mulled.py

@@ -88,7 +88,7 @@ def _list_cached_mulled_images_from_path(self):
         contents = os.listdir(self.path)
         sorted_images = version_sorted(contents)
         raw_images = map(lambda name: identifier_to_cached_target(name, self.hash_func), sorted_images)
-        return list([i for i in raw_images if i is not None])
+        return list(i for i in raw_images if i is not None)
 
     @abstractmethod
     def list_cached_mulled_images_from_path(self):

lib/galaxy/tool_util/deps/mulled/mulled_search.py

@@ -217,12 +217,12 @@ def singularity_search(search_string):
 def readable_output(json, organization='biocontainers', channel='bioconda'):
 
     # if json is empty:
-    if sum([len(json[destination][results]) for destination in json for results in json[destination]]) == 0:
+    if sum(len(json[destination][results]) for destination in json for results in json[destination]) == 0:
         sys.stdout.write('No results found for that query.\n')
         return
 
     # return results for quay, conda and singularity together
-    if sum([len(json[destination][results]) for destination in ['quay', 'conda', 'singularity', ] for results in json.get(destination, [])]) > 0:
+    if sum(len(json[destination][results]) for destination in ['quay', 'conda', 'singularity', ] for results in json.get(destination, [])) > 0:
         sys.stdout.write("The query returned the following result(s).\n")
         # put quay, conda etc results as lists in lines
         lines = [['LOCATION', 'NAME', 'VERSION', 'COMMAND\n']]
@@ -262,7 +262,7 @@ def readable_output(json, organization='biocontainers', channel='bioconda'):
             sys.stdout.write(
                 "".join((line[0].ljust(col_width0), line[1])))  # output
 
-    if sum([len(json['github'][results]) for results in json.get('github', [])]) > 0:
+    if sum(len(json['github'][results]) for results in json.get('github', [])) > 0:
         sys.stdout.write('\n' if 'lines' in locals() else '')
         sys.stdout.write(
             "Other result(s) on the bioconda-recipes GitHub repository:\n")

lib/galaxy/tool_util/deps/requirements.py

@@ -140,7 +140,7 @@ def __len__(self):
         return len(self.tool_requirements)
 
     def __hash__(self):
-        return sum([r.__hash__() for r in self.tool_requirements])
+        return sum(r.__hash__() for r in self.tool_requirements)
 
     def to_dict(self):
         return [r.to_dict() for r in self.tool_requirements]

lib/galaxy/tool_util/edam_util.py

@@ -1,4 +1,3 @@
-import io
 import os
 from typing import Dict, Optional, TextIO
 
@@ -16,7 +15,7 @@
 def load_edam_tree(path: Optional[str] = None):
     if path is not None:
         assert os.path.exists(path), f"Failed to load EDAM tabular data at [{path}] path does not exist."
-        handle = io.open(path, "r")
+        handle = open(path)
     else:
         assert tabular_stream is not None, "Failed to load optional import from edam-onotology package, install using [pip install edam-ontology]."
         handle = tabular_stream()

lib/galaxy/tool_util/linters/inputs.py

@@ -1,4 +1,3 @@
-
 """This module contains a linting functions for tool inputs."""
 from galaxy.util import string_as_bool
 from ._util import is_datasource, is_valid_cheetah_placeholder
@@ -132,9 +131,9 @@ def lint_inputs(tool_xml, lint_ctx):
             # lint statically defined options
             if any(['value' not in option.attrib for option in select_options]):
                 lint_ctx.error(f"Select parameter [{param_name}] has option without value")
-            if len(set([option.text.strip() for option in select_options if option.text is not None])) != len(select_options):
+            if len({option.text.strip() for option in select_options if option.text is not None}) != len(select_options):
                 lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same text content")
-            if len(set([option.attrib.get("value") for option in select_options])) != len(select_options):
+            if len({option.attrib.get("value") for option in select_options}) != len(select_options):
                 lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same value")
 
             if param_attrib.get("display") == "checkboxes":

lib/galaxy/tool_util/toolbox/views/edam.py

@@ -142,8 +142,7 @@ def _get_edam_sec(self, tool):
         if self.include_topics:
             edam.extend(tool.edam_topics)
         if len(edam) > 0:
-            for term in edam:
-                yield term
+            yield from edam
         else:
             yield 'uncategorized'
 

lib/galaxy/util/pastescript/serve.py

@@ -369,7 +369,7 @@ def run(self, args):
             print('(try running python setup.py develop)')
             return 2
         print('Known commands:')
-        longest = max([len(n) for n, c in commands])
+        longest = max(len(n) for n, c in commands)
         for name, command in commands:
             print(f'  {self.pad(name, length=longest)}  {command.load().summary}')
         return 2

lib/galaxy/visualization/data_providers/genome.py

@@ -987,7 +987,7 @@ def _nth_read_iterator(read_iterator, threshold):
             seq = read.seq
             strand = decode_strand(read.flag, 0x0010)
             if read.cigar is not None:
-                read_len = sum([cig[1] for cig in read.cigar])  # Use cigar to determine length
+                read_len = sum(cig[1] for cig in read.cigar)  # Use cigar to determine length
             else:
                 read_len = len(seq)  # If no cigar, just use sequence length
 

lib/galaxy/web/framework/middleware/error.py

@@ -308,10 +308,11 @@ def extraData(self):
                 wsgi_vars[name] = value
         if self.environ['wsgi.version'] != (1, 0):
             wsgi_vars['wsgi.version'] = self.environ['wsgi.version']
-        proc_desc = tuple([int(bool(self.environ[key]))
-                           for key in ('wsgi.multiprocess',
-                                       'wsgi.multithread',
-                                       'wsgi.run_once')])
+        proc_desc = tuple(int(bool(self.environ[key])) for key in (
+            'wsgi.multiprocess',
+            'wsgi.multithread',
+            'wsgi.run_once'
+        ))
         wsgi_vars['wsgi process'] = self.process_combos[proc_desc]
         wsgi_vars['application'] = self.middleware.application
         if 'paste.config' in self.environ:

lib/galaxy/webapps/base/controller.py

@@ -1471,7 +1471,7 @@ def _scan_json_block(self, meta, prefix=""):
         else:
             # BUG: Everything is cast to string, which can lead to false positives
             # for cross type comparisions, ie "True" == True
-            yield prefix, (f"{meta}").encode("utf8", errors='replace')
+            yield prefix, (f"{meta}").encode()
 
 
 def sort_by_attr(seq, attr):

lib/galaxy/webapps/galaxy/api/library_contents.py

@@ -375,7 +375,7 @@ def _scan_json_block(self, meta, prefix=""):
         else:
             # BUG: Everything is cast to string, which can lead to false positives
             # for cross type comparisions, ie "True" == True
-            yield prefix, (f"{meta}").encode("utf8", errors='replace')
+            yield prefix, (f"{meta}").encode()
 
     @expose_api
     def update(self, trans, id, library_id, payload, **kwd):