From 9b242375f817a06b75086ec17aeab3e5098302d0 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 16 Jul 2020 10:50:12 -0700 Subject: [PATCH] add pyupgrade to pre-commit and run --- .pre-commit-config.yaml | 7 +++++++ gym-unity/setup.py | 2 +- ml-agents-envs/mlagents_envs/communicator.py | 2 +- .../unity_to_external_pb2_grpc.py | 4 ++-- ml-agents-envs/mlagents_envs/environment.py | 17 ++++++++--------- ml-agents-envs/mlagents_envs/exception.py | 2 +- .../mlagents_envs/rpc_communicator.py | 2 +- .../environment_parameters_channel.py | 2 +- .../side_channel/float_properties_channel.py | 2 +- .../side_channel/side_channel_manager.py | 2 +- .../mlagents_envs/tests/test_side_channel.py | 4 ++-- ml-agents-envs/setup.py | 2 +- ml-agents/mlagents/trainers/buffer.py | 4 ++-- ml-agents/mlagents/trainers/cli_utils.py | 2 +- .../mlagents/trainers/components/bc/model.py | 2 +- .../reward_signals/curiosity/model.py | 2 +- .../components/reward_signals/gail/model.py | 2 +- .../reward_signals/reward_signal_factory.py | 2 +- ml-agents/mlagents/trainers/ghost/trainer.py | 2 +- ml-agents/mlagents/trainers/policy/tf_policy.py | 4 ++-- ml-agents/mlagents/trainers/ppo/trainer.py | 2 +- ml-agents/mlagents/trainers/stats.py | 6 +++--- .../trainers/tests/test_training_status.py | 2 +- .../mlagents/trainers/trainer/rl_trainer.py | 2 +- .../mlagents/trainers/trainer_controller.py | 2 +- ml-agents/mlagents/trainers/training_status.py | 2 +- ml-agents/setup.py | 3 +-- .../tests/yamato/check_coverage_percent.py | 1 - utils/validate_release_links.py | 2 +- utils/validate_versions.py | 4 ++-- 30 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6be882c18..05c0614310 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,6 +39,13 @@ repos: # flake8-tidy-imports is used for banned-modules, not actually tidying additional_dependencies: [flake8-comprehensions==3.2.2, flake8-tidy-imports==4.1.0, flake8-bugbear==20.1.4] +- repo: https://github.com/asottile/pyupgrade + rev: v2.7.0 + hooks: + - id: pyupgrade + args: [--py3-plus] + exclude: .*barracuda.py + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.5.0 hooks: diff --git a/gym-unity/setup.py b/gym-unity/setup.py index a68509f093..4f6975e88b 100755 --- a/gym-unity/setup.py +++ b/gym-unity/setup.py @@ -23,7 +23,7 @@ def run(self): tag = os.getenv("CIRCLE_TAG") if tag != EXPECTED_TAG: - info = "Git tag: {0} does not match the expected tag of this app: {1}".format( + info = "Git tag: {} does not match the expected tag of this app: {}".format( tag, EXPECTED_TAG ) sys.exit(info) diff --git a/ml-agents-envs/mlagents_envs/communicator.py b/ml-agents-envs/mlagents_envs/communicator.py index 0982d69117..af4a29a781 100644 --- a/ml-agents-envs/mlagents_envs/communicator.py +++ b/ml-agents-envs/mlagents_envs/communicator.py @@ -3,7 +3,7 @@ from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto -class Communicator(object): +class Communicator: def __init__(self, worker_id=0, base_port=5005): """ Python side of the communication. Must be used in pair with the right Unity Communicator equivalent. diff --git a/ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py b/ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py index bedeb10059..9ad433a210 100644 --- a/ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py +++ b/ml-agents-envs/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py @@ -4,7 +4,7 @@ from mlagents_envs.communicator_objects import unity_message_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__message__pb2 -class UnityToExternalProtoStub(object): +class UnityToExternalProtoStub: # missing associated documentation comment in .proto file pass @@ -21,7 +21,7 @@ def __init__(self, channel): ) -class UnityToExternalProtoServicer(object): +class UnityToExternalProtoServicer: # missing associated documentation comment in .proto file pass diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 777a4b5f9c..ac69398d50 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -326,8 +326,8 @@ def behavior_specs(self) -> MappingType[str, BehaviorSpec]: def _assert_behavior_exists(self, behavior_name: str) -> None: if behavior_name not in self._env_specs: raise UnityActionException( - "The group {0} does not correspond to an existing agent group " - "in the environment".format(behavior_name) + f"The group {behavior_name} does not correspond to an existing " + f"agent group in the environment" ) def set_actions(self, behavior_name: BehaviorName, action: np.ndarray) -> None: @@ -339,9 +339,9 @@ def set_actions(self, behavior_name: BehaviorName, action: np.ndarray) -> None: expected_shape = (len(self._env_state[behavior_name][0]), spec.action_size) if action.shape != expected_shape: raise UnityActionException( - "The behavior {0} needs an input of dimension {1} for " - "(, ) but received input of " - "dimension {2}".format(behavior_name, expected_shape, action.shape) + f"The behavior {behavior_name} needs an input of dimension " + f"{expected_shape} for (, ) but " + f"received input of dimension {action.shape}" ) if action.dtype != expected_type: action = action.astype(expected_type) @@ -357,10 +357,9 @@ def set_action_for_agent( expected_shape = (spec.action_size,) if action.shape != expected_shape: raise UnityActionException( - f"The Agent {0} with BehaviorName {1} needs an input of dimension " - f"{2} but received input of dimension {3}".format( - agent_id, behavior_name, expected_shape, action.shape - ) + f"The Agent {agent_id} with BehaviorName {behavior_name} needs " + f"an input of dimension {expected_shape} but received input of " + f"dimension {action.shape}" ) expected_type = np.float32 if spec.is_action_continuous() else np.int32 if action.dtype != expected_type: diff --git a/ml-agents-envs/mlagents_envs/exception.py b/ml-agents-envs/mlagents_envs/exception.py index dd78cff175..13a5dd3559 100644 --- a/ml-agents-envs/mlagents_envs/exception.py +++ b/ml-agents-envs/mlagents_envs/exception.py @@ -75,4 +75,4 @@ class UnityWorkerInUseException(UnityException): def __init__(self, worker_id): message = self.MESSAGE_TEMPLATE.format(str(worker_id)) - super(UnityWorkerInUseException, self).__init__(message) + super().__init__(message) diff --git a/ml-agents-envs/mlagents_envs/rpc_communicator.py b/ml-agents-envs/mlagents_envs/rpc_communicator.py index ae118fe30d..20ff3cc9a3 100644 --- a/ml-agents-envs/mlagents_envs/rpc_communicator.py +++ b/ml-agents-envs/mlagents_envs/rpc_communicator.py @@ -81,7 +81,7 @@ def check_port(self, port): s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: s.bind(("localhost", port)) - except socket.error: + except OSError: raise UnityWorkerInUseException(self.worker_id) finally: s.close() diff --git a/ml-agents-envs/mlagents_envs/side_channel/environment_parameters_channel.py b/ml-agents-envs/mlagents_envs/side_channel/environment_parameters_channel.py index 2d379cbb3f..9629f1b5d5 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/environment_parameters_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/environment_parameters_channel.py @@ -22,7 +22,7 @@ class SamplerTypes(IntEnum): MULTIRANGEUNIFORM = 2 def __init__(self) -> None: - channel_id = uuid.UUID(("534c891e-810f-11ea-a9d0-822485860400")) + channel_id = uuid.UUID("534c891e-810f-11ea-a9d0-822485860400") super().__init__(channel_id) def on_message_received(self, msg: IncomingMessage) -> None: diff --git a/ml-agents-envs/mlagents_envs/side_channel/float_properties_channel.py b/ml-agents-envs/mlagents_envs/side_channel/float_properties_channel.py index 37dc6f0745..6c6701129f 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/float_properties_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/float_properties_channel.py @@ -13,7 +13,7 @@ class FloatPropertiesChannel(SideChannel): def __init__(self, channel_id: uuid.UUID = None) -> None: self._float_properties: Dict[str, float] = {} if channel_id is None: - channel_id = uuid.UUID(("60ccf7d0-4f7e-11ea-b238-784f4387d1f7")) + channel_id = uuid.UUID("60ccf7d0-4f7e-11ea-b238-784f4387d1f7") super().__init__(channel_id) def on_message_received(self, msg: IncomingMessage) -> None: diff --git a/ml-agents-envs/mlagents_envs/side_channel/side_channel_manager.py b/ml-agents-envs/mlagents_envs/side_channel/side_channel_manager.py index 5534cc20d2..a7783c64f5 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/side_channel_manager.py +++ b/ml-agents-envs/mlagents_envs/side_channel/side_channel_manager.py @@ -33,7 +33,7 @@ def process_side_channel_message(self, data: bytes) -> None: ) if len(message_data) != message_len: raise UnityEnvironmentException( - "The message received by the side channel {0} was " + "The message received by the side channel {} was " "unexpectedly short. Make sure your Unity Environment " "sending side channel data properly.".format(channel_id) ) diff --git a/ml-agents-envs/mlagents_envs/tests/test_side_channel.py b/ml-agents-envs/mlagents_envs/tests/test_side_channel.py index 1667bb44cb..8a4ef7a301 100644 --- a/ml-agents-envs/mlagents_envs/tests/test_side_channel.py +++ b/ml-agents-envs/mlagents_envs/tests/test_side_channel.py @@ -84,8 +84,8 @@ def test_raw_bytes(): sender = RawBytesChannel(guid) receiver = RawBytesChannel(guid) - sender.send_raw_data("foo".encode("ascii")) - sender.send_raw_data("bar".encode("ascii")) + sender.send_raw_data(b"foo") + sender.send_raw_data(b"bar") data = SideChannelManager([sender]).generate_side_channel_messages() SideChannelManager([receiver]).process_side_channel_message(data) diff --git a/ml-agents-envs/setup.py b/ml-agents-envs/setup.py index eb8e8af0d2..96ef768b78 100644 --- a/ml-agents-envs/setup.py +++ b/ml-agents-envs/setup.py @@ -23,7 +23,7 @@ def run(self): tag = os.getenv("CIRCLE_TAG") if tag != EXPECTED_TAG: - info = "Git tag: {0} does not match the expected tag of this app: {1}".format( + info = "Git tag: {} does not match the expected tag of this app: {}".format( tag, EXPECTED_TAG ) sys.exit(info) diff --git a/ml-agents/mlagents/trainers/buffer.py b/ml-agents/mlagents/trainers/buffer.py index b97723282a..2d24ec9665 100644 --- a/ml-agents/mlagents/trainers/buffer.py +++ b/ml-agents/mlagents/trainers/buffer.py @@ -134,7 +134,7 @@ def __init__(self): super().__init__() def __str__(self): - return ", ".join(["'{0}' : {1}".format(k, str(self[k])) for k in self.keys()]) + return ", ".join(["'{}' : {}".format(k, str(self[k])) for k in self.keys()]) def reset_agent(self) -> None: """ @@ -275,7 +275,7 @@ def resequence_and_append( key_list = list(self.keys()) if not self.check_length(key_list): raise BufferException( - "The length of the fields {0} were not of same length".format(key_list) + "The length of the fields {} were not of same length".format(key_list) ) for field_key in key_list: target_buffer[field_key].extend( diff --git a/ml-agents/mlagents/trainers/cli_utils.py b/ml-agents/mlagents/trainers/cli_utils.py index 44624e6657..41f2c3d87a 100644 --- a/ml-agents/mlagents/trainers/cli_utils.py +++ b/ml-agents/mlagents/trainers/cli_utils.py @@ -232,7 +232,7 @@ def load_config(config_path: str) -> Dict[str, Any]: try: with open(config_path) as data_file: return _load_config(data_file) - except IOError: + except OSError: abs_path = os.path.abspath(config_path) raise TrainerConfigError(f"Config file could not be found at {abs_path}.") except UnicodeDecodeError: diff --git a/ml-agents/mlagents/trainers/components/bc/model.py b/ml-agents/mlagents/trainers/components/bc/model.py index caef0611a0..07de10ca4a 100644 --- a/ml-agents/mlagents/trainers/components/bc/model.py +++ b/ml-agents/mlagents/trainers/components/bc/model.py @@ -3,7 +3,7 @@ from mlagents.trainers.policy.tf_policy import TFPolicy -class BCModel(object): +class BCModel: def __init__( self, policy: TFPolicy, learning_rate: float = 3e-4, anneal_steps: int = 0 ): diff --git a/ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py b/ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py index 066d9dcb88..decff58043 100644 --- a/ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py +++ b/ml-agents/mlagents/trainers/components/reward_signals/curiosity/model.py @@ -5,7 +5,7 @@ from mlagents.trainers.policy.tf_policy import TFPolicy -class CuriosityModel(object): +class CuriosityModel: def __init__( self, policy: TFPolicy, encoding_size: int = 128, learning_rate: float = 3e-4 ): diff --git a/ml-agents/mlagents/trainers/components/reward_signals/gail/model.py b/ml-agents/mlagents/trainers/components/reward_signals/gail/model.py index 529212ef1e..8e5554499a 100644 --- a/ml-agents/mlagents/trainers/components/reward_signals/gail/model.py +++ b/ml-agents/mlagents/trainers/components/reward_signals/gail/model.py @@ -8,7 +8,7 @@ EPSILON = 1e-7 -class GAILModel(object): +class GAILModel: def __init__( self, policy: TFPolicy, diff --git a/ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py b/ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py index c3d79c5387..f8ddc6ab4a 100644 --- a/ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py +++ b/ml-agents/mlagents/trainers/components/reward_signals/reward_signal_factory.py @@ -31,7 +31,7 @@ def create_reward_signal( """ rcls = NAME_TO_CLASS.get(name) if not rcls: - raise UnityTrainerException("Unknown reward signal type {0}".format(name)) + raise UnityTrainerException("Unknown reward signal type {}".format(name)) class_inst = rcls(policy, settings) return class_inst diff --git a/ml-agents/mlagents/trainers/ghost/trainer.py b/ml-agents/mlagents/trainers/ghost/trainer.py index d3047e193e..735926f2e5 100644 --- a/ml-agents/mlagents/trainers/ghost/trainer.py +++ b/ml-agents/mlagents/trainers/ghost/trainer.py @@ -58,7 +58,7 @@ def __init__( :param artifact_path: Path to store artifacts from this trainer. """ - super(GhostTrainer, self).__init__( + super().__init__( brain_name, trainer_settings, training, artifact_path, reward_buff_cap ) diff --git a/ml-agents/mlagents/trainers/policy/tf_policy.py b/ml-agents/mlagents/trainers/policy/tf_policy.py index d4ab657067..cd77d073aa 100644 --- a/ml-agents/mlagents/trainers/policy/tf_policy.py +++ b/ml-agents/mlagents/trainers/policy/tf_policy.py @@ -158,7 +158,7 @@ def _load_graph(self, model_path: str, reset_global_steps: bool = False) -> None ckpt = tf.train.get_checkpoint_state(model_path) if ckpt is None: raise UnityPolicyException( - "The model {0} could not be loaded. Make " + "The model {} could not be loaded. Make " "sure you specified the right " "--run-id and that the previous run you are loading from had the same " "behavior names.".format(model_path) @@ -167,7 +167,7 @@ def _load_graph(self, model_path: str, reset_global_steps: bool = False) -> None self.saver.restore(self.sess, ckpt.model_checkpoint_path) except tf.errors.NotFoundError: raise UnityPolicyException( - "The model {0} was found but could not be loaded. Make " + "The model {} was found but could not be loaded. Make " "sure the model is from the same version of ML-Agents, has the same behavior parameters, " "and is using the same trainer configuration as the current run.".format( model_path diff --git a/ml-agents/mlagents/trainers/ppo/trainer.py b/ml-agents/mlagents/trainers/ppo/trainer.py index 0822a683ea..6ac85103ac 100644 --- a/ml-agents/mlagents/trainers/ppo/trainer.py +++ b/ml-agents/mlagents/trainers/ppo/trainer.py @@ -44,7 +44,7 @@ def __init__( :param seed: The seed the model will be initialized with :param artifact_path: The directory within which to store artifacts from this trainer. """ - super(PPOTrainer, self).__init__( + super().__init__( brain_name, trainer_settings, training, artifact_path, reward_buff_cap ) self.hyperparameters: PPOSettings = cast( diff --git a/ml-agents/mlagents/trainers/stats.py b/ml-agents/mlagents/trainers/stats.py index c8a0a394b6..bd568c4f6c 100644 --- a/ml-agents/mlagents/trainers/stats.py +++ b/ml-agents/mlagents/trainers/stats.py @@ -127,7 +127,7 @@ def add_property( ) -> None: if property_type == StatsPropertyType.HYPERPARAMETERS: logger.info( - """Hyperparameters for behavior name {0}: \n{1}""".format( + """Hyperparameters for behavior name {}: \n{}""".format( category, self._dict_to_str(value, 0) ) ) @@ -150,7 +150,7 @@ def _dict_to_str(self, param_dict: Dict[str, Any], num_tabs: int) -> str: [ "\t" + " " * num_tabs - + "{0}:\t{1}".format( + + "{}:\t{}".format( x, self._dict_to_str(param_dict[x], num_tabs + 1) ) for x in param_dict @@ -226,7 +226,7 @@ def _dict_to_tensorboard(self, name: str, input_dict: Dict[str, Any]) -> str: s_op = tf.summary.text( name, tf.convert_to_tensor( - ([[str(x), str(input_dict[x])] for x in input_dict]) + [[str(x), str(input_dict[x])] for x in input_dict] ), ) s = sess.run(s_op) diff --git a/ml-agents/mlagents/trainers/tests/test_training_status.py b/ml-agents/mlagents/trainers/tests/test_training_status.py index d166bea5e3..d1fae24a8f 100644 --- a/ml-agents/mlagents/trainers/tests/test_training_status.py +++ b/ml-agents/mlagents/trainers/tests/test_training_status.py @@ -20,7 +20,7 @@ def test_globaltrainingstatus(tmpdir): GlobalTrainingStatus.set_parameter_state("Category1", StatusType.LESSON_NUM, 3) GlobalTrainingStatus.save_state(path_dir) - with open(path_dir, "r") as fp: + with open(path_dir) as fp: test_json = json.load(fp) assert "Category1" in test_json diff --git a/ml-agents/mlagents/trainers/trainer/rl_trainer.py b/ml-agents/mlagents/trainers/trainer/rl_trainer.py index d3ff17109a..a55d93b8c9 100644 --- a/ml-agents/mlagents/trainers/trainer/rl_trainer.py +++ b/ml-agents/mlagents/trainers/trainer/rl_trainer.py @@ -32,7 +32,7 @@ class RLTrainer(Trainer): # pylint: disable=abstract-method """ def __init__(self, *args, **kwargs): - super(RLTrainer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # collected_rewards is a dictionary from name of reward signal to a dictionary of agent_id to cumulative reward # used for reporting only. We always want to report the environment reward to Tensorboard, regardless # of what reward signals are actually present. diff --git a/ml-agents/mlagents/trainers/trainer_controller.py b/ml-agents/mlagents/trainers/trainer_controller.py index b1ed198b3d..fc7e7390ad 100644 --- a/ml-agents/mlagents/trainers/trainer_controller.py +++ b/ml-agents/mlagents/trainers/trainer_controller.py @@ -30,7 +30,7 @@ from mlagents.trainers.agent_processor import AgentManager -class TrainerController(object): +class TrainerController: def __init__( self, trainer_factory: TrainerFactory, diff --git a/ml-agents/mlagents/trainers/training_status.py b/ml-agents/mlagents/trainers/training_status.py index a9814a39f3..cc8841c11c 100644 --- a/ml-agents/mlagents/trainers/training_status.py +++ b/ml-agents/mlagents/trainers/training_status.py @@ -67,7 +67,7 @@ def load_state(path: str) -> None: :param path: Path to the JSON file containing the state. """ try: - with open(path, "r") as f: + with open(path) as f: loaded_dict = json.load(f) # Compare the metadata _metadata = loaded_dict[StatusType.STATS_METADATA.value] diff --git a/ml-agents/setup.py b/ml-agents/setup.py index 106fe146f2..8bc6a8a1bb 100644 --- a/ml-agents/setup.py +++ b/ml-agents/setup.py @@ -1,4 +1,3 @@ -from io import open import os import sys @@ -25,7 +24,7 @@ def run(self): tag = os.getenv("CIRCLE_TAG") if tag != EXPECTED_TAG: - info = "Git tag: {0} does not match the expected tag of this app: {1}".format( + info = "Git tag: {} does not match the expected tag of this app: {}".format( tag, EXPECTED_TAG ) sys.exit(info) diff --git a/ml-agents/tests/yamato/check_coverage_percent.py b/ml-agents/tests/yamato/check_coverage_percent.py index e3f2f11f76..025db1375a 100644 --- a/ml-agents/tests/yamato/check_coverage_percent.py +++ b/ml-agents/tests/yamato/check_coverage_percent.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys import os diff --git a/utils/validate_release_links.py b/utils/validate_release_links.py index cf24bc4d14..58d7c77fba 100755 --- a/utils/validate_release_links.py +++ b/utils/validate_release_links.py @@ -77,7 +77,7 @@ def check_file(filename: str, global_allow_pattern: Pattern) -> List[str]: Validate a single file and return any offending lines. """ bad_lines = [] - with open(filename, "r") as f: + with open(filename) as f: for line in f: if not RELEASE_PATTERN.search(line): continue diff --git a/utils/validate_versions.py b/utils/validate_versions.py index 0bcf37c6e2..dfc187bf84 100755 --- a/utils/validate_versions.py +++ b/utils/validate_versions.py @@ -94,7 +94,7 @@ def set_version( def set_package_version(new_version: str) -> None: - with open(MLAGENTS_PACKAGE_JSON_PATH, "r") as f: + with open(MLAGENTS_PACKAGE_JSON_PATH) as f: package_json = json.load(f) if "version" in package_json: package_json["version"] = new_version @@ -104,7 +104,7 @@ def set_package_version(new_version: str) -> None: def set_extension_package_version(new_version: str) -> None: - with open(MLAGENTS_EXTENSIONS_PACKAGE_JSON_PATH, "r") as f: + with open(MLAGENTS_EXTENSIONS_PACKAGE_JSON_PATH) as f: package_json = json.load(f) package_json["dependencies"]["com.unity.ml-agents"] = new_version with open(MLAGENTS_EXTENSIONS_PACKAGE_JSON_PATH, "w") as f: