Permalink
Browse files

SERVER-23312 Format Python files with yapf

  • Loading branch information...
hptabster committed Mar 26, 2018
1 parent d62d631 commit 36148ad8bbdb94162b2926f4700d935ee4dc5994
Showing with 3,113 additions and 4,036 deletions.
  1. +2 −1 {buildscripts → }/.style.yapf
  2. +2 −1 buildscripts/.pylintrc
  3. +10 −7 buildscripts/aggregate_tracefiles.py
  4. +57 −130 buildscripts/aws_ec2.py
  5. +19 −38 buildscripts/burn_in_tests.py
  6. +25 −28 buildscripts/bypass_compile_and_fetch_binaries.py
  7. +1 −0 buildscripts/ciconfig/__init__.py
  8. +9 −5 buildscripts/ciconfig/evergreen.py
  9. +4 −6 buildscripts/ciconfig/tags.py
  10. +75 −38 buildscripts/clang_format.py
  11. +6 −13 buildscripts/collect_resource_info.py
  12. +5 −10 buildscripts/combine_reports.py
  13. +45 −49 buildscripts/errorcodes.py
  14. +24 −7 buildscripts/eslint.py
  15. +30 −40 buildscripts/evergreen_run_tests.py
  16. +18 −32 buildscripts/fetch_test_lifecycle.py
  17. +5 −0 buildscripts/gdb/mongo.py
  18. +14 −14 buildscripts/gdb/mongo_lock.py
  19. +9 −10 buildscripts/gdb/mongo_printers.py
  20. +7 −4 buildscripts/generate_compile_expansions_shared_cache.py
  21. +10 −13 buildscripts/git.py
  22. +54 −97 buildscripts/hang_analyzer.py
  23. +2 −2 buildscripts/idl/idl/ast.py
  24. +18 −72 buildscripts/idl/idl/bson.py
  25. +36 −48 buildscripts/idl/idl/cpp_types.py
  26. +23 −28 buildscripts/idl/idl/enum_types.py
  27. +46 −37 buildscripts/idl/idl/errors.py
  28. +56 −70 buildscripts/idl/idl/generator.py
  29. +54 −49 buildscripts/idl/idl/parser.py
  30. +34 −63 buildscripts/idl/idl/struct_types.py
  31. +4 −4 buildscripts/idl/idl/syntax.py
  32. +6 −14 buildscripts/idl/idlc.py
  33. +2 −1 buildscripts/idl/tests/test_generator.py
  34. +12 −31 buildscripts/idl/tests/test_import.py
  35. +5 −15 buildscripts/idl/tests/test_parser.py
  36. +9 −18 buildscripts/jiraclient.py
  37. +62 −67 buildscripts/lint.py
  38. +1 −1 buildscripts/linter/yapf.py
  39. +20 −14 buildscripts/make_archive.py
  40. +16 −11 buildscripts/make_vcxproj.py
  41. +12 −1 buildscripts/moduleconfig.py
  42. +20 −21 buildscripts/mongosymb.py
  43. +52 −43 buildscripts/msitrim.py
  44. +100 −80 buildscripts/packager-enterprise.py
  45. +252 −184 buildscripts/packager.py
  46. +5 −7 buildscripts/promote_silent_failures.py
  47. +6 −5 buildscripts/prune_check.py
  48. +9 −11 buildscripts/pylinters.py
  49. +94 −163 buildscripts/remote_operations.py
  50. +1 −1 buildscripts/requirements.txt
  51. +8 −11 buildscripts/resmoke.py
  52. +1 −0 buildscripts/resmokeconfig/loggers/__init__.py
  53. +1 −0 buildscripts/resmokeconfig/suites/__init__.py
  54. +2 −0 buildscripts/resmokelib/__init__.py
  55. +13 −15 buildscripts/resmokelib/config.py
  56. +3 −3 buildscripts/resmokelib/core/network.py
  57. +8 −15 buildscripts/resmokelib/core/process.py
  58. +17 −23 buildscripts/resmokelib/logging/buildlogger.py
  59. +0 −1 buildscripts/resmokelib/logging/flush.py
  60. +1 −1 buildscripts/resmokelib/logging/formatters.py
  61. +2 −6 buildscripts/resmokelib/logging/handlers.py
  62. +12 −6 buildscripts/resmokelib/logging/loggers.py
  63. +24 −39 buildscripts/resmokelib/parser.py
  64. +40 −32 buildscripts/resmokelib/selector.py
  65. +2 −5 buildscripts/resmokelib/sighandler.py
  66. +5 −5 buildscripts/resmokelib/suitesconfig.py
  67. +12 −30 buildscripts/resmokelib/testing/executor.py
  68. +0 −2 buildscripts/resmokelib/testing/fixtures/__init__.py
  69. +1 −3 buildscripts/resmokelib/testing/fixtures/interface.py
  70. +17 −29 buildscripts/resmokelib/testing/fixtures/replicaset.py
  71. +26 −56 buildscripts/resmokelib/testing/fixtures/shardedcluster.py
  72. +4 −11 buildscripts/resmokelib/testing/fixtures/standalone.py
  73. +9 −15 buildscripts/resmokelib/testing/hook_test_archival.py
  74. +2 −2 buildscripts/resmokelib/testing/hooks/__init__.py
  75. +2 −2 buildscripts/resmokelib/testing/hooks/cleanup.py
  76. +4 −9 buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py
  77. +2 −5 buildscripts/resmokelib/testing/hooks/dbhash.py
  78. +11 −17 buildscripts/resmokelib/testing/hooks/initialsync.py
  79. +0 −1 buildscripts/resmokelib/testing/hooks/interface.py
  80. +5 −5 buildscripts/resmokelib/testing/hooks/jsfile.py
  81. +2 −5 buildscripts/resmokelib/testing/hooks/oplog.py
  82. +27 −29 buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
  83. +10 −13 buildscripts/resmokelib/testing/hooks/stepdown.py
  84. +2 −5 buildscripts/resmokelib/testing/hooks/validate.py
  85. +2 −2 buildscripts/resmokelib/testing/job.py
  86. +2 −2 buildscripts/resmokelib/testing/report.py
  87. +18 −19 buildscripts/resmokelib/testing/suite.py
  88. +3 −4 buildscripts/resmokelib/testing/summary.py
  89. +0 −1 buildscripts/resmokelib/testing/testcases/__init__.py
  90. +4 −10 buildscripts/resmokelib/testing/testcases/benchmark_test.py
  91. +2 −6 buildscripts/resmokelib/testing/testcases/cpp_integration_test.py
  92. +2 −7 buildscripts/resmokelib/testing/testcases/cpp_unittest.py
  93. +3 −9 buildscripts/resmokelib/testing/testcases/dbtest.py
  94. +3 −11 buildscripts/resmokelib/testing/testcases/fsm_workload_test.py
  95. +2 −3 buildscripts/resmokelib/testing/testcases/interface.py
  96. +3 −11 buildscripts/resmokelib/testing/testcases/json_schema_test.py
  97. +3 −10 buildscripts/resmokelib/testing/testcases/jsrunnerfile.py
  98. +9 −24 buildscripts/resmokelib/testing/testcases/jstest.py
  99. +2 −5 buildscripts/resmokelib/testing/testcases/mongos_test.py
  100. +2 −2 buildscripts/resmokelib/testing/testcases/sleeptest.py
  101. +30 −56 buildscripts/resmokelib/utils/archival.py
  102. +0 −1 buildscripts/resmokelib/utils/globstar.py
  103. +2 −3 buildscripts/resmokelib/utils/jscomment.py
  104. +0 −1 buildscripts/resmokelib/utils/queue.py
  105. +3 −4 buildscripts/resmokelib/utils/registry.py
  106. +1 −1 buildscripts/scons.py
  107. +5 −7 buildscripts/scons_cache_prune.py
  108. +35 −62 buildscripts/setup_multiversion_mongodb.py
  109. +42 −84 buildscripts/test_failures.py
  110. +1 −0 buildscripts/tests/__init__.py
  111. +1 −0 buildscripts/tests/ciconfig/__init__.py
  112. +16 −16 buildscripts/tests/ciconfig/test_evergreen.py
  113. +1 −2 buildscripts/tests/ciconfig/test_tags.py
  114. +1 −0 buildscripts/tests/resmokelib/__init__.py
  115. +6 −7 buildscripts/tests/resmokelib/logging/test_buildlogger.py
  116. +39 −42 buildscripts/tests/resmokelib/test_archival.py
  117. +137 −151 buildscripts/tests/resmokelib/test_selector.py
  118. +1 −0 buildscripts/tests/resmokelib/testing/__init__.py
  119. +0 −1 buildscripts/tests/resmokelib/testing/fixtures/test_interface.py
  120. +1 −0 buildscripts/tests/resmokelib/testing/hooks/__init__.py
  121. +14 −42 buildscripts/tests/resmokelib/testing/hooks/test_combine_benchmark_results.py
  122. +35 −90 buildscripts/tests/test_aws_ec2.py
  123. +23 −34 buildscripts/tests/test_fetch_test_lifecycle.py
  124. +0 −1 buildscripts/tests/test_git.py
  125. +13 −19 buildscripts/tests/test_remote_operations.py
  126. +383 −376 buildscripts/tests/test_test_failures.py
  127. +165 −158 buildscripts/tests/test_update_test_lifecycle.py
  128. +112 −192 buildscripts/update_test_lifecycle.py
  129. +15 −24 buildscripts/utils.py
  130. +2 −0 etc/evergreen.yml
  131. +306 −531 pytests/powertest.py
@@ -3,4 +3,5 @@
based_on_style = pep8
column_limit = 100
indent_dictionary_value = True

split_before_named_assigns = False
each_dict_entry_on_separate_line = False
@@ -1,10 +1,11 @@
# See https://www.pylint.org/
[MESSAGES CONTROL]
# C0301 - line-too-long - some of the type annotations are longer then 100 columns
# C0330 - bad-continuation - ignore conflicts produced by yapf formatting
# E0401 - import-error - ignore imports that fail to load
# I0011 - locally-disabled - ignore warnings about disable pylint checks
# R0903 - too-few-public-method - pylint does not always know best
# W0511 - fixme - ignore TODOs in comments
# W0611 - unused-import - typing module is needed for mypy

disable=fixme,import-error,line-too-long,locally-disabled,too-few-public-methods,unused-import
disable=bad-continuation,fixme,import-error,line-too-long,locally-disabled,too-few-public-methods,unused-import
@@ -2,11 +2,12 @@
import os
import sys
from optparse import OptionParser

""" This script aggregates several tracefiles into one tracefile
All but the last argument are input tracefiles or .txt files which list tracefiles.
The last argument is the tracefile to which the output will be written
"""


def aggregate(inputs, output):
"""Aggregates the tracefiles given in inputs to a tracefile given by output"""
args = ['lcov']
@@ -17,18 +18,20 @@ def aggregate(inputs, output):
args += ['-o', output]

print ' '.join(args)

return subprocess.call(args)

return subprocess.call(args)


def getfilesize(path):
if not os.path.isfile(path):
return 0
return os.path.getsize(path)

def main ():

def main():
inputs = []

usage = "usage: %prog input1.info input2.info ... output.info"
usage = "usage: %prog input1.info input2.info ... output.info"
parser = OptionParser(usage=usage)

(options, args) = parser.parse_args()
@@ -43,12 +46,12 @@ def main ():
inputs.append(path)

elif ext == '.txt':
inputs += [line.strip() for line in open(path)
if getfilesize(line.strip()) > 0]
inputs += [line.strip() for line in open(path) if getfilesize(line.strip()) > 0]
else:
return "unrecognized file type"

return aggregate(inputs, args[-1])


if __name__ == '__main__':
sys.exit(main())
@@ -1,5 +1,4 @@
#!/usr/bin/env python

"""AWS EC2 instance launcher and controller."""

from __future__ import print_function
@@ -21,16 +20,9 @@ class AwsEc2(object):
"""Class to support controlling AWS EC2 istances."""

InstanceStatus = collections.namedtuple("InstanceStatus", [
"instance_id",
"image_id",
"instance_type",
"state",
"private_ip_address",
"public_ip_address",
"private_dns_name",
"public_dns_name",
"tags"
])
"instance_id", "image_id", "instance_type", "state", "private_ip_address",
"public_ip_address", "private_dns_name", "public_dns_name", "tags"
])

def __init__(self):
try:
@@ -46,8 +38,7 @@ def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
"""Wait up to 'wait_time_secs' for instance to be in 'state'.
Return 0 if 'state' reached, 1 otherwise."""
if show_progress:
print("Waiting for instance {} to reach '{}' state".format(instance, state),
end="",
print("Waiting for instance {} to reach '{}' state".format(instance, state), end="",
file=sys.stdout)
reached_state = False
end_time = time.time() + wait_time_secs
@@ -83,8 +74,7 @@ def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False):
"""Controls an AMI instance. Returns 0 & status information, if successful."""
if mode not in _MODES:
raise ValueError(
"Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))
raise ValueError("Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))

sys.stdout.flush()
instance = self.connection.Instance(image_id)
@@ -112,23 +102,17 @@ def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False

ret = 0
if wait_time_secs > 0:
ret = self.wait_for_state(
instance=instance,
state=state,
wait_time_secs=wait_time_secs,
show_progress=show_progress)
ret = self.wait_for_state(instance=instance, state=state, wait_time_secs=wait_time_secs,
show_progress=show_progress)
try:
# Always provide status after executing command.
status = self.InstanceStatus(
getattr(instance, "instance_id", None),
getattr(instance, "image_id", None),
getattr(instance, "instance_type", None),
getattr(instance, "state", None),
getattr(instance, "instance_id", None), getattr(instance, "image_id", None),
getattr(instance, "instance_type", None), getattr(instance, "state", None),
getattr(instance, "private_ip_address", None),
getattr(instance, "public_ip_address", None),
getattr(instance, "private_dns_name", None),
getattr(instance, "public_dns_name", None),
getattr(instance, "tags", None))
getattr(instance, "public_dns_name", None), getattr(instance, "tags", None))
except botocore.exceptions.ClientError as err:
return 1, err.message

@@ -151,18 +135,9 @@ def tag_instance(self, image_id, tags):
time.sleep(i + 1)
instance.create_tags(Tags=tags)

def launch_instance(self,
ami,
instance_type,
block_devices=None,
key_name=None,
security_group_ids=None,
security_groups=None,
subnet_id=None,
tags=None,
wait_time_secs=0,
show_progress=False,
**kwargs):
def launch_instance(self, ami, instance_type, block_devices=None, key_name=None,
security_group_ids=None, security_groups=None, subnet_id=None, tags=None,
wait_time_secs=0, show_progress=False, **kwargs):
"""Launches and tags an AMI instance.
Returns the tuple (0, status_information), if successful."""
@@ -187,22 +162,15 @@ def launch_instance(self,
kwargs["KeyName"] = key_name

try:
instances = self.connection.create_instances(
ImageId=ami,
InstanceType=instance_type,
MaxCount=1,
MinCount=1,
**kwargs)
instances = self.connection.create_instances(ImageId=ami, InstanceType=instance_type,
MaxCount=1, MinCount=1, **kwargs)
except (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) as err:
return 1, err.message

instance = instances[0]
if wait_time_secs > 0:
self.wait_for_state(
instance=instance,
state="running",
wait_time_secs=wait_time_secs,
show_progress=show_progress)
self.wait_for_state(instance=instance, state="running", wait_time_secs=wait_time_secs,
show_progress=show_progress)

self.tag_instance(instance.instance_id, tags)

@@ -218,93 +186,60 @@ def main():
control_options = optparse.OptionGroup(parser, "Control options")
create_options = optparse.OptionGroup(parser, "Create options")

parser.add_option("--mode",
dest="mode",
choices=_MODES,
default="status",
help="Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES)))
parser.add_option("--mode", dest="mode", choices=_MODES, default="status",
help=("Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES))))

control_options.add_option("--imageId",
dest="image_id",
default=None,
control_options.add_option("--imageId", dest="image_id", default=None,
help="EC2 image_id to perform operation on [REQUIRED for control].")

control_options.add_option("--waitTimeSecs",
dest="wait_time_secs",
type=int,
default=5 * 60,
help="Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'.")
control_options.add_option("--waitTimeSecs", dest="wait_time_secs", type=int, default=5 * 60,
help=("Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'."))

create_options.add_option("--ami",
dest="ami",
default=None,
create_options.add_option("--ami", dest="ami", default=None,
help="EC2 AMI to launch [REQUIRED for create].")

create_options.add_option("--blockDevice",
dest="block_devices",
metavar="DEVICE-NAME DEVICE-SIZE-GB",
action="append",
default=[],
create_options.add_option("--blockDevice", dest="block_devices",
metavar="DEVICE-NAME DEVICE-SIZE-GB", action="append", default=[],
nargs=2,
help="EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance.")

create_options.add_option("--instanceType",
dest="instance_type",
default="t1.micro",
help=("EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance."))

create_options.add_option("--instanceType", dest="instance_type", default="t1.micro",
help="EC2 instance type to launch, defaults to '%default'.")

create_options.add_option("--keyName",
dest="key_name",
default=None,
create_options.add_option("--keyName", dest="key_name", default=None,
help="EC2 key name [REQUIRED for create].")

create_options.add_option("--securityGroupIds",
dest="security_group_ids",
action="append",
create_options.add_option("--securityGroupIds", dest="security_group_ids", action="append",
default=[],
help="EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once.")
help=("EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once."))

create_options.add_option("--securityGroup",
dest="security_groups",
action="append",
create_options.add_option("--securityGroup", dest="security_groups", action="append",
default=[],
help="EC2 security group. More than one security group can be added,"
" by specifying this option more than once.")
help=("EC2 security group. More than one security group can be added,"
" by specifying this option more than once."))

create_options.add_option("--subnetId",
dest="subnet_id",
default=None,
create_options.add_option("--subnetId", dest="subnet_id", default=None,
help="EC2 subnet id to use in VPC.")

create_options.add_option("--tagExpireHours",
dest="tag_expire_hours",
type=int,
default=2,
create_options.add_option("--tagExpireHours", dest="tag_expire_hours", type=int, default=2,
help="EC2 tag expire time in hours, defaults to '%default'.")

create_options.add_option("--tagName",
dest="tag_name",
default="",
create_options.add_option("--tagName", dest="tag_name", default="",
help="EC2 tag and instance name.")

create_options.add_option("--tagOwner",
dest="tag_owner",
default="",
help="EC2 tag owner.")
create_options.add_option("--tagOwner", dest="tag_owner", default="", help="EC2 tag owner.")

create_options.add_option("--extraArgs",
dest="extra_args",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
default=None,
help="EC2 create instance keyword args. The argument is specified as"
" bracketed YAML - i.e. JSON with support for single quoted"
" and unquoted keys. Example, '{DryRun: True}'")
create_options.add_option(
"--extraArgs", dest="extra_args", metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
default=None, help=("EC2 create instance keyword args. The argument is specified as"
" bracketed YAML - i.e. JSON with support for single quoted"
" and unquoted keys. Example, '{DryRun: True}'"))

parser.add_option_group(control_options)
parser.add_option_group(create_options)
@@ -331,34 +266,25 @@ def main():
# The 'expire-on' key is a UTC time.
expire_dt = datetime.datetime.utcnow() + datetime.timedelta(hours=options.tag_expire_hours)
tags = [{"Key": "expire-on", "Value": expire_dt.strftime("%Y-%m-%d %H:%M:%S")},
{"Key": "Name", "Value": options.tag_name},
{"Key": "owner", "Value": options.tag_owner}]
{"Key": "Name",
"Value": options.tag_name}, {"Key": "owner", "Value": options.tag_owner}]

my_kwargs = {}
if options.extra_args is not None:
my_kwargs = yaml.safe_load(options.extra_args)

(ret_code, instance_status) = aws_ec2.launch_instance(
ami=options.ami,
instance_type=options.instance_type,
block_devices=block_devices,
key_name=options.key_name,
security_group_ids=options.security_group_ids,
security_groups=options.security_groups,
subnet_id=options.subnet_id,
tags=tags,
wait_time_secs=options.wait_time_secs,
show_progress=True,
**my_kwargs)
ami=options.ami, instance_type=options.instance_type, block_devices=block_devices,
key_name=options.key_name, security_group_ids=options.security_group_ids,
security_groups=options.security_groups, subnet_id=options.subnet_id, tags=tags,
wait_time_secs=options.wait_time_secs, show_progress=True, **my_kwargs)
else:
if not getattr(options, "image_id", None):
parser.print_help()
parser.error("Missing required control option")

(ret_code, instance_status) = aws_ec2.control_instance(
mode=options.mode,
image_id=options.image_id,
wait_time_secs=options.wait_time_secs,
mode=options.mode, image_id=options.image_id, wait_time_secs=options.wait_time_secs,
show_progress=True)

print("Return code: {}, Instance status:".format(ret_code))
@@ -370,5 +296,6 @@ def main():

sys.exit(ret_code)


if __name__ == "__main__":
main()
Oops, something went wrong.

0 comments on commit 36148ad

Please sign in to comment.