Skip to content

Commit

Permalink
SERVER-23312 Format Python files with yapf
Browse files Browse the repository at this point in the history
  • Loading branch information
@hptabster
hptabster committed Mar 26, 2018
1 parent d62d631 commit 36148ad
Show file tree
Hide file tree
Showing 131 changed files with 3,113 additions and 4,036 deletions.
3 changes: 2 additions & 1 deletion buildscripts/.style.yapf → .style.yapf
View file
Expand Up @@ -3,4 +3,5 @@
based_on_style = pep8
column_limit = 100
indent_dictionary_value = True

split_before_named_assigns = False
each_dict_entry_on_separate_line = False
3 changes: 2 additions & 1 deletion buildscripts/.pylintrc
View file
@@ -1,10 +1,11 @@
# See https://www.pylint.org/
[MESSAGES CONTROL]
# C0301 - line-too-long - some of the type annotations are longer then 100 columns
# C0330 - bad-continuation - ignore conflicts produced by yapf formatting
# E0401 - import-error - ignore imports that fail to load
# I0011 - locally-disabled - ignore warnings about disable pylint checks
# R0903 - too-few-public-method - pylint does not always know best
# W0511 - fixme - ignore TODOs in comments
# W0611 - unused-import - typing module is needed for mypy

disable=fixme,import-error,line-too-long,locally-disabled,too-few-public-methods,unused-import
disable=bad-continuation,fixme,import-error,line-too-long,locally-disabled,too-few-public-methods,unused-import
17 changes: 10 additions & 7 deletions buildscripts/aggregate_tracefiles.py
View file
Expand Up @@ -2,11 +2,12 @@
import os
import sys
from optparse import OptionParser

""" This script aggregates several tracefiles into one tracefile
All but the last argument are input tracefiles or .txt files which list tracefiles.
The last argument is the tracefile to which the output will be written
"""


def aggregate(inputs, output):
"""Aggregates the tracefiles given in inputs to a tracefile given by output"""
args = ['lcov']
Expand All @@ -17,18 +18,20 @@ def aggregate(inputs, output):
args += ['-o', output]

print ' '.join(args)

return subprocess.call(args)

return subprocess.call(args)


def getfilesize(path):
if not os.path.isfile(path):
return 0
return os.path.getsize(path)

def main ():

def main():
inputs = []

usage = "usage: %prog input1.info input2.info ... output.info"
usage = "usage: %prog input1.info input2.info ... output.info"
parser = OptionParser(usage=usage)

(options, args) = parser.parse_args()
Expand All @@ -43,12 +46,12 @@ def main ():
inputs.append(path)

elif ext == '.txt':
inputs += [line.strip() for line in open(path)
if getfilesize(line.strip()) > 0]
inputs += [line.strip() for line in open(path) if getfilesize(line.strip()) > 0]
else:
return "unrecognized file type"

return aggregate(inputs, args[-1])


if __name__ == '__main__':
sys.exit(main())
187 changes: 57 additions & 130 deletions buildscripts/aws_ec2.py
View file
@@ -1,5 +1,4 @@
#!/usr/bin/env python

"""AWS EC2 instance launcher and controller."""

from __future__ import print_function
Expand All @@ -21,16 +20,9 @@ class AwsEc2(object):
"""Class to support controlling AWS EC2 istances."""

InstanceStatus = collections.namedtuple("InstanceStatus", [
"instance_id",
"image_id",
"instance_type",
"state",
"private_ip_address",
"public_ip_address",
"private_dns_name",
"public_dns_name",
"tags"
])
"instance_id", "image_id", "instance_type", "state", "private_ip_address",
"public_ip_address", "private_dns_name", "public_dns_name", "tags"
])

def __init__(self):
try:
Expand All @@ -46,8 +38,7 @@ def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
"""Wait up to 'wait_time_secs' for instance to be in 'state'.
Return 0 if 'state' reached, 1 otherwise."""
if show_progress:
print("Waiting for instance {} to reach '{}' state".format(instance, state),
end="",
print("Waiting for instance {} to reach '{}' state".format(instance, state), end="",
file=sys.stdout)
reached_state = False
end_time = time.time() + wait_time_secs
Expand Down Expand Up @@ -83,8 +74,7 @@ def wait_for_state(instance, state, wait_time_secs=0, show_progress=False):
def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False):
"""Controls an AMI instance. Returns 0 & status information, if successful."""
if mode not in _MODES:
raise ValueError(
"Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))
raise ValueError("Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))

sys.stdout.flush()
instance = self.connection.Instance(image_id)
Expand Down Expand Up @@ -112,23 +102,17 @@ def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False

ret = 0
if wait_time_secs > 0:
ret = self.wait_for_state(
instance=instance,
state=state,
wait_time_secs=wait_time_secs,
show_progress=show_progress)
ret = self.wait_for_state(instance=instance, state=state, wait_time_secs=wait_time_secs,
show_progress=show_progress)
try:
# Always provide status after executing command.
status = self.InstanceStatus(
getattr(instance, "instance_id", None),
getattr(instance, "image_id", None),
getattr(instance, "instance_type", None),
getattr(instance, "state", None),
getattr(instance, "instance_id", None), getattr(instance, "image_id", None),
getattr(instance, "instance_type", None), getattr(instance, "state", None),
getattr(instance, "private_ip_address", None),
getattr(instance, "public_ip_address", None),
getattr(instance, "private_dns_name", None),
getattr(instance, "public_dns_name", None),
getattr(instance, "tags", None))
getattr(instance, "public_dns_name", None), getattr(instance, "tags", None))
except botocore.exceptions.ClientError as err:
return 1, err.message

Expand All @@ -151,18 +135,9 @@ def tag_instance(self, image_id, tags):
time.sleep(i + 1)
instance.create_tags(Tags=tags)

def launch_instance(self,
ami,
instance_type,
block_devices=None,
key_name=None,
security_group_ids=None,
security_groups=None,
subnet_id=None,
tags=None,
wait_time_secs=0,
show_progress=False,
**kwargs):
def launch_instance(self, ami, instance_type, block_devices=None, key_name=None,
security_group_ids=None, security_groups=None, subnet_id=None, tags=None,
wait_time_secs=0, show_progress=False, **kwargs):
"""Launches and tags an AMI instance.
Returns the tuple (0, status_information), if successful."""
Expand All @@ -187,22 +162,15 @@ def launch_instance(self,
kwargs["KeyName"] = key_name

try:
instances = self.connection.create_instances(
ImageId=ami,
InstanceType=instance_type,
MaxCount=1,
MinCount=1,
**kwargs)
instances = self.connection.create_instances(ImageId=ami, InstanceType=instance_type,
MaxCount=1, MinCount=1, **kwargs)
except (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) as err:
return 1, err.message

instance = instances[0]
if wait_time_secs > 0:
self.wait_for_state(
instance=instance,
state="running",
wait_time_secs=wait_time_secs,
show_progress=show_progress)
self.wait_for_state(instance=instance, state="running", wait_time_secs=wait_time_secs,
show_progress=show_progress)

self.tag_instance(instance.instance_id, tags)

Expand All @@ -218,93 +186,60 @@ def main():
control_options = optparse.OptionGroup(parser, "Control options")
create_options = optparse.OptionGroup(parser, "Create options")

parser.add_option("--mode",
dest="mode",
choices=_MODES,
default="status",
help="Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES)))
parser.add_option("--mode", dest="mode", choices=_MODES, default="status",
help=("Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES))))

control_options.add_option("--imageId",
dest="image_id",
default=None,
control_options.add_option("--imageId", dest="image_id", default=None,
help="EC2 image_id to perform operation on [REQUIRED for control].")

control_options.add_option("--waitTimeSecs",
dest="wait_time_secs",
type=int,
default=5 * 60,
help="Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'.")
control_options.add_option("--waitTimeSecs", dest="wait_time_secs", type=int, default=5 * 60,
help=("Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'."))

create_options.add_option("--ami",
dest="ami",
default=None,
create_options.add_option("--ami", dest="ami", default=None,
help="EC2 AMI to launch [REQUIRED for create].")

create_options.add_option("--blockDevice",
dest="block_devices",
metavar="DEVICE-NAME DEVICE-SIZE-GB",
action="append",
default=[],
create_options.add_option("--blockDevice", dest="block_devices",
metavar="DEVICE-NAME DEVICE-SIZE-GB", action="append", default=[],
nargs=2,
help="EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance.")

create_options.add_option("--instanceType",
dest="instance_type",
default="t1.micro",
help=("EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance."))

create_options.add_option("--instanceType", dest="instance_type", default="t1.micro",
help="EC2 instance type to launch, defaults to '%default'.")

create_options.add_option("--keyName",
dest="key_name",
default=None,
create_options.add_option("--keyName", dest="key_name", default=None,
help="EC2 key name [REQUIRED for create].")

create_options.add_option("--securityGroupIds",
dest="security_group_ids",
action="append",
create_options.add_option("--securityGroupIds", dest="security_group_ids", action="append",
default=[],
help="EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once.")
help=("EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once."))

create_options.add_option("--securityGroup",
dest="security_groups",
action="append",
create_options.add_option("--securityGroup", dest="security_groups", action="append",
default=[],
help="EC2 security group. More than one security group can be added,"
" by specifying this option more than once.")
help=("EC2 security group. More than one security group can be added,"
" by specifying this option more than once."))

create_options.add_option("--subnetId",
dest="subnet_id",
default=None,
create_options.add_option("--subnetId", dest="subnet_id", default=None,
help="EC2 subnet id to use in VPC.")

create_options.add_option("--tagExpireHours",
dest="tag_expire_hours",
type=int,
default=2,
create_options.add_option("--tagExpireHours", dest="tag_expire_hours", type=int, default=2,
help="EC2 tag expire time in hours, defaults to '%default'.")

create_options.add_option("--tagName",
dest="tag_name",
default="",
create_options.add_option("--tagName", dest="tag_name", default="",
help="EC2 tag and instance name.")

create_options.add_option("--tagOwner",
dest="tag_owner",
default="",
help="EC2 tag owner.")
create_options.add_option("--tagOwner", dest="tag_owner", default="", help="EC2 tag owner.")

create_options.add_option("--extraArgs",
dest="extra_args",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
default=None,
help="EC2 create instance keyword args. The argument is specified as"
" bracketed YAML - i.e. JSON with support for single quoted"
" and unquoted keys. Example, '{DryRun: True}'")
create_options.add_option(
"--extraArgs", dest="extra_args", metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
default=None, help=("EC2 create instance keyword args. The argument is specified as"
" bracketed YAML - i.e. JSON with support for single quoted"
" and unquoted keys. Example, '{DryRun: True}'"))

parser.add_option_group(control_options)
parser.add_option_group(create_options)
Expand All @@ -331,34 +266,25 @@ def main():
# The 'expire-on' key is a UTC time.
expire_dt = datetime.datetime.utcnow() + datetime.timedelta(hours=options.tag_expire_hours)
tags = [{"Key": "expire-on", "Value": expire_dt.strftime("%Y-%m-%d %H:%M:%S")},
{"Key": "Name", "Value": options.tag_name},
{"Key": "owner", "Value": options.tag_owner}]
{"Key": "Name",
"Value": options.tag_name}, {"Key": "owner", "Value": options.tag_owner}]

my_kwargs = {}
if options.extra_args is not None:
my_kwargs = yaml.safe_load(options.extra_args)

(ret_code, instance_status) = aws_ec2.launch_instance(
ami=options.ami,
instance_type=options.instance_type,
block_devices=block_devices,
key_name=options.key_name,
security_group_ids=options.security_group_ids,
security_groups=options.security_groups,
subnet_id=options.subnet_id,
tags=tags,
wait_time_secs=options.wait_time_secs,
show_progress=True,
**my_kwargs)
ami=options.ami, instance_type=options.instance_type, block_devices=block_devices,
key_name=options.key_name, security_group_ids=options.security_group_ids,
security_groups=options.security_groups, subnet_id=options.subnet_id, tags=tags,
wait_time_secs=options.wait_time_secs, show_progress=True, **my_kwargs)
else:
if not getattr(options, "image_id", None):
parser.print_help()
parser.error("Missing required control option")

(ret_code, instance_status) = aws_ec2.control_instance(
mode=options.mode,
image_id=options.image_id,
wait_time_secs=options.wait_time_secs,
mode=options.mode, image_id=options.image_id, wait_time_secs=options.wait_time_secs,
show_progress=True)

print("Return code: {}, Instance status:".format(ret_code))
Expand All @@ -370,5 +296,6 @@ def main():

sys.exit(ret_code)


if __name__ == "__main__":
main()

0 comments on commit 36148ad

Please sign in to comment.