Skip to content

Commit

Permalink
Support more acceptance tests from ./assess helper
Browse files Browse the repository at this point in the history
  • Loading branch information
Tim McNamara committed May 9, 2019
1 parent 384b393 commit 98bb443
Showing 1 changed file with 267 additions and 12 deletions.
279 changes: 267 additions & 12 deletions acceptancetests/assess
View file
Expand Up @@ -65,7 +65,9 @@ def list_tests():
test_files = glob.glob(test_pattern)
test_files = (f.split('/')[-1] for f in test_files)
test_files = (f[7:-3] for f in test_files)
return sorted(test_files)
test_files = sorted(test_files)
test_files.remove("heterogeneous_control") # TODO(tsm) fixme
return test_files


def default_juju_bin():
Expand Down Expand Up @@ -154,6 +156,8 @@ def parse_args():
pass_through.add_argument('--region', default=None, help='Override environment region.')
pass_through.add_argument('--to', default=None,
help='Place the controller at a location.')
pass_through.add_argument("--agent", help="Which agent to test (Only relevant for log_rotation test)",
choices=["machine", "unit"])
pass_through.add_argument('--agent-url', action='store', default=None,
help='URL for retrieving agent binaries.')
pass_through.add_argument('--agent-stream', action='store', default=None,
Expand All @@ -162,14 +166,114 @@ def parse_args():
default=SERIES)
pass_through.add_argument("--keep-env", action="store_true",
help="Preserve the testing directories, e.g. HOME_DIR, DATA_DIR, ... after the test completes")

pass_through.add_argument("--logging-config",
help="Override logging configuration for a deployment. [default: \"<root>=INFO;unit=INFO\"]",
help="Override logging configuration for a deployment. [default: \"<root>=INFO;unit=INFO\" unless test defines its own defaults]",
default="<root>=INFO;unit=INFO")
pass_through.add_argument("--logging-module", help="Logger to use for the test. (Ignored by most tests)")
pass_through.add_argument("--charm-bundle", "--bundle", metavar="BUNDLE", dest="charm_bundle",
help="Overwrite default bundle (Relevant only for deploy_lxd_profile_bundle, deploy_webscale tests)")
pass_through.add_argument("--charm-series", metavar="SERIES",
help="Series for charms. (Relevant for recovery test only) [default: xenial]",
default="xenial")
pass_through.add_argument("--charm-devel-series", "--devel-series", dest="devel_series", metavar="SERIES",
help="The series to use when testing series in development (Relevant for multi_series_charms only)")
pass_through.add_argument("--agent-file",
help="Path to the agent file to be used for bootstrap. (Relevant only for the agent_metadata test)")
pass_through.add_argument("--bootstrap-part",
help="Which part of the bootstrap process to test. (Revelant only for the bootstrap test) [default: base]",
choices=['base', 'metadata', 'to'], default="base")
pass_through.add_argument("--local-metadata-source",
help="Path to directory with pre-loaded metadata. (Revelant only for the bootstrap test)")
pass_through.add_argument("--remote-metadata-source",
help="URL to image metadata (Relevant only for the mixed_images test)")
pass_through.add_argument("--export-to",
help="Part to where an exported bundle will be written to. (Relevant only for the bundle_export test)")
pass_through.add_argument("--caas-provider",
help="Provider for CaaS models. (Relevant only for caas_deploy_charms test) [default: microk8s]",
choices=["microk8s", "k8s"])
pass_through.add_argument('--caas-image-repo',
help="Docker image repo to use. (Relevant only for caas_deploy_charms test) [default: jujuqabot]",
default="jujuqabot")
pass_through.add_argument("--clouds-yaml",
help="Path to a valid clouds.yaml file (Relevant for add_cloud, cloud, cloud_display tests only)")
pass_through.add_argument("--cloud",
help="Specific cloud to test. Overwrites --substrate. (Relevant only for cloud test)")
pass_through.add_argument("--cloud-test", help="Sub-test to run (Relevant for cloud test only)",
choices={"combined", "provisioning", "kill-controller"})
pass_through.add_argument("--machine-series", metavar="SERIES", action="append",
help="Series for machine, use multiple times for multiple machines. (Only relevant for provisioning sub-test of cloud test) [default: xenial]",
default="xenial")
pass_through.add_argument("--machine-type", "--virt", dest="machine_type",
help="Virtualisation technology for machine (only relevant for container_networking, constraints tests) [default: lxd]",
choices=["lxc", "lxd", "kvm"], default="lxd")
pass_through.add_argument("--space-constraint", metavar="C",
help="Adds --constraints spaces=C to the underlying hypervisor (only relevant for container_networking test). ")
pass_through.add_argument("--secondary-env", metavar="ENV",
help="(Only relevant for the cross_model_relations test)")
pass_through.add_argument("--secondary-region", "--other-region", metavar="REGION", dest="secondary_region",
help="(Only relevant for the cross_model_relations, model_defaults tests)")
pass_through.add_argument("--reporting-uri", metavar="URI",
help="URI to send metrics to. (Only relevant for webscale_deploy test) [default: \"http://root:root@localhost:8086\"]",
default="http://root:root@localhost:8086")
pass_through.add_argument("--report-git-sha",
help="Provide git SHA to reporting server (Only relevant for webscale_deploy test)")
pass_through.add_argument("--report-juju-version",
help="Provide Juju version to reporting server (Only relevant for webscale_deploy test)")
pass_through.add_argument("--db-snap-path",
help="Path/URL to a .snap file that will be used as the MongoDB instance under test (Only relevant for webscale_deploy test)")
pass_through.add_argument("--db-asserts-path",
help="Path/URL to a .assert file that will be used as the MongoDB instance under test (Only relevant for webscale_deploy test)")
pass_through.add_argument("--mongo-memory-profile",
help="Name of a the memory profile to set MongoDB to (Only relevant for webscale_deploy test)",
choices=["low", "default"], default="default")
pass_through.add_argument("--mongo-server-side-txns", action="store_true",
help="Use MongoDB 4's server-side transactions functionality (Only relevant for webscale_deploy test)")
pass_through.add_argument("--stable-juju-bin",
help="Path to a binary that will be treated as the stable version of Juju (Relevant only for the model_migration_versions, upgrade tests)")
pass_through.add_argument("--devel-juju-agent",
help="Path to agent to use when bootstrapping with devel binary (Relevant only for the upgrade test)")
pass_through.add_argument("--stable-juju-agent",
help="Path to agent to use when bootstrapping with stable binary (Relevant only for the upgrade test)")
pass_through.add_argument("--reboot", action="store_true",
help="Reboot machines and re-run tests (Relevant only in the network_health test)")
pass_through.add_argument("--maas", action="store_true",
help="Run test under MAAS (Relevant only in the network_health test)")
pass_through.add_argument("--model", help="Model to test against (Relevant only in the network_health test)")
# pass_through.add_argument("--stack", choices={"maas", "iaas", "caas"}, help="(Relevant only in the network_health, webscale_deploy tests)") # TODO(tsm) add when relevant
pass_through.add_argument("--proxy-scenario", "--proxy-test",
choices={'both-proxied', 'client-proxied', 'controller-proxied'}, dest="proxy_scenario",
help="Sub-test to run (Only relevant in proxy test)", default="both-proxied")
pass_through.add_argument(
'--client-interface', default='eth0',
help="The interface used by the client to access the internet. (Only relevant in proxy test)")
pass_through.add_argument(
'--controller-interface', default='lxdbr0',
help="The interface used by the controller to access the internet. (Only relevant in proxy test)")
backup = pass_through.add_mutually_exclusive_group()
backup.add_argument("--backup-strategy", choices=["ha", "standard"],
help="(Relevant for the recovery test only) [default: standard]", default="standard")
backup.add_argument("--backup", action="store_true", help="Equivalent to --backup-strategy=standard")
backup.add_argument("--ha-backup", action="store_true", help="Equivalent to --backup-strategy=ha")
pass_through.add_argument('--large', '--large-test-enabled', action='store_true', dest="large_test_enabled",
help="Use a large file for testing. (Relevant only with resources test)")
pass_through.add_argument('--agent-timeout', metavar="S", type=int, default=1800,
help='The time to wait for agents to start (Relevant only with resources test) [default: 1800]')
pass_through.add_argument('--resource-timeout', metavar="S", type=int, default=1800,
help='The time to wait for agents to start (Relevant only with resources test) [default: 1800]')
pass_through.add_argument("--clean-environment",
help="Clean up a the environments between runs, rather than creating one from scratch (Only relevant for spaces_subnets test)",
action="store_true")
pass_through.add_argument('--from-series', default="xenial",
help='Series to start machine and units with')
pass_through.add_argument('--to-series', default="bionic",
help='Series to upgrade machine and units to')

cloud_opts = arg_parser.add_argument_group("options to pass through to the cloud substrate")
cloud_opts.add_argument("--cloud-region", metavar="REGION", help="The region to host the test. [default: aws:us-east-1]")
cloud_opts.add_argument("--cloud-region", metavar="REGION",
help="The region to host the test. [default: aws:us-east-1]")

exe_options = arg_parser.add_argument_group("executables:")
exe_options = arg_parser.add_argument_group("executables")
juju_bin_help = "Path to the Juju binary to be used for testing."
if juju_bin is not None:
juju_bin_help = juju_bin_help + " [default: {}]".format(juju_bin)
Expand Down Expand Up @@ -202,7 +306,21 @@ def parse_args():

if args.substrate != "lxd":
if args.cloud_region not in valid_regions:
raise ValueError("--cloud-region parameter must be one of {} when --substrate={}".format(valid_regions, args.substrate))
raise arg_parser.error(
"--cloud-region parameter must be one of {} when --substrate={}".format(valid_regions, args.substrate))

if args.assess in ("add_cloud", "cloud"):
if not args.clouds_yaml:
arg_parser.error("--clouds-yaml must be set for the {} test".format(args.assess))
elif args.assess == "agent_metadata":
if not args.agent_file:
arg_parser.error("--agent-file must be set for the agent_metadata test")
elif args.assess == "deploy_webscale":
if args.logging_config == "<root>=INFO;unit=INFO":
args.logging_config = "juju.state.txn=TRACE;<root>=INFO;unit=INFO"
elif args.assess == "log_rotation":
if not args.agent:
arg_parser.error("--agent must be set when testing log_rotation.")

return args

Expand Down Expand Up @@ -253,8 +371,8 @@ def main():
args.juju,
args.log_dir,
# args.run_name,
"--series=" + args.series,
"--logging-config=" + args.logging_config,
"--series", args.series,
"--logging-config", args.logging_config,
]
if args.debug:
testrun_argv += ["--debug"]
Expand All @@ -263,14 +381,151 @@ def main():
if args.keep_env:
testrun_argv += ["--keep-env"]
if args.region:
testrun_argv += ["--region=" + args.region]
testrun_argv += ["--region", args.region]
if args.to:
testrun_argv.extend(["--to", args.to])
testrun_argv += ["--to", args.to]
if args.agent_url:
testrun_argv.extend(["--agent-url", args.agent_url])
testrun_argv += ["--agent-url", args.agent_url]
if args.agent_stream:
testrun_argv.extend(["--agent-stream", args.agent_stream])

testrun_argv += ["--agent-stream", args.agent_stream]

assess = args.assess
if assess == "recovery":
backup_arg = "--backup"
if args.backup_strategy == "ha" or args.ha_backup:
backup_arg = "--ha-backup"

testrun_argv.append(backup_arg)
testrun_argv.extend(["--charm-series", args.charm_series])
elif assess == "add_cloud":
testrun_argv = [
args.python,
testrun_file,
args.clouds_yaml,
args.juju
]
elif assess == "agent_metadata":
testrun_argv.extend(['--agent-file', args.agent - file])
elif assess == "bootstrap":
testrun_argv.insert(1, args.bootstrap_part)
if args.local_metadata_source:
testrun_argv.extend(["--local-metadata-source", args.local_metadata_source])
elif assess == "bundle_export":
if args.export_to:
testrun_argv.extend(["--filename", args.export_to])
elif assess == "caas_deploy_charms":
provider = "K8S_CORE"
if args.caas_provider == "microk8s":
provider = "MICROK8S"
testrun_argv.extend(["--caas-provider", provider])
testrun_argv.extend(["--caas-image-repo", args.caas_image_repo])
elif assess == "cloud":
testrun_argv.insert(2, args.cloud_test)
testrun_argv.pop(3) # args.substrate,
testrun_argv.insert(3, args.cloud)
testrun_argv.insert(3, args.clouds_yaml)
for machine_series in args.machine_series:
testrun_argv.extend(["--machine-series", machine_series])
elif assess == "cloud_display":
testrun_argv = [
args.python,
testrun_file,
args.clouds_yaml,
args.juju,
]
elif assess == "container_networking":
machine_types = {
'kvm': 'KVM_MACHINE',
'lxc': 'LXC_MACHINE',
'lxd': 'LXD_MACHINE',
}
machine_type = machine_types[args.machine_type]
testrun_argv.extend(["--machine-type", machine_type])

if args.space_constraint:
testrun_argv.extend(["--space-constraint", args.space_constraint])
elif assess == "constraints":
if args.machine_type == "kvm":
testrun_argv.append("--with-virttype-kvm")
elif assess == "cross_model_relations":
if args.secondary_env:
testrun_argv.extend(["--secondary-env", args.secondary_env])
if args.secondary_region:
testrun_argv.extend(["--secondary-region", args.secondary_region])
elif assess == "deploy_lxd_profile_bundle":
if args.charm_bundle:
testrun_argv.extend(["--charm-bundle", args.charm_bundle])
elif assess == "deploy_webscale":
# TODO(tsm) support --stack-type when it can be more than iaas
if args.charm_bundle:
testrun_argv.extend(["--charm-bundle", args.charm_bundle])
if args.logging_module:
testrun_argv.extend(["--logging-module", args.logging_module])
if args.reporting_uri:
testrun_argv.extend(["--reporting-uri"])
if args.report_git_sha:
testrun_argv.extend(["--git-sha", args.report_git_sha])
if args.report_juju_version:
testrun_argv.extend(["--juju-version", args.report_juju_version])
if args.db_snap_path:
testrun_argv.extend(["--db-snap-path", args.db_snap_path])
if args.db_asserts_path:
testrun_argv.extend(["--db-asserts-path", args.db_asserts_path])
testrun_argv.extend(["--mongo-memory-profile", args.mongo_memory_profile])
if args.mongo_server_side_txns:
testrun_argv.extend(["--with-mongo-server-side-txns", "true"])
elif assess == "log_rotation":
testrun_argv.insert(5, args.agent)
elif assess == "mixed_images":
if args.image_metadata_url:
testrun_argv.extend(["--image-metadata-url", args.image_metadata_url])
elif assess == "model_defaults":
if args.secondary_region:
testrun_argv.extend(["--other-region", args.secondary_region])
elif assess == "model_migration_versions":
if args.stable_juju_bin:
testrun_argv.extend(["--stable-juju-bin", args.stable_juju_bin])
elif assess == "multi_series_charms":
if args.devel_series:
testrun_argv.extend(["--devel-series", args.devel_series])
elif assess == "network_health":
if args.charm_bundle:
testrun_argv.extend(["--bundle", args.charm_bundle])
if args.model:
testrun_argv.extend(["--model", args.model])
if args.reboot:
testrun_argv.extend(["--reboot", "true"])
if args.maas:
testrun_argv.extend(["--maas", "true"])
elif assess == "proxy":
testrun_argv.insert(5, args.proxy_scenario)
if args.client_interface:
testrun_argv.extend(["--client-interface", args.client_interface])
if args.controller_interface:
testrun_argv.extend(["--controller-interface", args.controller_interface])
elif assess == "resources":
if args.large_test_enabled:
testrun_argv.append("--large-test-enabled")
testrun_argv.extend(["--agent-timeout", args.agent_timeout])
testrun_argv.extend(["--resource-timeout", args.resource_timeout])
elif assess == "spaces_subnets":
if args.clean_environment:
testrun_argv.append("--clean-environment")
elif assess == "upgrade":
if args.stable_juju_bin:
testrun_argv.extend(["--stable-juju-bin", args.stable_juju_bin])
if args.stable_juju_agent:
testrun_argv.extend(["--stable-juju-agent", args.stable_juju_agent])
if args.devel_juju_agent:
testrun_argv.extend(["--devel-juju-agent", args.devel_juju_agent])
elif assess == "upgrade_series":
testrun_argv.extend([
"--from-series", args.from_series,
"--to-series", args.to_series
])

testrun_argv = [str(arg) for arg in testrun_argv]
LOG.info("testing " + ' '.join(testrun_argv))
proc = subprocess.Popen(
testrun_argv,
env=testrun_env,
Expand Down

0 comments on commit 98bb443

Please sign in to comment.