diff --git a/Pipfile b/Pipfile index 9c6d029c..b72e172a 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,7 @@ click-help-colors = "*" colorama = "*" click-completion = "*" halo = "*" +pyyaml = "*" [dev-packages] twine = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 35d23dc5..49f943b5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "842e12146831fa76369dd8ba9966cc6bd4e78435dbf847472d4a8605f1400262" + "sha256": "5f6a5a92958160552d631faa3bc1643ab1238ff42316406eb79f31c086c88785" }, "pipfile-spec": 6, "requires": {}, @@ -148,9 +148,9 @@ }, "cursor": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:33f279a17789c04efd27a92501a0dad62bb011f8a4cdff93867c798d26508940" ], - "version": "==1.2.0" + "version": "==1.3.4" }, "datadog": { "hashes": [ @@ -203,11 +203,11 @@ }, "halo": { "hashes": [ - "sha256:44f55e408b55607193bfce728ac2fc8d8225e44445e2d9074a2d941cc05eb826", - "sha256:73c3f168a03f854ae4fabcc1d3bb69bbc41110e8abb91eb42390fc9876901a9e" + "sha256:9adbfe0a23ae7198a17895aac8845685eb0abeaa13c6a56b60410b347d0f503a", + "sha256:9ebf98b94a43f3b68e18c6d74dcb1ea58446b8457ce6fb1b2b4cac8d83733f80" ], "index": "pypi", - "version": "==0.0.26" + "version": "==0.0.28" }, "hyperopt": { "hashes": [ @@ -241,10 +241,10 @@ }, "log-symbols": { "hashes": [ - "sha256:2431e5fed65ff02d151f17a598c19ef91a2a381cf080d0be0fd6573fa416c5e5", - "sha256:ecc2f8f3ee586fd819d8c8696705914761cd8367d1f10597b9b49a0980ab6e55" + "sha256:4952106ff8b605ab7d5081dd2c7e6ca7374584eff7086f499c06edd1ce56dcca", + "sha256:cf0bbc6fe1a8e53f0d174a716bc625c4f87043cc21eb55dd8a740cfe22680556" ], - "version": "==0.0.13" + "version": "==0.0.14" }, "markupsafe": { "hashes": [ @@ -281,10 +281,10 @@ }, "marshmallow": { "hashes": [ - "sha256:9cedfc5b6f568d57e8a2cf3d293fbd81b05e5ef557854008d03e25660a39ccfd", - "sha256:a4d99922116a76e5abd8f997ec0519086e24814b7e1e1344bebe2a312ba50235" + "sha256:43bef4d33b7adb1f66eba8074095208b38dec96bed51f7a9bf2e687750f226d8", + "sha256:b240f5e14bc641c257f4b7bda3951d7e71963ebf66bd519078267f1f961cbd15" ], - "version": "==2.19.5" + "version": "==2.20.1" }, "networkx": { "hashes": [ @@ -389,6 +389,25 @@ ], "version": "==2.3.0" }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "index": "pypi", + "version": "==5.1.2" + }, "requests": { "extras": [ "security" @@ -477,10 +496,10 @@ }, "tqdm": { "hashes": [ - "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61", - "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab" + "sha256:1dc82f87a8726602fa7177a091b5e8691d6523138a8f7acd08e58088f51e389f", + "sha256:47220a4f2aeebbc74b0ab317584264ea44c745e1fd5ff316b675cd0aff8afad8" ], - "version": "==4.32.2" + "version": "==4.33.0" }, "urllib3": { "hashes": [ @@ -556,11 +575,11 @@ }, "configparser": { "hashes": [ - "sha256:8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32", - "sha256:da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75" + "sha256:45d1272aad6cfd7a8a06cf5c73f2ceb6a190f6acc1fa707e7f82a4c053b28b18", + "sha256:bc37850f0cc42a1725a796ef7d92690651bf1af37d744cc63161dac62cabee17" ], "markers": "python_version < '3'", - "version": "==3.7.4" + "version": "==3.8.1" }, "contextlib2": { "hashes": [ @@ -572,48 +591,49 @@ }, "coverage": { "hashes": [ - "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", - "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", - "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", - "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", - "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", - "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", - "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", - "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", - "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", - "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", - "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", - "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", - "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", - "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", - "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", - "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", - "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", - "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", - "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", - "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", - "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", - "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", - "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", - "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", - "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", - "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", - "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", - "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", - "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", - "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", - "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" ], "index": "pypi", - "version": "==4.5.3" + "version": "==4.5.4" }, "docutils": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" ], - "version": "==0.14" + "version": "==0.15.2" }, "filelock": { "hashes": [ @@ -646,10 +666,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", + "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" ], - "version": "==0.18" + "version": "==0.19" }, "jinja2": { "hashes": [ @@ -710,10 +730,10 @@ }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" ], - "version": "==19.0" + "version": "==19.1" }, "pathlib2": { "hashes": [ @@ -760,25 +780,25 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pytest": { "hashes": [ - "sha256:6aa9bc2f6f6504d7949e9df2a756739ca06e58ffda19b5e53c725f7b03fb4aae", - "sha256:b77ae6f2d1a760760902a7676887b665c086f71e3461c64ed2a312afcedc00d6" + "sha256:8fc39199bdda3d9d025d3b1f4eb99a192c20828030ea7c9a0d2840721de7d347", + "sha256:d100a02770f665f5dcf7e3f08202db29857fee6d15f34c942be0a511f39814f0" ], "index": "pypi", - "version": "==4.6.4" + "version": "==4.6.5" }, "pytz": { "hashes": [ - "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", - "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" ], - "version": "==2019.1" + "version": "==2019.2" }, "readme-renderer": { "hashes": [ @@ -869,10 +889,10 @@ }, "tqdm": { "hashes": [ - "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61", - "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab" + "sha256:1dc82f87a8726602fa7177a091b5e8691d6523138a8f7acd08e58088f51e389f", + "sha256:47220a4f2aeebbc74b0ab317584264ea44c745e1fd5ff316b675cd0aff8afad8" ], - "version": "==4.32.2" + "version": "==4.33.0" }, "twine": { "hashes": [ @@ -900,10 +920,10 @@ }, "virtualenv": { "hashes": [ - "sha256:861bbce3a418110346c70f5c7a696fdcf23a261424e1d28aa4f9362fc2ccbc19", - "sha256:ba8ce6a961d842320681fb90a3d564d0e5134f41dacd0e2bae7f02441dde2d52" + "sha256:6cb2e4c18d22dbbe283d0a0c31bb7d90771a606b2cb3415323eea008eaee6a9d", + "sha256:909fe0d3f7c9151b2df0a2cb53e55bdb7b0d61469353ff7a49fd47b0f0ab9285" ], - "version": "==16.6.2" + "version": "==16.7.2" }, "wcwidth": { "hashes": [ diff --git a/gradient/cli/common.py b/gradient/cli/common.py index ce307240..9c880fa5 100644 --- a/gradient/cli/common.py +++ b/gradient/cli/common.py @@ -2,8 +2,14 @@ import json import click +import yaml +from click.exceptions import Exit from click_didyoumean import DYMMixin from click_help_colors import HelpColorsGroup +from gradient.cli import cli_types + +OPTIONS_FILE_PARAMETER_NAME = "options_file" + api_key_option = click.option( "--apiKey", @@ -47,3 +53,77 @@ def decorator(f): f.invoke = functools.partial(new_invoke, f) return decorator + + +def get_option_name(options_strings): + for opt in options_strings: + if not opt.startswith("-"): + return opt + + if opt.startswith("--"): + return opt[2:] + + +class ReadValueFromConfigFile(click.Parameter): + def handle_parse_result(self, ctx, opts, args): + config_file = ctx.params.get(OPTIONS_FILE_PARAMETER_NAME) + if config_file: + with open(config_file) as f: + config_data = yaml.load(f, Loader=yaml.FullLoader) + option_name = get_option_name(self.opts) + value = config_data.get(option_name) + if value is not None: + if isinstance(value, dict): + value = json.dumps(value) + + opts[self.name] = value + + return super(ReadValueFromConfigFile, self).handle_parse_result( + ctx, opts, args) + + +class ArgumentReadValueFromConfigFile(ReadValueFromConfigFile, click.Argument): + pass + + +class OptionReadValueFromConfigFile(ReadValueFromConfigFile, click.Option): + pass + + +def generate_options_template(ctx, param, value): + if not value: + return value + + params = {} + for param in ctx.command.params: + option_name = get_option_name(param.opts) + option_value = ctx.params.get(param.name) or param.default + + if isinstance(param.type, cli_types.ChoiceType): + for key, val in param.type.type_map.items(): + if val == option_value: + option_value = key + + params[option_name] = option_value + + with open(value, "w") as f: + yaml.safe_dump(params, f, default_flow_style=False) + + raise Exit # to stop execution without executing the command + + +def options_file(f): + options = [ + click.option( + "--optionsFile", + OPTIONS_FILE_PARAMETER_NAME, + help="Path to YAML file with predefined options", + ), + click.option( + "--optionsFileTemplate", + callback=generate_options_template, + expose_value=False, + help="Generate template options file" + ) + ] + return functools.reduce(lambda x, opt: opt(x), reversed(options), f) diff --git a/gradient/cli/experiments.py b/gradient/cli/experiments.py index dcf0de65..ed8c0efb 100644 --- a/gradient/cli/experiments.py +++ b/gradient/cli/experiments.py @@ -48,67 +48,80 @@ def common_experiments_create_options(f): "--name", required=True, help="Name of new experiment", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--ports", help="Port to use in new experiment", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workspace", "workspace", help="Path to workspace directory, archive, S3 or git repository", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workspaceArchive", "workspace_archive", help="Path to workspace .zip archive", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workspaceUrl", "workspace_url", help="Project git repository url", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--ignoreFiles", "ignore_files", - help="Ignore certain files from uploading" + help="Ignore certain files from uploading", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workingDirectory", "working_directory", help="Working directory for the experiment", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--artifactDirectory", "artifact_directory", help="Artifacts directory", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--clusterId", "cluster_id", help="Cluster ID", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--experimentEnv", "experiment_env", type=json_string, help="Environment variables in a JSON", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--projectId", "project_id", required=True, help="Project ID", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--modelType", "model_type", help="Model type", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--modelPath", "model_path", help="Model path", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--isPreemptible", @@ -116,6 +129,7 @@ def common_experiments_create_options(f): type=bool, is_flag=True, help="Flag: is preemptible", + cls=common.OptionReadValueFromConfigFile, ), api_key_option ] @@ -130,24 +144,28 @@ def common_experiment_create_multi_node_options(f): type=ChoiceType(MULTI_NODE_EXPERIMENT_TYPES_MAP, case_sensitive=False), required=True, help="Experiment Type", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerContainer", "worker_container", required=True, help="Worker container", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerMachineType", "worker_machine_type", required=True, help="Worker machine type", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerCommand", "worker_command", required=True, help="Worker command", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerCount", @@ -155,24 +173,28 @@ def common_experiment_create_multi_node_options(f): type=int, required=True, help="Worker count", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerContainer", "parameter_server_container", required=True, help="Parameter server container", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerMachineType", "parameter_server_machine_type", required=True, help="Parameter server machine type", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerCommand", "parameter_server_command", required=True, help="Parameter server command", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerCount", @@ -180,52 +202,62 @@ def common_experiment_create_multi_node_options(f): type=int, required=True, help="Parameter server count", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerContainerUser", "worker_container_user", help="Worker container user", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerRegistryUsername", "worker_registry_username", help="Worker container registry username", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerRegistryPassword", "worker_registry_password", help="Worker registry password", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--workerRegistryUrl", "worker_registry_url", help="Worker registry URL", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerContainerUser", "parameter_server_container_user", help="Parameter server container user", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerRegistryUsername", "parameter_server_registry_username", help="Parameter server registry username", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerRegistryPassword", "parameter_server_registry_password", help="Parameter server registry password", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--parameterServerRegistryUrl", "parameter_server_registry_url", help="Parameter server registry URL", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--vpc", "use_vpc", type=bool, is_flag=True, + cls=common.OptionReadValueFromConfigFile, ), ] return functools.reduce(lambda x, opt: opt(x), reversed(options), f) @@ -237,43 +269,51 @@ def common_experiments_create_single_node_options(f): "--container", required=True, help="Container", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--machineType", "machine_type", required=True, help="Machine type", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--command", required=True, help="Container entrypoint command", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--containerUser", "container_user", help="Container user", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--registryUsername", "registry_username", help="Registry username", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--registryPassword", "registry_password", help="Registry password", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--registryUrl", "registry_url", help="Registry URL", + cls=common.OptionReadValueFromConfigFile, ), click.option( "--vpc", "use_vpc", type=bool, is_flag=True, + cls=common.OptionReadValueFromConfigFile, ), ] return functools.reduce(lambda x, opt: opt(x), reversed(options), f) @@ -288,14 +328,14 @@ def show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive https://docs.paperspace.com If you depend on functionality not listed there, please file an issue.""" - # click.echo(click.style(msg, fg='red'), err=True) logger.Logger().error(msg) @create_experiment.command(name="multinode", help="Create multi node experiment") @common_experiments_create_options @common_experiment_create_multi_node_options -def create_multi_node(api_key, use_vpc, **kwargs): +@common.options_file +def create_multi_node(api_key, use_vpc, options_file, **kwargs): show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs) utils.validate_workspace_input(kwargs) @@ -311,7 +351,8 @@ def create_multi_node(api_key, use_vpc, **kwargs): @create_experiment.command(name="singlenode", help="Create single node experiment") @common_experiments_create_options @common_experiments_create_single_node_options -def create_single_node(api_key, use_vpc, **kwargs): +@common.options_file +def create_single_node(api_key, use_vpc, options_file, **kwargs): show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs) utils.validate_workspace_input(kwargs) @@ -335,8 +376,9 @@ def create_single_node(api_key, use_vpc, **kwargs): default=True, help="Don't show logs. Only create, start and exit", ) +@common.options_file @click.pass_context -def create_and_start_multi_node(ctx, api_key, show_logs, use_vpc, **kwargs): +def create_and_start_multi_node(ctx, api_key, show_logs, use_vpc, options_file, **kwargs): show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs) utils.validate_workspace_input(kwargs) @@ -362,8 +404,9 @@ def create_and_start_multi_node(ctx, api_key, show_logs, use_vpc, **kwargs): default=True, help="Don't show logs. Only create, start and exit", ) +@common.options_file @click.pass_context -def create_and_start_single_node(ctx, api_key, show_logs, use_vpc, **kwargs): +def create_and_start_single_node(ctx, api_key, show_logs, use_vpc, options_file, **kwargs): show_workspace_deprecation_warning_if_workspace_archive_or_workspace_archive_was_used(kwargs) utils.validate_workspace_input(kwargs) @@ -379,27 +422,30 @@ def create_and_start_single_node(ctx, api_key, show_logs, use_vpc, **kwargs): @experiments.command("start", help="Start experiment") -@click.argument("experiment-id") -@api_key_option +@click.argument("id", cls=common.ArgumentReadValueFromConfigFile) @click.option( "--logs", "show_logs", is_flag=True, help="Show logs", ) +@common.options_file @click.option( "--vpc", "use_vpc", type=bool, is_flag=True, + cls=common.OptionReadValueFromConfigFile ) +@api_key_option +@common.options_file @click.pass_context -def start_experiment(ctx, experiment_id, show_logs, api_key, use_vpc): +def start_experiment(ctx, id, show_logs, api_key, options_file, use_vpc): command = experiments_commands.StartExperimentCommand(api_key=api_key) - command.execute(experiment_id, use_vpc=use_vpc) + command.execute(id, use_vpc=use_vpc) if show_logs: - ctx.invoke(list_logs, experiment_id=experiment_id, line=0, limit=100, follow=True, api_key=api_key) + ctx.invoke(list_logs, experiment_id=id, line=0, limit=100, follow=True, api_key=api_key) @experiments.command("stop", help="Stop experiment") @@ -411,7 +457,8 @@ def start_experiment(ctx, experiment_id, show_logs, api_key, use_vpc): type=bool, is_flag=True, ) -def stop_experiment(experiment_id, api_key, use_vpc): +@common.options_file +def stop_experiment(experiment_id, api_key, options_file, use_vpc): command = experiments_commands.StopExperimentCommand(api_key=api_key) command.execute(experiment_id, use_vpc=use_vpc) @@ -419,7 +466,8 @@ def stop_experiment(experiment_id, api_key, use_vpc): @experiments.command("list", help="List experiments") @click.option("--projectId", "-p", "project_ids", multiple=True) @api_key_option -def list_experiments(project_ids, api_key): +@common.options_file +def list_experiments(project_ids, api_key, options_file): command = experiments_commands.ListExperimentsCommand(api_key=api_key) command.execute(project_id=project_ids) @@ -427,7 +475,8 @@ def list_experiments(project_ids, api_key): @experiments.command("details", help="Show detail of an experiment") @click.argument("experiment-id") @api_key_option -def get_experiment_details(experiment_id, api_key): +@common.options_file +def get_experiment_details(experiment_id, options_file, api_key): command = experiments_commands.GetExperimentCommand(api_key=api_key) command.execute(experiment_id) @@ -457,6 +506,7 @@ def get_experiment_details(experiment_id, api_key): default=False ) @api_key_option -def list_logs(experiment_id, line, limit, follow, api_key=None): +@common.options_file +def list_logs(experiment_id, line, limit, follow, options_file, api_key=None): command = experiments_commands.ExperimentLogsCommand(api_key=api_key) command.execute(experiment_id, line, limit, follow) diff --git a/setup.py b/setup.py index c8f7a5fb..a60627b8 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run(self): 'prompt_toolkit<3.0', 'marshmallow<3.0', 'attrs<=19', + 'PyYAML', ], entry_points={'console_scripts': [ 'gradient = gradient:main.main', @@ -86,6 +87,7 @@ def run(self): 'mock', 'twine', 'sphinx', + 'pathlib2;python_version<"3.4"', ], }, cmdclass={ diff --git a/tests/config_files/create_multi_node_experiment.yaml b/tests/config_files/create_multi_node_experiment.yaml new file mode 100644 index 00000000..7984f38a --- /dev/null +++ b/tests/config_files/create_multi_node_experiment.yaml @@ -0,0 +1,34 @@ +apiKey: some_key +artifactDirectory: /artdir +clusterId: 2a +experimentEnv: + key: val +experimentType: MPI +ignoreFiles: file1,file2 +isPreemptible: true +modelPath: some-model-path +modelType: some-model-type +name: multinode_mpi +parameterServerCommand: ls +parameterServerContainer: pscon +parameterServerContainerUser: pscuser +parameterServerCount: 2 +parameterServerMachineType: psmtype +parameterServerRegistryPassword: psrpass +parameterServerRegistryUrl: psrurl +parameterServerRegistryUsername: psrcus +ports: '3456' +projectId: prq70zy79 +vpc: false +workerCommand: wcom +workerContainer: wcon +workerContainerUser: usr +workerCount: 2 +workerMachineType: mty +workerRegistryPassword: rpass +workerRegistryUrl: rurl +workerRegistryUsername: rusr +workingDirectory: /dir +workspace: null +workspaceArchive: null +workspaceUrl: wurl diff --git a/tests/config_files/create_single_node_experiment.yaml b/tests/config_files/create_single_node_experiment.yaml new file mode 100644 index 00000000..069f2ef2 --- /dev/null +++ b/tests/config_files/create_single_node_experiment.yaml @@ -0,0 +1,24 @@ +apiKey: some_key +artifactDirectory: /artifact/dir/ +clusterId: 42c +command: testCommand +container: testContainer +containerUser: conUser +experimentEnv: + key: val +ignoreFiles: file1,file2 +isPreemptible: true +machineType: testType +modelPath: some-model-path +modelType: some-model-type +name: exp1 +ports: '4567' +projectId: testHandle +registryPassword: passwd +registryUrl: registryUrl +registryUsername: userName +vpc: true +workingDirectory: /work/dir/ +workspace: null +workspaceArchive: null +workspaceUrl: wsp.url diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..781f34e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +@pytest.fixture +def create_single_node_experiment_config_path(): + p = Path(__file__) + fixture_dir = p.parent / "config_files" / "create_single_node_experiment.yaml" + return str(fixture_dir.resolve()) + + +@pytest.fixture +def create_multi_node_experiment_config_path(): + p = Path(__file__) + fixture_dir = p.parent / "config_files" / "create_multi_node_experiment.yaml" + return str(fixture_dir.resolve()) diff --git a/tests/functional/test_experiments.py b/tests/functional/test_experiments.py index 28e80d61..6c9ad128 100644 --- a/tests/functional/test_experiments.py +++ b/tests/functional/test_experiments.py @@ -45,6 +45,10 @@ class TestExperimentsCreateSingleNode(object): "--ignoreFiles", "file1,file2", "--isPreemptible", ] + FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE = [ + "experiments", "create", "singlenode", + "--optionsFile", # path added in test, + ] BASIC_OPTIONS_REQUEST = { "name": u"exp1", "projectHandle": u"testHandle", @@ -111,6 +115,24 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru data=None) assert result.exit_code == 0 + @mock.patch("gradient.api_sdk.clients.http_client.requests.post") + def test_should_read_options_from_config_file( + self, post_patched, create_single_node_experiment_config_path): + post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) + command = self.FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE[:] + [create_single_node_experiment_config_path] + + runner = CliRunner() + result = runner.invoke(cli.cli, command) + + assert self.EXPECTED_STDOUT in result.output, result.exc_info + post_patched.assert_called_once_with(self.URL_V2, + headers=self.EXPECTED_HEADERS, + json=self.FULL_OPTIONS_REQUEST, + params=None, + files=None, + data=None) + assert result.exit_code == 0 + @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_send_data_to_v2_url_when_vpc_switch_was_used(self, post_patched): post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) @@ -136,6 +158,7 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru runner = CliRunner() result = runner.invoke(cli.cli, self.FULL_OPTIONS_COMMAND) + assert self.EXPECTED_STDOUT in result.output, result.exc_info assert self.EXPECTED_STDOUT in result.output post_patched.assert_called_once_with(self.URL, headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY, @@ -219,6 +242,10 @@ class TestExperimentsCreateMultiNode(object): "--ignoreFiles", "file1,file2", "--isPreemptible", ] + FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE = [ + "experiments", "create", "multinode", + "--optionsFile", # path added in test, + ] BASIC_OPTIONS_REQUEST = { u"name": u"multinode_mpi", u"projectHandle": u"prq70zy79", @@ -303,6 +330,24 @@ def test_should_send_proper_data_and_print_message_when_create_experiment_was_ru data=None) assert result.exit_code == 0 + @mock.patch("gradient.api_sdk.clients.http_client.requests.post") + def test_should_read_options_from_config_file( + self, post_patched, create_multi_node_experiment_config_path): + post_patched.return_value = MockResponse(self.RESPONSE_JSON_200, 200, self.RESPONSE_CONTENT_200) + command = self.FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE[:] + [create_multi_node_experiment_config_path] + + runner = CliRunner() + result = runner.invoke(cli.cli, command) + + assert self.EXPECTED_STDOUT in result.output, result.exc_info + post_patched.assert_called_once_with(self.URL, + headers=self.EXPECTED_HEADERS, + json=self.FULL_OPTIONS_REQUEST, + params=None, + files=None, + data=None) + assert result.exit_code == 0 + @mock.patch("gradient.api_sdk.clients.http_client.requests.post") def test_should_send_proper_data_and_print_message_when_create_experiment_was_run_with_full_options(self, post_patched): @@ -376,6 +421,10 @@ class TestExperimentsCreateAndStartSingleNode(TestExperimentsCreateSingleNode): "--ignoreFiles", "file1,file2", "--isPreemptible", ] + FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE = [ + "experiments", "run", "singlenode", + "--optionsFile", # path added in test, + ] BASIC_OPTIONS_COMMAND_WITH_VPC_SWITCH = [ "experiments", "run", "singlenode", "--name", "exp1", @@ -444,6 +493,10 @@ class TestExperimentsCreateAndStartMultiNode(TestExperimentsCreateMultiNode): "--ignoreFiles", "file1,file2", "--isPreemptible", ] + FULL_OPTIONS_COMMAND_WITH_CONFIG_FILE = [ + "experiments", "run", "multinode", + "--optionsFile", # path added in test, + ] BASIC_OPTIONS_COMMAND_WITH_VPC_SWITCH = [ "experiments", "run", "multinode", "--name", "multinode_mpi",