From b2d3eed82b678c3243079fc25e6b47d44c28c841 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 25 Sep 2020 16:32:55 +0800 Subject: [PATCH] SUBMARINE-633. [SDK] Support run experiment with synced code ### What is this PR for? - Support sync code when submitting the experiment - Update `OpenAPI.json` - update example, https://github.com/apache/submarine/blob/master/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb - Update experiment integration tests ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-633 ### How should this be tested? https://travis-ci.org/github/pingsutw/hadoop-submarine/builds/730205378 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Kevin Su Closes #411 from pingsutw/SUBMARINE-633 and squashes the following commits: 02659c1 [Kevin Su] SUBMARINE-633. [SDK] Support run experiment with synced code --- dev-support/pysubmarine/openapi.json | 52 ++- .../example/submarine_experiment_sdk.ipynb | 419 ++++++------------ .../submarine/experiment/__init__.py | 4 +- .../submarine/experiment/models/__init__.py | 4 +- .../models/{environment.py => code_spec.py} | 64 ++- .../experiment/models/environment_spec.py | 240 ++++++++++ .../experiment/models/experiment_spec.py | 38 +- .../experiment/models/kernel_spec.py | 188 ++++++++ .../experiment/test_experiment_client.py | 10 +- 9 files changed, 712 insertions(+), 307 deletions(-) rename submarine-sdk/pysubmarine/submarine/experiment/models/{environment.py => code_spec.py} (72%) create mode 100644 submarine-sdk/pysubmarine/submarine/experiment/models/environment_spec.py create mode 100644 submarine-sdk/pysubmarine/submarine/experiment/models/kernel_spec.py diff --git a/dev-support/pysubmarine/openapi.json b/dev-support/pysubmarine/openapi.json index be2a3123c1..ce237b6665 100644 --- a/dev-support/pysubmarine/openapi.json +++ b/dev-support/pysubmarine/openapi.json @@ -274,9 +274,32 @@ } } }, - "Environment" : { + "CodeSpec" : { "type" : "object", "properties" : { + "syncMode" : { + "type" : "string" + }, + "url" : { + "type" : "string" + } + } + }, + "EnvironmentSpec" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "dockerImage" : { + "type" : "string" + }, + "kernelSpec" : { + "$ref" : "#/components/schemas/KernelSpec" + }, + "description" : { + "type" : "string" + }, "image" : { "type" : "string" } @@ -312,13 +335,16 @@ "$ref" : "#/components/schemas/ExperimentMeta" }, "environment" : { - "$ref" : "#/components/schemas/Environment" + "$ref" : "#/components/schemas/EnvironmentSpec" }, "spec" : { "type" : "object", "additionalProperties" : { "$ref" : "#/components/schemas/ExperimentTaskSpec" } + }, + "code" : { + "$ref" : "#/components/schemas/CodeSpec" } } }, @@ -357,7 +383,27 @@ "type" : "string" } } + }, + "KernelSpec" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "channels" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "dependencies" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } } } } -} +} \ No newline at end of file diff --git a/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb b/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb index 2083a3fb35..79aafee9e3 100644 --- a/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb +++ b/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb @@ -12,19 +12,16 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "pycharm": { - "is_executing": false - } - }, + "metadata": {}, "outputs": [], "source": [ "from __future__ import print_function\n", "import submarine\n", - "from submarine.experiment.models.environment import Environment\n", + "from submarine.experiment.models.environment_spec import EnvironmentSpec\n", "from submarine.experiment.models.experiment_spec import ExperimentSpec\n", "from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec\n", - "from submarine.experiment.models.experiment_meta import ExperimentMeta" + "from submarine.experiment.models.experiment_meta import ExperimentMeta\n", + "from submarine.experiment.models.code_spec import CodeSpec" ] }, { @@ -39,13 +36,12 @@ "execution_count": 2, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" } }, "outputs": [], "source": [ - "submarine_client = submarine.ExperimentClient(host='http://localhost:8080')" + "submarine_client = submarine.ExperimentClient(host='http://submarine:8080')" ] }, { @@ -66,26 +62,27 @@ "execution_count": 3, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" } }, "outputs": [], "source": [ - "environment = Environment(image='gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0')\n", + "environment = EnvironmentSpec(image='gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0')\n", "experiment_meta = ExperimentMeta(name='mnist-dist',\n", " namespace='default',\n", " framework='Tensorflow',\n", - " cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100'\n", - " , env_vars={'ENV1': 'ENV1'})\n", + " cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", + " env_vars={'ENV1': 'ENV1'})\n", "\n", "worker_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n", " replicas=1)\n", "ps_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n", " replicas=1)\n", + "code_spec = CodeSpec(sync_mode='git', url='https://github.com/apache/submarine.git')\n", "\n", "experiment_spec = ExperimentSpec(meta=experiment_meta,\n", " environment=environment,\n", + " code=code_spec,\n", " spec={'Ps' : ps_spec,'Worker': worker_spec})\n" ] }, @@ -101,7 +98,6 @@ "execution_count": 4, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" }, "scrolled": true @@ -109,36 +105,7 @@ "outputs": [ { "data": { - "text/plain": [ - "{'experimentId': 'experiment_1592969710478_0001',\n", - " 'name': 'mnist-dist',\n", - " 'uid': '360886c6-b5cc-11ea-b5f2-025000000001',\n", - " 'status': 'Accepted',\n", - " 'acceptedTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'createdTime': None,\n", - " 'runningTime': None,\n", - " 'finishedTime': None,\n", - " 'spec': {'meta': {'name': 'mnist-dist',\n", - " 'namespace': 'default',\n", - " 'framework': 'Tensorflow',\n", - " 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", - " 'envVars': {'ENV1': 'ENV1'}},\n", - " 'environment': {'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n", - " 'spec': {'Ps': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n", - " 'Worker': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}}}}}" - ] + "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Accepted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': None,\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" }, "execution_count": 4, "metadata": {}, @@ -162,43 +129,13 @@ "execution_count": 5, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" } }, "outputs": [ { "data": { - "text/plain": [ - "{'experimentId': 'experiment_1592969710478_0001',\n", - " 'name': 'mnist-dist',\n", - " 'uid': '360886c6-b5cc-11ea-b5f2-025000000001',\n", - " 'status': 'Running',\n", - " 'acceptedTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'createdTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'runningTime': '2020-06-24T11:38:49.000+08:00',\n", - " 'finishedTime': None,\n", - " 'spec': {'meta': {'name': 'mnist-dist',\n", - " 'namespace': 'default',\n", - " 'framework': 'Tensorflow',\n", - " 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", - " 'envVars': {'ENV1': 'ENV1'}},\n", - " 'environment': {'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n", - " 'spec': {'Ps': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n", - " 'Worker': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}}}}}" - ] + "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Created',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" }, "execution_count": 5, "metadata": {}, @@ -219,10 +156,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" }, "scrolled": true @@ -230,38 +166,9 @@ "outputs": [ { "data": { - "text/plain": [ - "[{'experimentId': 'experiment_1592969710478_0001',\n", - " 'name': 'mnist-dist',\n", - " 'uid': '360886c6-b5cc-11ea-b5f2-025000000001',\n", - " 'status': 'Running',\n", - " 'acceptedTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'createdTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'runningTime': '2020-06-24T11:38:49.000+08:00',\n", - " 'finishedTime': None,\n", - " 'spec': {'meta': {'name': 'mnist-dist',\n", - " 'namespace': 'default',\n", - " 'framework': 'Tensorflow',\n", - " 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", - " 'envVars': {'ENV1': 'ENV1'}},\n", - " 'environment': {'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n", - " 'spec': {'Ps': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n", - " 'Worker': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}}}}}]" - ] + "text/plain": "[{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Running',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': '2020-09-25T16:53:19.000+08:00',\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}]" }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -280,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": { "scrolled": false }, @@ -291,10 +198,10 @@ "text": [ "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", " from ._conv import register_converters as _register_converters\n", - "2020-06-24 03:39:55.150301: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", - "2020-06-24 03:39:55.154457: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n", - "2020-06-24 03:39:55.154492: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n", - "2020-06-24 03:39:55.155476: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n" + "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", + "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n", + "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n", + "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n" ] } ], @@ -306,15 +213,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Get specific experiment training log " + "### Get specific experiment training log" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" }, "scrolled": true @@ -324,18 +230,26 @@ "name": "stderr", "output_type": "stream", "text": [ + "The logs of Pod mnist-dist-ps-0:\n", + "\n", + "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + " from ._conv import register_converters as _register_converters\n", + "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", + "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n", + "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n", + "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n", "The logs of Pod mnist-dist-worker-0:\n", "\n", "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", " from ._conv import register_converters as _register_converters\n", - "2020-06-24 03:39:55.118374: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", - "2020-06-24 03:39:55.148641: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> mnist-dist-ps-0.default.svc:2222}\n", - "2020-06-24 03:39:55.148726: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:2222}\n", - "2020-06-24 03:39:55.150348: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n", + "2020-09-25 08:53:21.614236: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", + "2020-09-25 08:53:21.622645: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> mnist-dist-ps-0.default.svc:2222}\n", + "2020-09-25 08:53:21.622666: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:2222}\n", + "2020-09-25 08:53:21.627061: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n", "WARNING:tensorflow:From /var/tf_dist_mnist/dist_mnist.py:239: __init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please switch to tf.train.MonitoredTrainingSession\n", - "2020-06-24 03:39:55.880787: I tensorflow/core/distributed_runtime/master_session.cc:1017] Start master session 23a80a92d64440cc with config: device_filters: \"/job:ps\" device_filters: \"/job:worker/task:0\" allow_soft_placement: true\n", + "2020-09-25 08:53:21.893834: I tensorflow/core/distributed_runtime/master_session.cc:1017] Start master session 1f6c5256b60e4b78 with config: device_filters: \"/job:ps\" device_filters: \"/job:worker/task:0\" allow_soft_placement: true\n", "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n", "Extracting /tmp/mnist-data/train-images-idx3-ubyte.gz\n", "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n", @@ -348,117 +262,111 @@ "task index = 0\n", "Worker 0: Initializing session...\n", "Worker 0: Session initialization complete.\n", - "Training begins @ 1592969996.537955\n", - "1592969997.322857: Worker 0: training step 1 done (global step: 0)\n", - "1592969997.333140: Worker 0: training step 2 done (global step: 1)\n", - "1592969997.342255: Worker 0: training step 3 done (global step: 2)\n", - "1592969997.350622: Worker 0: training step 4 done (global step: 3)\n", - "1592969997.358247: Worker 0: training step 5 done (global step: 4)\n", - "1592969997.365204: Worker 0: training step 6 done (global step: 5)\n", - "1592969997.376976: Worker 0: training step 7 done (global step: 6)\n", - "1592969997.383788: Worker 0: training step 8 done (global step: 7)\n", - "1592969997.389909: Worker 0: training step 9 done (global step: 8)\n", - "1592969997.399034: Worker 0: training step 10 done (global step: 9)\n", - "1592969997.406169: Worker 0: training step 11 done (global step: 10)\n", - "1592969997.413243: Worker 0: training step 12 done (global step: 11)\n", - "1592969997.419582: Worker 0: training step 13 done (global step: 12)\n", - "1592969997.426087: Worker 0: training step 14 done (global step: 13)\n", - "1592969997.432481: Worker 0: training step 15 done (global step: 14)\n", - "1592969997.438895: Worker 0: training step 16 done (global step: 15)\n", - "1592969997.445008: Worker 0: training step 17 done (global step: 16)\n", - "1592969997.451046: Worker 0: training step 18 done (global step: 17)\n", - "1592969997.458387: Worker 0: training step 19 done (global step: 18)\n", - "1592969997.464300: Worker 0: training step 20 done (global step: 19)\n", - "1592969997.470169: Worker 0: training step 21 done (global step: 20)\n", - "1592969997.492154: Worker 0: training step 22 done (global step: 21)\n", - "1592969997.500725: Worker 0: training step 23 done (global step: 22)\n", - "1592969997.510641: Worker 0: training step 24 done (global step: 23)\n", - "1592969997.519666: Worker 0: training step 25 done (global step: 24)\n", - "1592969997.527392: Worker 0: training step 26 done (global step: 25)\n", - "1592969997.535852: Worker 0: training step 27 done (global step: 26)\n", - "1592969997.544154: Worker 0: training step 28 done (global step: 27)\n", - "1592969997.550987: Worker 0: training step 29 done (global step: 28)\n", - "1592969997.558344: Worker 0: training step 30 done (global step: 29)\n", - "1592969997.564822: Worker 0: training step 31 done (global step: 30)\n", - "1592969997.571622: Worker 0: training step 32 done (global step: 31)\n", - "1592969997.578554: Worker 0: training step 33 done (global step: 32)\n", - "1592969997.595638: Worker 0: training step 34 done (global step: 33)\n", - "1592969997.603068: Worker 0: training step 35 done (global step: 34)\n", - "1592969997.611962: Worker 0: training step 36 done (global step: 35)\n", - "1592969997.618786: Worker 0: training step 37 done (global step: 36)\n", - "1592969997.625508: Worker 0: training step 38 done (global step: 37)\n", - "1592969997.634181: Worker 0: training step 39 done (global step: 38)\n", - "1592969997.642113: Worker 0: training step 40 done (global step: 39)\n", - "1592969997.649647: Worker 0: training step 41 done (global step: 40)\n", - "1592969997.656734: Worker 0: training step 42 done (global step: 41)\n", - "1592969997.665110: Worker 0: training step 43 done (global step: 42)\n", - "1592969997.673620: Worker 0: training step 44 done (global step: 43)\n", - "1592969997.693670: Worker 0: training step 45 done (global step: 44)\n", - "1592969997.700257: Worker 0: training step 46 done (global step: 45)\n", - "1592969997.705834: Worker 0: training step 47 done (global step: 46)\n", - "1592969997.714062: Worker 0: training step 48 done (global step: 47)\n", - "1592969997.720700: Worker 0: training step 49 done (global step: 48)\n", - "1592969997.746550: Worker 0: training step 50 done (global step: 49)\n", - "1592969997.755566: Worker 0: training step 51 done (global step: 50)\n", - "1592969997.768644: Worker 0: training step 52 done (global step: 51)\n", - "1592969997.775591: Worker 0: training step 53 done (global step: 52)\n", - "1592969997.782266: Worker 0: training step 54 done (global step: 53)\n", - "1592969997.789567: Worker 0: training step 55 done (global step: 54)\n", - "1592969997.796607: Worker 0: training step 56 done (global step: 55)\n", - "1592969997.804746: Worker 0: training step 57 done (global step: 56)\n", - "1592969997.811790: Worker 0: training step 58 done (global step: 57)\n", - "1592969997.820524: Worker 0: training step 59 done (global step: 58)\n", - "1592969997.828779: Worker 0: training step 60 done (global step: 59)\n", - "1592969997.837011: Worker 0: training step 61 done (global step: 60)\n", - "1592969997.844103: Worker 0: training step 62 done (global step: 61)\n", - "1592969997.850421: Worker 0: training step 63 done (global step: 62)\n", - "1592969997.857403: Worker 0: training step 64 done (global step: 63)\n", - "1592969997.863736: Worker 0: training step 65 done (global step: 64)\n", - "1592969997.893540: Worker 0: training step 66 done (global step: 65)\n", - "1592969997.901177: Worker 0: training step 67 done (global step: 66)\n", - "1592969997.907805: Worker 0: training step 68 done (global step: 67)\n", - "1592969997.916197: Worker 0: training step 69 done (global step: 68)\n", - "1592969997.924106: Worker 0: training step 70 done (global step: 69)\n", - "1592969997.946289: Worker 0: training step 71 done (global step: 70)\n", - "1592969997.953352: Worker 0: training step 72 done (global step: 71)\n", - "1592969997.959779: Worker 0: training step 73 done (global step: 72)\n", - "1592969997.966829: Worker 0: training step 74 done (global step: 73)\n", - "1592969997.975579: Worker 0: training step 75 done (global step: 74)\n", - "1592969997.981944: Worker 0: training step 76 done (global step: 75)\n", - "1592969997.992360: Worker 0: training step 77 done (global step: 76)\n", - "1592969997.998984: Worker 0: training step 78 done (global step: 77)\n", - "1592969998.005780: Worker 0: training step 79 done (global step: 78)\n", - "1592969998.019416: Worker 0: training step 80 done (global step: 79)\n", - "1592969998.026951: Worker 0: training step 81 done (global step: 80)\n", - "1592969998.033177: Worker 0: training step 82 done (global step: 81)\n", - "1592969998.040482: Worker 0: training step 83 done (global step: 82)\n", - "1592969998.047058: Worker 0: training step 84 done (global step: 83)\n", - "1592969998.053640: Worker 0: training step 85 done (global step: 84)\n", - "1592969998.060095: Worker 0: training step 86 done (global step: 85)\n", - "1592969998.066217: Worker 0: training step 87 done (global step: 86)\n", - "1592969998.071884: Worker 0: training step 88 done (global step: 87)\n", - "1592969998.078604: Worker 0: training step 89 done (global step: 88)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "1592969998.099135: Worker 0: training step 90 done (global step: 89)\n", - "1592969998.105798: Worker 0: training step 91 done (global step: 90)\n", - "1592969998.112137: Worker 0: training step 92 done (global step: 91)\n", - "1592969998.118540: Worker 0: training step 93 done (global step: 92)\n", - "1592969998.125359: Worker 0: training step 94 done (global step: 93)\n", - "1592969998.131841: Worker 0: training step 95 done (global step: 94)\n", - "1592969998.137258: Worker 0: training step 96 done (global step: 95)\n", - "1592969998.143857: Worker 0: training step 97 done (global step: 96)\n", - "1592969998.150290: Worker 0: training step 98 done (global step: 97)\n", - "1592969998.158311: Worker 0: training step 99 done (global step: 98)\n", - "1592969998.164789: Worker 0: training step 100 done (global step: 99)\n", - "1592969998.172325: Worker 0: training step 101 done (global step: 100)\n", - "Training ends @ 1592969998.172426\n", - "Training elapsed time: 1.634471 s\n", - "After 100 training step(s), validation cross entropy = 1084.43\n" + "Training begins @ 1601024002.812838\n", + "1601024003.476335: Worker 0: training step 1 done (global step: 0)\n", + "1601024004.576477: Worker 0: training step 2 done (global step: 1)\n", + "1601024005.576241: Worker 0: training step 3 done (global step: 2)\n", + "1601024006.076146: Worker 0: training step 4 done (global step: 3)\n", + "1601024007.176262: Worker 0: training step 5 done (global step: 4)\n", + "1601024007.475472: Worker 0: training step 6 done (global step: 5)\n", + "1601024007.582544: Worker 0: training step 7 done (global step: 6)\n", + "1601024007.681255: Worker 0: training step 8 done (global step: 7)\n", + "1601024007.788009: Worker 0: training step 9 done (global step: 8)\n", + "1601024007.881521: Worker 0: training step 10 done (global step: 9)\n", + "1601024008.080983: Worker 0: training step 11 done (global step: 10)\n", + "1601024008.183470: Worker 0: training step 12 done (global step: 11)\n", + "1601024008.579565: Worker 0: training step 13 done (global step: 12)\n", + "1601024008.681914: Worker 0: training step 14 done (global step: 13)\n", + "1601024009.079380: Worker 0: training step 15 done (global step: 14)\n", + "1601024009.678963: Worker 0: training step 16 done (global step: 15)\n", + "1601024010.179325: Worker 0: training step 17 done (global step: 16)\n", + "1601024010.278798: Worker 0: training step 18 done (global step: 17)\n", + "1601024010.481322: Worker 0: training step 19 done (global step: 18)\n", + "1601024011.476729: Worker 0: training step 20 done (global step: 19)\n", + "1601024011.976693: Worker 0: training step 21 done (global step: 20)\n", + "1601024012.376771: Worker 0: training step 22 done (global step: 21)\n", + "1601024012.679685: Worker 0: training step 23 done (global step: 22)\n", + "1601024012.690023: Worker 0: training step 24 done (global step: 23)\n", + "1601024013.385506: Worker 0: training step 25 done (global step: 24)\n", + "1601024013.679623: Worker 0: training step 26 done (global step: 25)\n", + "1601024013.880934: Worker 0: training step 27 done (global step: 26)\n", + "1601024014.179751: Worker 0: training step 28 done (global step: 27)\n", + "1601024014.379176: Worker 0: training step 29 done (global step: 28)\n", + "1601024014.580685: Worker 0: training step 30 done (global step: 29)\n", + "1601024014.876878: Worker 0: training step 31 done (global step: 30)\n", + "1601024014.980178: Worker 0: training step 32 done (global step: 31)\n", + "1601024015.778837: Worker 0: training step 33 done (global step: 32)\n", + "1601024015.876221: Worker 0: training step 34 done (global step: 33)\n", + "1601024016.176451: Worker 0: training step 35 done (global step: 34)\n", + "1601024016.579983: Worker 0: training step 36 done (global step: 35)\n", + "1601024016.679807: Worker 0: training step 37 done (global step: 36)\n", + "1601024016.780042: Worker 0: training step 38 done (global step: 37)\n", + "1601024016.879108: Worker 0: training step 39 done (global step: 38)\n", + "1601024017.579612: Worker 0: training step 40 done (global step: 39)\n", + "1601024017.985924: Worker 0: training step 41 done (global step: 40)\n", + "1601024018.375762: Worker 0: training step 42 done (global step: 41)\n", + "1601024018.582078: Worker 0: training step 43 done (global step: 42)\n", + "1601024018.679796: Worker 0: training step 44 done (global step: 43)\n", + "1601024018.779481: Worker 0: training step 45 done (global step: 44)\n", + "1601024018.876627: Worker 0: training step 46 done (global step: 45)\n", + "1601024018.889815: Worker 0: training step 47 done (global step: 46)\n", + "1601024019.280158: Worker 0: training step 48 done (global step: 47)\n", + "1601024019.289733: Worker 0: training step 49 done (global step: 48)\n", + "1601024019.779576: Worker 0: training step 50 done (global step: 49)\n", + "1601024020.182737: Worker 0: training step 51 done (global step: 50)\n", + "1601024020.381563: Worker 0: training step 52 done (global step: 51)\n", + "1601024020.780347: Worker 0: training step 53 done (global step: 52)\n", + "1601024021.279405: Worker 0: training step 54 done (global step: 53)\n", + "1601024021.376435: Worker 0: training step 55 done (global step: 54)\n", + "1601024021.479132: Worker 0: training step 56 done (global step: 55)\n", + "1601024021.495602: Worker 0: training step 57 done (global step: 56)\n", + "1601024021.777862: Worker 0: training step 58 done (global step: 57)\n", + "1601024022.376070: Worker 0: training step 59 done (global step: 58)\n", + "1601024022.679886: Worker 0: training step 60 done (global step: 59)\n", + "1601024022.779463: Worker 0: training step 61 done (global step: 60)\n", + "1601024022.982541: Worker 0: training step 62 done (global step: 61)\n", + "1601024023.577791: Worker 0: training step 63 done (global step: 62)\n", + "1601024023.983038: Worker 0: training step 64 done (global step: 63)\n", + "1601024024.577057: Worker 0: training step 65 done (global step: 64)\n", + "1601024024.684178: Worker 0: training step 66 done (global step: 65)\n", + "1601024024.779040: Worker 0: training step 67 done (global step: 66)\n", + "1601024024.879844: Worker 0: training step 68 done (global step: 67)\n", + "1601024025.077337: Worker 0: training step 69 done (global step: 68)\n", + "1601024025.381255: Worker 0: training step 70 done (global step: 69)\n", + "1601024025.488024: Worker 0: training step 71 done (global step: 70)\n", + "1601024025.776012: Worker 0: training step 72 done (global step: 71)\n", + "1601024026.479723: Worker 0: training step 73 done (global step: 72)\n", + "1601024026.884347: Worker 0: training step 74 done (global step: 73)\n", + "1601024027.175786: Worker 0: training step 75 done (global step: 74)\n", + "1601024027.380380: Worker 0: training step 76 done (global step: 75)\n", + "1601024027.389443: Worker 0: training step 77 done (global step: 76)\n", + "1601024027.481207: Worker 0: training step 78 done (global step: 77)\n", + "1601024027.982406: Worker 0: training step 79 done (global step: 78)\n", + "1601024028.081975: Worker 0: training step 80 done (global step: 79)\n", + "1601024028.577323: Worker 0: training step 81 done (global step: 80)\n", + "1601024028.876491: Worker 0: training step 82 done (global step: 81)\n", + "1601024028.978793: Worker 0: training step 83 done (global step: 82)\n", + "1601024029.076089: Worker 0: training step 84 done (global step: 83)\n", + "1601024029.086477: Worker 0: training step 85 done (global step: 84)\n", + "1601024029.385806: Worker 0: training step 86 done (global step: 85)\n", + "1601024029.479636: Worker 0: training step 87 done (global step: 86)\n", + "1601024029.582871: Worker 0: training step 88 done (global step: 87)\n", + "1601024029.681764: Worker 0: training step 89 done (global step: 88)\n", + "1601024029.776664: Worker 0: training step 90 done (global step: 89)\n", + "1601024029.790441: Worker 0: training step 91 done (global step: 90)\n", + "1601024029.883305: Worker 0: training step 92 done (global step: 91)\n", + "1601024030.083825: Worker 0: training step 93 done (global step: 92)\n", + "1601024030.278688: Worker 0: training step 94 done (global step: 93)\n", + "1601024030.378878: Worker 0: training step 95 done (global step: 94)\n", + "1601024030.479512: Worker 0: training step 96 done (global step: 95)\n", + "1601024030.579052: Worker 0: training step 97 done (global step: 96)\n", + "1601024030.981217: Worker 0: training step 98 done (global step: 97)\n", + "1601024031.175745: Worker 0: training step 99 done (global step: 98)\n", + "1601024031.479927: Worker 0: training step 100 done (global step: 99)\n", + "1601024031.882804: Worker 0: training step 101 done (global step: 100)\n", + "Training ends @ 1601024031.883017\n", + "Training elapsed time: 29.070179 s\n", + "After 100 training step(s), validation cross entropy = 1220.89\n" ] } ], @@ -475,48 +383,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": { "pycharm": { - "is_executing": false, "name": "#%%\n" } }, "outputs": [ { "data": { - "text/plain": [ - "{'experimentId': 'experiment_1592969710478_0001',\n", - " 'name': 'mnist-dist',\n", - " 'uid': '360886c6-b5cc-11ea-b5f2-025000000001',\n", - " 'status': 'Deleted',\n", - " 'acceptedTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'createdTime': '2020-06-24T11:38:47.000+08:00',\n", - " 'runningTime': '2020-06-24T11:38:49.000+08:00',\n", - " 'finishedTime': '2020-06-24T11:40:00.000+08:00',\n", - " 'spec': {'meta': {'name': 'mnist-dist',\n", - " 'namespace': 'default',\n", - " 'framework': 'Tensorflow',\n", - " 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", - " 'envVars': {'ENV1': 'ENV1'}},\n", - " 'environment': {'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n", - " 'spec': {'Ps': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n", - " 'Worker': {'replicas': 1,\n", - " 'resources': 'cpu=1,memory=1024M',\n", - " 'name': None,\n", - " 'image': None,\n", - " 'cmd': None,\n", - " 'envVars': None,\n", - " 'resourceMap': {'memory': '1024M', 'cpu': '1'}}}}}" - ] + "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Deleted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': '2020-09-25T16:53:19.000+08:00',\n 'finishedTime': '2020-09-25T16:53:54.000+08:00',\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -550,17 +428,8 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.7" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } } }, "nbformat": 4, "nbformat_minor": 1 -} +} \ No newline at end of file diff --git a/submarine-sdk/pysubmarine/submarine/experiment/__init__.py b/submarine-sdk/pysubmarine/submarine/experiment/__init__.py index 54566fae8f..7335ab0b90 100644 --- a/submarine-sdk/pysubmarine/submarine/experiment/__init__.py +++ b/submarine-sdk/pysubmarine/submarine/experiment/__init__.py @@ -37,10 +37,12 @@ ApiTypeError, ApiValueError, OpenApiException) # import models into sdk package -from submarine.experiment.models.environment import Environment +from submarine.experiment.models.code_spec import CodeSpec +from submarine.experiment.models.environment_spec import EnvironmentSpec from submarine.experiment.models.experiment_meta import ExperimentMeta from submarine.experiment.models.experiment_spec import ExperimentSpec from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec from submarine.experiment.models.json_response import JsonResponse +from submarine.experiment.models.kernel_spec import KernelSpec __version__ = "0.5.0-SNAPSHOT" diff --git a/submarine-sdk/pysubmarine/submarine/experiment/models/__init__.py b/submarine-sdk/pysubmarine/submarine/experiment/models/__init__.py index 642d63ffba..83e02aed96 100644 --- a/submarine-sdk/pysubmarine/submarine/experiment/models/__init__.py +++ b/submarine-sdk/pysubmarine/submarine/experiment/models/__init__.py @@ -30,8 +30,10 @@ from __future__ import absolute_import # import models into model package -from submarine.experiment.models.environment import Environment +from submarine.experiment.models.code_spec import CodeSpec +from submarine.experiment.models.environment_spec import EnvironmentSpec from submarine.experiment.models.experiment_meta import ExperimentMeta from submarine.experiment.models.experiment_spec import ExperimentSpec from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec from submarine.experiment.models.json_response import JsonResponse +from submarine.experiment.models.kernel_spec import KernelSpec diff --git a/submarine-sdk/pysubmarine/submarine/experiment/models/environment.py b/submarine-sdk/pysubmarine/submarine/experiment/models/code_spec.py similarity index 72% rename from submarine-sdk/pysubmarine/submarine/experiment/models/environment.py rename to submarine-sdk/pysubmarine/submarine/experiment/models/code_spec.py index 1f00e78cfe..33dc6976d8 100644 --- a/submarine-sdk/pysubmarine/submarine/experiment/models/environment.py +++ b/submarine-sdk/pysubmarine/submarine/experiment/models/code_spec.py @@ -34,7 +34,7 @@ from submarine.experiment.configuration import Configuration -class Environment(object): +class CodeSpec(object): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech @@ -49,45 +49,71 @@ class Environment(object): and the value is json key in definition. """ openapi_types = { - 'image': 'str' + 'sync_mode': 'str', + 'url': 'str' } attribute_map = { - 'image': 'image' + 'sync_mode': 'syncMode', + 'url': 'url' } - def __init__(self, image=None, local_vars_configuration=None): # noqa: E501 - """Environment - a model defined in OpenAPI""" # noqa: E501 + def __init__(self, sync_mode=None, url=None, local_vars_configuration=None): # noqa: E501 + """CodeSpec - a model defined in OpenAPI""" # noqa: E501 if local_vars_configuration is None: local_vars_configuration = Configuration() self.local_vars_configuration = local_vars_configuration - self._image = None + self._sync_mode = None + self._url = None self.discriminator = None - if image is not None: - self.image = image + if sync_mode is not None: + self.sync_mode = sync_mode + if url is not None: + self.url = url @property - def image(self): - """Gets the image of this Environment. # noqa: E501 + def sync_mode(self): + """Gets the sync_mode of this CodeSpec. # noqa: E501 - :return: The image of this Environment. # noqa: E501 + :return: The sync_mode of this CodeSpec. # noqa: E501 :rtype: str """ - return self._image + return self._sync_mode - @image.setter - def image(self, image): - """Sets the image of this Environment. + @sync_mode.setter + def sync_mode(self, sync_mode): + """Sets the sync_mode of this CodeSpec. - :param image: The image of this Environment. # noqa: E501 + :param sync_mode: The sync_mode of this CodeSpec. # noqa: E501 :type: str """ - self._image = image + self._sync_mode = sync_mode + + @property + def url(self): + """Gets the url of this CodeSpec. # noqa: E501 + + + :return: The url of this CodeSpec. # noqa: E501 + :rtype: str + """ + return self._url + + @url.setter + def url(self, url): + """Sets the url of this CodeSpec. + + + :param url: The url of this CodeSpec. # noqa: E501 + :type: str + """ + + self._url = url def to_dict(self): """Returns the model properties as a dict""" @@ -123,14 +149,14 @@ def __repr__(self): def __eq__(self, other): """Returns true if both objects are equal""" - if not isinstance(other, Environment): + if not isinstance(other, CodeSpec): return False return self.to_dict() == other.to_dict() def __ne__(self, other): """Returns true if both objects are not equal""" - if not isinstance(other, Environment): + if not isinstance(other, CodeSpec): return True return self.to_dict() != other.to_dict() diff --git a/submarine-sdk/pysubmarine/submarine/experiment/models/environment_spec.py b/submarine-sdk/pysubmarine/submarine/experiment/models/environment_spec.py new file mode 100644 index 0000000000..829f3601c6 --- /dev/null +++ b/submarine-sdk/pysubmarine/submarine/experiment/models/environment_spec.py @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# coding: utf-8 + +""" + Submarine Experiment API + + The Submarine REST API allows you to create, list, and get experiments. The API is hosted under the /v1/experiment route on the Submarine server. For example, to list experiments on a server hosted at http://localhost:8080, access http://localhost:8080/api/v1/experiment/ # noqa: E501 + + The version of the OpenAPI document: 0.5.0-SNAPSHOT + Contact: dev@submarine.apache.org + Generated by: https://openapi-generator.tech +""" + + +import pprint +import re # noqa: F401 + +import six + +from submarine.experiment.configuration import Configuration + + +class EnvironmentSpec(object): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + """ + Attributes: + openapi_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + openapi_types = { + 'name': 'str', + 'docker_image': 'str', + 'kernel_spec': 'KernelSpec', + 'description': 'str', + 'image': 'str' + } + + attribute_map = { + 'name': 'name', + 'docker_image': 'dockerImage', + 'kernel_spec': 'kernelSpec', + 'description': 'description', + 'image': 'image' + } + + def __init__(self, name=None, docker_image=None, kernel_spec=None, description=None, image=None, local_vars_configuration=None): # noqa: E501 + """EnvironmentSpec - a model defined in OpenAPI""" # noqa: E501 + if local_vars_configuration is None: + local_vars_configuration = Configuration() + self.local_vars_configuration = local_vars_configuration + + self._name = None + self._docker_image = None + self._kernel_spec = None + self._description = None + self._image = None + self.discriminator = None + + if name is not None: + self.name = name + if docker_image is not None: + self.docker_image = docker_image + if kernel_spec is not None: + self.kernel_spec = kernel_spec + if description is not None: + self.description = description + if image is not None: + self.image = image + + @property + def name(self): + """Gets the name of this EnvironmentSpec. # noqa: E501 + + + :return: The name of this EnvironmentSpec. # noqa: E501 + :rtype: str + """ + return self._name + + @name.setter + def name(self, name): + """Sets the name of this EnvironmentSpec. + + + :param name: The name of this EnvironmentSpec. # noqa: E501 + :type: str + """ + + self._name = name + + @property + def docker_image(self): + """Gets the docker_image of this EnvironmentSpec. # noqa: E501 + + + :return: The docker_image of this EnvironmentSpec. # noqa: E501 + :rtype: str + """ + return self._docker_image + + @docker_image.setter + def docker_image(self, docker_image): + """Sets the docker_image of this EnvironmentSpec. + + + :param docker_image: The docker_image of this EnvironmentSpec. # noqa: E501 + :type: str + """ + + self._docker_image = docker_image + + @property + def kernel_spec(self): + """Gets the kernel_spec of this EnvironmentSpec. # noqa: E501 + + + :return: The kernel_spec of this EnvironmentSpec. # noqa: E501 + :rtype: KernelSpec + """ + return self._kernel_spec + + @kernel_spec.setter + def kernel_spec(self, kernel_spec): + """Sets the kernel_spec of this EnvironmentSpec. + + + :param kernel_spec: The kernel_spec of this EnvironmentSpec. # noqa: E501 + :type: KernelSpec + """ + + self._kernel_spec = kernel_spec + + @property + def description(self): + """Gets the description of this EnvironmentSpec. # noqa: E501 + + + :return: The description of this EnvironmentSpec. # noqa: E501 + :rtype: str + """ + return self._description + + @description.setter + def description(self, description): + """Sets the description of this EnvironmentSpec. + + + :param description: The description of this EnvironmentSpec. # noqa: E501 + :type: str + """ + + self._description = description + + @property + def image(self): + """Gets the image of this EnvironmentSpec. # noqa: E501 + + + :return: The image of this EnvironmentSpec. # noqa: E501 + :rtype: str + """ + return self._image + + @image.setter + def image(self, image): + """Sets the image of this EnvironmentSpec. + + + :param image: The image of this EnvironmentSpec. # noqa: E501 + :type: str + """ + + self._image = image + + def to_dict(self): + """Returns the model properties as a dict""" + result = {} + + for attr, _ in six.iteritems(self.openapi_types): + value = getattr(self, attr) + if isinstance(value, list): + result[attr] = list(map( + lambda x: x.to_dict() if hasattr(x, "to_dict") else x, + value + )) + elif hasattr(value, "to_dict"): + result[attr] = value.to_dict() + elif isinstance(value, dict): + result[attr] = dict(map( + lambda item: (item[0], item[1].to_dict()) + if hasattr(item[1], "to_dict") else item, + value.items() + )) + else: + result[attr] = value + + return result + + def to_str(self): + """Returns the string representation of the model""" + return pprint.pformat(self.to_dict()) + + def __repr__(self): + """For `print` and `pprint`""" + return self.to_str() + + def __eq__(self, other): + """Returns true if both objects are equal""" + if not isinstance(other, EnvironmentSpec): + return False + + return self.to_dict() == other.to_dict() + + def __ne__(self, other): + """Returns true if both objects are not equal""" + if not isinstance(other, EnvironmentSpec): + return True + + return self.to_dict() != other.to_dict() diff --git a/submarine-sdk/pysubmarine/submarine/experiment/models/experiment_spec.py b/submarine-sdk/pysubmarine/submarine/experiment/models/experiment_spec.py index 921a159a67..1e47e77e2f 100644 --- a/submarine-sdk/pysubmarine/submarine/experiment/models/experiment_spec.py +++ b/submarine-sdk/pysubmarine/submarine/experiment/models/experiment_spec.py @@ -50,17 +50,19 @@ class ExperimentSpec(object): """ openapi_types = { 'meta': 'ExperimentMeta', - 'environment': 'Environment', - 'spec': 'dict(str, ExperimentTaskSpec)' + 'environment': 'EnvironmentSpec', + 'spec': 'dict(str, ExperimentTaskSpec)', + 'code': 'CodeSpec' } attribute_map = { 'meta': 'meta', 'environment': 'environment', - 'spec': 'spec' + 'spec': 'spec', + 'code': 'code' } - def __init__(self, meta=None, environment=None, spec=None, local_vars_configuration=None): # noqa: E501 + def __init__(self, meta=None, environment=None, spec=None, code=None, local_vars_configuration=None): # noqa: E501 """ExperimentSpec - a model defined in OpenAPI""" # noqa: E501 if local_vars_configuration is None: local_vars_configuration = Configuration() @@ -69,6 +71,7 @@ def __init__(self, meta=None, environment=None, spec=None, local_vars_configurat self._meta = None self._environment = None self._spec = None + self._code = None self.discriminator = None if meta is not None: @@ -77,6 +80,8 @@ def __init__(self, meta=None, environment=None, spec=None, local_vars_configurat self.environment = environment if spec is not None: self.spec = spec + if code is not None: + self.code = code @property def meta(self): @@ -105,7 +110,7 @@ def environment(self): :return: The environment of this ExperimentSpec. # noqa: E501 - :rtype: Environment + :rtype: EnvironmentSpec """ return self._environment @@ -115,7 +120,7 @@ def environment(self, environment): :param environment: The environment of this ExperimentSpec. # noqa: E501 - :type: Environment + :type: EnvironmentSpec """ self._environment = environment @@ -141,6 +146,27 @@ def spec(self, spec): self._spec = spec + @property + def code(self): + """Gets the code of this ExperimentSpec. # noqa: E501 + + + :return: The code of this ExperimentSpec. # noqa: E501 + :rtype: CodeSpec + """ + return self._code + + @code.setter + def code(self, code): + """Sets the code of this ExperimentSpec. + + + :param code: The code of this ExperimentSpec. # noqa: E501 + :type: CodeSpec + """ + + self._code = code + def to_dict(self): """Returns the model properties as a dict""" result = {} diff --git a/submarine-sdk/pysubmarine/submarine/experiment/models/kernel_spec.py b/submarine-sdk/pysubmarine/submarine/experiment/models/kernel_spec.py new file mode 100644 index 0000000000..e8f3e30152 --- /dev/null +++ b/submarine-sdk/pysubmarine/submarine/experiment/models/kernel_spec.py @@ -0,0 +1,188 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# coding: utf-8 + +""" + Submarine Experiment API + + The Submarine REST API allows you to create, list, and get experiments. The API is hosted under the /v1/experiment route on the Submarine server. For example, to list experiments on a server hosted at http://localhost:8080, access http://localhost:8080/api/v1/experiment/ # noqa: E501 + + The version of the OpenAPI document: 0.5.0-SNAPSHOT + Contact: dev@submarine.apache.org + Generated by: https://openapi-generator.tech +""" + + +import pprint +import re # noqa: F401 + +import six + +from submarine.experiment.configuration import Configuration + + +class KernelSpec(object): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + """ + Attributes: + openapi_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + openapi_types = { + 'name': 'str', + 'channels': 'list[str]', + 'dependencies': 'list[str]' + } + + attribute_map = { + 'name': 'name', + 'channels': 'channels', + 'dependencies': 'dependencies' + } + + def __init__(self, name=None, channels=None, dependencies=None, local_vars_configuration=None): # noqa: E501 + """KernelSpec - a model defined in OpenAPI""" # noqa: E501 + if local_vars_configuration is None: + local_vars_configuration = Configuration() + self.local_vars_configuration = local_vars_configuration + + self._name = None + self._channels = None + self._dependencies = None + self.discriminator = None + + if name is not None: + self.name = name + if channels is not None: + self.channels = channels + if dependencies is not None: + self.dependencies = dependencies + + @property + def name(self): + """Gets the name of this KernelSpec. # noqa: E501 + + + :return: The name of this KernelSpec. # noqa: E501 + :rtype: str + """ + return self._name + + @name.setter + def name(self, name): + """Sets the name of this KernelSpec. + + + :param name: The name of this KernelSpec. # noqa: E501 + :type: str + """ + + self._name = name + + @property + def channels(self): + """Gets the channels of this KernelSpec. # noqa: E501 + + + :return: The channels of this KernelSpec. # noqa: E501 + :rtype: list[str] + """ + return self._channels + + @channels.setter + def channels(self, channels): + """Sets the channels of this KernelSpec. + + + :param channels: The channels of this KernelSpec. # noqa: E501 + :type: list[str] + """ + + self._channels = channels + + @property + def dependencies(self): + """Gets the dependencies of this KernelSpec. # noqa: E501 + + + :return: The dependencies of this KernelSpec. # noqa: E501 + :rtype: list[str] + """ + return self._dependencies + + @dependencies.setter + def dependencies(self, dependencies): + """Sets the dependencies of this KernelSpec. + + + :param dependencies: The dependencies of this KernelSpec. # noqa: E501 + :type: list[str] + """ + + self._dependencies = dependencies + + def to_dict(self): + """Returns the model properties as a dict""" + result = {} + + for attr, _ in six.iteritems(self.openapi_types): + value = getattr(self, attr) + if isinstance(value, list): + result[attr] = list(map( + lambda x: x.to_dict() if hasattr(x, "to_dict") else x, + value + )) + elif hasattr(value, "to_dict"): + result[attr] = value.to_dict() + elif isinstance(value, dict): + result[attr] = dict(map( + lambda item: (item[0], item[1].to_dict()) + if hasattr(item[1], "to_dict") else item, + value.items() + )) + else: + result[attr] = value + + return result + + def to_str(self): + """Returns the string representation of the model""" + return pprint.pformat(self.to_dict()) + + def __repr__(self): + """For `print` and `pprint`""" + return self.to_str() + + def __eq__(self, other): + """Returns true if both objects are equal""" + if not isinstance(other, KernelSpec): + return False + + return self.to_dict() == other.to_dict() + + def __ne__(self, other): + """Returns true if both objects are not equal""" + if not isinstance(other, KernelSpec): + return True + + return self.to_dict() != other.to_dict() diff --git a/submarine-sdk/pysubmarine/tests/experiment/test_experiment_client.py b/submarine-sdk/pysubmarine/tests/experiment/test_experiment_client.py index d94e74122e..0e7d37d024 100644 --- a/submarine-sdk/pysubmarine/tests/experiment/test_experiment_client.py +++ b/submarine-sdk/pysubmarine/tests/experiment/test_experiment_client.py @@ -16,7 +16,8 @@ import pytest import submarine -from submarine.experiment.models.environment import Environment +from submarine.experiment.models.code_spec import CodeSpec +from submarine.experiment.models.environment_spec import EnvironmentSpec from submarine.experiment.models.experiment_meta import ExperimentMeta from submarine.experiment.models.experiment_spec import ExperimentSpec from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec @@ -25,7 +26,8 @@ @pytest.mark.e2e def test_experiment_e2e(): submarine_client = submarine.ExperimentClient(host='http://localhost:8080') - environment = Environment(image='gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0') + environment = EnvironmentSpec( + image='gcr.io/kubeflow-ci/tf-dist-mnist-test:1.0') experiment_meta = ExperimentMeta( name='mnist-dist', namespace='default', @@ -36,8 +38,12 @@ def test_experiment_e2e(): worker_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M', replicas=1) ps_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M', replicas=1) + code_spec = CodeSpec(sync_mode='git', + url='https://github.com/apache/submarine.git') + experiment_spec = ExperimentSpec(meta=experiment_meta, environment=environment, + code=code_spec, spec={ 'Ps': ps_spec, 'Worker': worker_spec