Merge upstream

* Open agent helm * Update cli deploy * Fix operator
polyaxon · Nov 15, 2019 · ee7cb20 · ee7cb20
1 parent 5cf5b7f
commit ee7cb20
Show file tree

Hide file tree

Showing 39 changed files with 17,747 additions and 533 deletions.
diff --git a/.gitignore b/.gitignore
@@ -154,8 +154,8 @@ bin
 !vendor/**/zz_generated.*
 client/.awcache/
 
-# ignore devvalues
-values.dev.yaml
+# ignore dev/test values
+values.*.dev.yaml
 values.*.test.yaml
 values.*.test.json
 values.*.test.sh

diff --git a/cli/polyaxon/deploy/operators/compose.py b/cli/polyaxon/deploy/operators/compose.py
@@ -137,32 +137,7 @@ def _generate_intervals(config):
         if config.intervals:
             env.append(
                 template.format(
-                    "POLYAXON_INTERVALS_EXPERIMENTS_SCHEDULER",
-                    config.intervals.experimentsScheduler,
-                )
-            )
-            env.append(
-                template.format(
-                    "POLYAXON_INTERVALS_EXPERIMENTS_SYNC",
-                    config.intervals.experimentsSync,
-                )
-            )
-            env.append(
-                template.format(
-                    "POLYAXON_INTERVALS_CLUSTERS_UPDATE_SYSTEM_INFO",
-                    config.intervals.clustersUpdateSystemInfo,
-                )
-            )
-            env.append(
-                template.format(
-                    "POLYAXON_INTERVALS_CLUSTERS_UPDATE_SYSTEM_NODES",
-                    config.intervals.clustersUpdateSystemNodes,
-                )
-            )
-            env.append(
-                template.format(
-                    "POLYAXON_INTERVALS_PIPELINES_SCHEDULER",
-                    config.intervals.pipelinesScheduler,
+                    "POLYAXON_INTERVALS_RUNS_SCHEDULER", config.intervals.runsScheduler
                 )
             )
             env.append(
@@ -207,8 +182,6 @@ def generate_env(cls, config):
             env.append(
                 template.format("POLYAXON_CHART_VERSION", config.deploymentVersion)
             )
-        if config.clusterId:
-            env.append(template.format("POLYAXON_CLUSTER_ID", config.clusterId))
         if config.namespace:
             env.append(template.format("POLYAXON_K8S_NAMESPACE", config.namespace))
         env.append(

diff --git a/cli/polyaxon/deploy/schemas/deployment.py b/cli/polyaxon/deploy/schemas/deployment.py
@@ -93,7 +93,6 @@ class DeploymentSchema(BaseSchema):
         allow_none=True, validate=validate.OneOf(DeploymentTypes.VALUES)
     )
     deploymentVersion = fields.Str(allow_none=True)
-    clusterId = fields.Str(allow_none=True)
     namespace = fields.Str(allow_none=True)
     rbac = fields.Nested(RBACSchema, allow_none=True)
     polyaxonSecret = fields.Str(allow_none=True)
@@ -119,13 +118,7 @@ class DeploymentSchema(BaseSchema):
     streams = fields.Nested(ApiSchema, allow_none=True)
     scheduler = fields.Nested(WorkerSchema, allow_none=True)
     worker = fields.Nested(WorkerSchema, allow_none=True)
-    hpsearch = fields.Nested(WorkerSchema, allow_none=True)
-    eventsHandlers = fields.Nested(WorkerSchema, allow_none=True)
-    k8sEventsHandlers = fields.Nested(WorkerSchema, allow_none=True)
     beat = fields.Nested(ServiceSchema, allow_none=True)
-    crons = fields.Nested(ServiceSchema, allow_none=True)
-    eventMonitors = fields.Nested(EventMonitorsSchema, allow_none=True)
-    resourcesDaemon = fields.Nested(ThirdPartyServiceSchema, allow_none=True)
     tablesHook = fields.Nested(ServiceSchema, allow_none=True)
     hooks = fields.Nested(HooksSchema, allow_none=True)
     postgresql = fields.Nested(PostgresqlSchema, allow_none=True)
@@ -145,8 +138,6 @@ class DeploymentSchema(BaseSchema):
     reposAccessToken = fields.Str(allow_none=True)
     logLevel = fields.Str(allow_none=True)
     trackerBackend = fields.Str(allow_none=True)
-    dirs = fields.Dict(allow_none=True)
-    mountPaths = fields.Dict(allow_none=True)
     securityContext = fields.Nested(SecurityContextSchema, allow_none=True)
     externalServices = fields.Nested(ExternalServicesSchema, allow_none=True)
     debugMode = fields.Bool(allow_none=True)
@@ -175,7 +166,6 @@ def __init__(
         self,  # noqa
         deploymentType=None,
         deploymentVersion=None,
-        clusterId=None,
         namespace=None,
         rbac=None,
         polyaxonSecret=None,
@@ -200,13 +190,7 @@ def __init__(
         streams=None,
         scheduler=None,
         worker=None,
-        hpsearch=None,
-        eventsHandlers=None,
-        k8sEventsHandlers=None,
         beat=None,
-        crons=None,
-        eventMonitors=None,
-        resourcesDaemon=None,
         tablesHook=None,
         hooks=None,
         postgresql=None,
@@ -224,8 +208,6 @@ def __init__(
         reposAccessToken=None,
         logLevel=None,
         trackerBackend=None,
-        dirs=None,
-        mountPaths=None,
         securityContext=None,
         externalServices=None,
         debugMode=None,
@@ -236,7 +218,6 @@ def __init__(
         check_rabbitmq(rabbitmq, externalServices, broker)
         self.deploymentType = deploymentType
         self.deploymentVersion = deploymentVersion
-        self.clusterId = clusterId
         self.namespace = namespace
         self.rbac = rbac
         self.polyaxonSecret = polyaxonSecret
@@ -261,13 +242,7 @@ def __init__(
         self.streams = streams
         self.scheduler = scheduler
         self.worker = worker
-        self.hpsearch = hpsearch
-        self.eventsHandlers = eventsHandlers
-        self.k8sEventsHandlers = k8sEventsHandlers
         self.beat = beat
-        self.crons = crons
-        self.eventMonitors = eventMonitors
-        self.resourcesDaemon = resourcesDaemon
         self.tablesHook = tablesHook
         self.hooks = hooks
         self.postgresql = postgresql
@@ -285,8 +260,6 @@ def __init__(
         self.reposAccessToken = reposAccessToken
         self.logLevel = logLevel
         self.trackerBackend = trackerBackend
-        self.dirs = dirs
-        self.mountPaths = mountPaths
         self.securityContext = securityContext
         self.externalServices = externalServices
         self.debugMode = debugMode

diff --git a/cli/polyaxon/deploy/schemas/intervals.py b/cli/polyaxon/deploy/schemas/intervals.py
@@ -23,11 +23,7 @@
 
 
 class IntervalsSchema(BaseSchema):
-    experimentsScheduler = fields.Int(default=None)
-    experimentsSync = fields.Int(default=None)
-    clustersUpdateSystemInfo = fields.Int(default=None)
-    clustersUpdateSystemNodes = fields.Int(default=None)
-    pipelinesScheduler = fields.Int(default=None)
+    runsScheduler = fields.Int(default=None)
     operationsDefaultRetryDelay = fields.Int(default=None)
     operationsMaxRetryDelay = fields.Int(default=None)
 
@@ -39,29 +35,17 @@ def schema_config():
 class IntervalsConfig(BaseConfig):
     SCHEMA = IntervalsSchema
     REDUCED_ATTRIBUTES = [
-        "experimentsScheduler",
-        "experimentsSync",
-        "clustersUpdateSystemInfo",
-        "clustersUpdateSystemNodes",
-        "pipelinesScheduler",
+        "runsScheduler",
         "operationsDefaultRetryDelay",
         "operationsMaxRetryDelay",
     ]
 
     def __init__(
         self,  # noqa
-        experimentsScheduler=None,
-        experimentsSync=None,
-        clustersUpdateSystemInfo=None,
-        clustersUpdateSystemNodes=None,
-        pipelinesScheduler=None,
+        runsScheduler=None,
         operationsDefaultRetryDelay=None,
         operationsMaxRetryDelay=None,
     ):
-        self.experimentsScheduler = experimentsScheduler
-        self.experimentsSync = experimentsSync
-        self.clustersUpdateSystemInfo = clustersUpdateSystemInfo
-        self.clustersUpdateSystemNodes = clustersUpdateSystemNodes
-        self.pipelinesScheduler = pipelinesScheduler
+        self.runsScheduler = runsScheduler
         self.operationsDefaultRetryDelay = operationsDefaultRetryDelay
         self.operationsMaxRetryDelay = operationsMaxRetryDelay
diff --git a/cli/polyaxon/deploy/schemas/persistence.py b/cli/polyaxon/deploy/schemas/persistence.py
@@ -105,9 +105,6 @@ def validate_named_persistence(values, persistence):
 
 class PersistenceSchema(BaseSchema):
     logs = fields.Nested(PersistenceEntitySchema, allow_none=True)
-    repos = fields.Nested(PersistenceEntitySchema, allow_none=True)
-    upload = fields.Nested(PersistenceEntitySchema, allow_none=True)
-    data = fields.Dict(allow_none=True)
     outputs = fields.Dict(allow_none=True)
 
     @staticmethod
@@ -122,15 +119,10 @@ def validate_named_persistence(self, data):
 
 class PersistenceConfig(BaseConfig):
     SCHEMA = PersistenceSchema
-    REDUCED_ATTRIBUTES = ["logs", "repos", "upload", "data", "outputs"]
+    REDUCED_ATTRIBUTES = ["logs", "outputs"]
 
-    def __init__(self, logs=None, repos=None, upload=None, data=None, outputs=None):
+    def __init__(self, logs=None, outputs=None):
         self.logs = logs
-        self.repos = repos
-        self.upload = upload
-        if data:
-            validate_named_persistence(data, "data")
-        self.data = data
         if outputs:
             validate_named_persistence(outputs, "outputs")
         self.outputs = outputs
diff --git a/cli/requirements/requirements-base.txt b/cli/requirements/requirements-base.txt
@@ -7,7 +7,7 @@ Jinja2==2.10.3
 marshmallow==3.0.0rc5
 python-dateutil>=2.7.3
 pytz>=2019.2
-rhea==0.5.4
+rhea==0.5.5
 ujson>=1.35
 psutil>=5.4.7
 requests>=2.20.1

diff --git a/cli/setup.py b/cli/setup.py
@@ -73,7 +73,7 @@ def run_tests(self):
         "python-dateutil>=2.7.3",
         "pytz>=2019.2",
         "ujson>=1.35",
-        "rhea==0.5.4",
+        "rhea==0.5.5",
         "ujson>=1.35",
         "psutil>=5.4.7",
         "requests>=2.20.1",