Bump agent version to 1.4.6

charts/agent/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: An enterprise-grade open-source platform for building, training, and monitoring large scale deep learning applications.
 name: agent
-version: 1.4.5
-appVersion: 1.4.5
+version: 1.4.6
+appVersion: 1.4.6
 keywords:
   - deep learning
   - a.i.

charts/agent/values.yaml

@@ -68,7 +68,7 @@ agentSecret:
 gateway:
   enabled: true
   image: polyaxon/polyaxon-gateway
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   replicas: 1
   resources:
@@ -90,7 +90,7 @@ gateway:
 streams:
   enabled: true
   image: polyaxon/polyaxon-streams
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   replicas: 1
   resources:
@@ -117,7 +117,7 @@ streams:
 
 init:
   image: polyaxon/polyaxon-init
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   resources:
     limits:
@@ -129,7 +129,7 @@ init:
 
 sidecar:
   image: polyaxon/polyaxon-sidecar
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   sleepInterval: 10
   syncInterval: 10
@@ -146,7 +146,7 @@ agent:
   instance:
   enabled: true
   image: polyaxon/polyaxon-agent
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   resources:
     limits:
@@ -160,7 +160,7 @@ operator:
   enabled: true
   skipCRD: false
   image: polyaxon/polyaxon-operator
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
   resources:
     limits:
@@ -172,7 +172,7 @@ operator:
 
 cleanHooks:
   image: polyaxon/polyaxon-cli
-  imageTag: 1.4.5
+  imageTag: 1.4.6
   imagePullPolicy: IfNotPresent
 
 securityContext:

config/setup/default.yaml

@@ -1,3 +1,3 @@
 deploymentChart: platform
-deploymentVersion: 1.4.4
+deploymentVersion: 1.4.6
 namespace: test

core/polyaxon/agents/agent.py

@@ -115,6 +115,7 @@ def sync_compatible_updates(self, compatible_updates: Dict):
                     V1ConnectionType.from_dict(c) for c in connections
                 ]
 
+            self.content = settings.AGENT_CONFIG.to_dict(dump=True)
             self.sync()
 
     def log_agent_running(self):

core/polyaxon/agents/base.py

@@ -22,6 +22,7 @@
 
 from kubernetes.client.rest import ApiException
 
+from polyaxon import settings
 from polyaxon.agents import converter
 from polyaxon.agents.spawners.spawner import Spawner
 from polyaxon.client import PolyaxonClient
@@ -42,6 +43,7 @@ def __init__(self, sleep_interval=None):
         self.spawner = Spawner()
         self.client = PolyaxonClient()
         self._graceful_shutdown = False
+        self.content = settings.AGENT_CONFIG.to_dict(dump=True)
 
     def get_info(self) -> polyaxon_sdk.V1Agent:
         raise NotImplementedError
@@ -274,6 +276,7 @@ def prepare_run_resource(
                 run_uuid=run_uuid,
                 content=content,
                 default_auth=True,
+                agent_content=self.content,
             )
         except PolypodException as e:
             self.log_run_failed(
@@ -301,6 +304,8 @@ def _submit_run(self, run_data: Tuple[str, str, str, str], sync_api=True):
             run_uuid=run_uuid,
             content=run_data[3],
         )
+        if not resource:
+            return
 
         try:
             self.spawner.create(
@@ -340,6 +345,8 @@ def create_run_and_sync(self, run_data: Tuple[str, str, str, str]):
             run_uuid=run_uuid,
             content=run_data[3],
         )
+        if not resource:
+            return
 
         try:
             self.spawner.create(
@@ -380,6 +387,8 @@ def make_and_create_run(
             content=run_data[3],
             default_auth=default_auth,
         )
+        if not resource:
+            return
 
         try:
             self.spawner.create(
@@ -406,6 +415,8 @@ def apply_run(self, run_data: Tuple[str, str, str, str]):
             run_uuid=run_uuid,
             content=run_data[3],
         )
+        if not resource:
+            return
 
         try:
             self.spawner.apply(

core/polyaxon/agents/converter.py

@@ -19,6 +19,7 @@
 from polyaxon.polypod.compiler import converter, make
 from polyaxon.polypod.compiler.config import PolypodConfig
 from polyaxon.polypod.compiler.converters import PLATFORM_CONVERTERS
+from polyaxon.schemas.cli.agent_config import AgentConfig
 
 
 def convert(
@@ -28,11 +29,15 @@ def convert(
     run_uuid: str,
     content: str,
     default_auth: bool,
+    agent_content: str = None,
 ) -> Dict:
     polypod_config = PolypodConfig()
     compiled_operation = CompiledOperationSpecification.read(content)
 
-    polypod_config.resolve(compiled_operation=compiled_operation)
+    polypod_config.resolve(
+        compiled_operation=compiled_operation,
+        agent_config=AgentConfig.read(agent_content) if agent_content else None,
+    )
     return converter.convert(
         compiled_operation=compiled_operation,
         owner_name=owner_name,

core/polyaxon/connections/schemas/connections.py

@@ -16,7 +16,7 @@
 
 import polyaxon_sdk
 
-from marshmallow import INCLUDE, ValidationError, fields
+from marshmallow import INCLUDE, ValidationError, fields, post_dump
 
 from polyaxon.connections.kinds import V1ConnectionKind
 from polyaxon.schemas.base import BaseCamelSchema, BaseConfig, BaseOneOfSchema
@@ -130,6 +130,12 @@ class Meta(BaseCamelSchema.Meta):
     def schema_config():
         return V1CustomConnection
 
+    @post_dump(pass_original=True)
+    def unmake_custom(self, data, obj, **kwargs):
+        value = self.schema_config().remove_reduced_attrs(data)
+        value.update({k: getattr(obj, k) for k in obj._schema_keys})
+        return value
+
 
 class V1CustomConnection(BaseConfig):
     UNKNOWN_BEHAVIOUR = INCLUDE
@@ -142,25 +148,6 @@ def __init__(self, **kwargs):
             self._schema_keys.add(k)
             self.__setattr__(k, v)
 
-    @classmethod
-    def obj_to_dict(
-        cls,
-        obj,
-        humanize_values=False,
-        unknown=None,
-        include_kind=False,
-        include_version=False,
-    ):
-        value = super().obj_to_dict(
-            obj=obj,
-            humanize_values=humanize_values,
-            unknown=cls.UNKNOWN_BEHAVIOUR,
-            include_kind=include_kind,
-            include_version=include_version,
-        )
-        value.update({k: getattr(obj, k) for k in obj._schema_keys})
-        return value
-
     @classmethod
     def from_dict(cls, value, unknown=None):
         return super().from_dict(value=value, unknown=cls.UNKNOWN_BEHAVIOUR)
@@ -180,7 +167,7 @@ def patch_git(self, schema: "GitConnectionSchema"):
         if schema.revision:
             if "revision" not in self._schema_keys:
                 self._schema_keys.add("revision")
-            setattr(self, "revision", schema.url)
+            setattr(self, "revision", schema.revision)
 
         if schema.flags:
             if "flags" not in self._schema_keys:

core/polyaxon/init/git.py

@@ -31,10 +31,13 @@
 from polyaxon.exceptions import PolyaxonClientException, PolyaxonContainerException
 from polyaxon.polyboard.artifacts import V1ArtifactKind, V1RunArtifact
 from polyaxon.utils.code_reference import (
+    add_remote,
     checkout_revision,
     get_code_reference,
-    set_remote,
+    git_fetch,
+    git_init,
 )
+from polyaxon.utils.path_utils import check_or_create_path
 
 _logger = logging.getLogger("polyaxon.repos.git")
 
@@ -59,7 +62,7 @@ def get_ssh_cmd():
 def get_clone_url(url: str) -> str:
     if not url:
         raise ValueError(
-            "Git initializer requires a valid url, receveied {}".format(url)
+            "Git initializer requires a valid url, received {}".format(url)
         )
 
     if has_cred_access():
@@ -88,8 +91,6 @@ def get_clone_url(url: str) -> str:
 
 def clone_git_repo(repo_path: str, url: str, flags: List[str] = None) -> str:
     if has_ssh_access():
-        flags = flags or []
-        flags.append("--recurse-submodules")
         return GitRepo.clone_from(
             url=url,
             to_path=repo_path,
@@ -99,6 +100,32 @@ def clone_git_repo(repo_path: str, url: str, flags: List[str] = None) -> str:
     return GitRepo.clone_from(url=url, to_path=repo_path, multi_options=flags)
 
 
+def clone_and_checkout_git_repo(
+    repo_path: str,
+    clone_url: str,
+    revision: str,
+    flags: List[str] = None,
+):
+    clone_git_repo(repo_path=repo_path, url=clone_url, flags=flags)
+    if revision:
+        checkout_revision(repo_path=repo_path, revision=revision)
+
+
+def fetch_git_repo(
+    repo_path: str,
+    clone_url: str,
+    revision: str,
+    flags: List[str] = None,
+):
+    check_or_create_path(repo_path, is_dir=True)
+    git_init(repo_path)
+    add_remote(repo_path, clone_url)
+    env = None
+    if has_ssh_access():
+        env = {"GIT_SSH_COMMAND": get_ssh_cmd()}
+    git_fetch(repo_path=repo_path, revision=revision, flags=flags, env=env)
+
+
 def create_code_repo(
     repo_path: str,
     url: str,
@@ -111,10 +138,15 @@ def create_code_repo(
     except Exception as e:
         raise PolyaxonContainerException("Error parsing url: {}.".format(url)) from e
 
-    clone_git_repo(repo_path=repo_path, url=clone_url, flags=flags)
-    set_remote(repo_path=repo_path, url=url)
-    if revision:
-        checkout_revision(repo_path=repo_path, revision=revision)
+    if flags and "--experimental-fetch" in flags:
+        flags.remove("--experimental-fetch")
+        fetch_git_repo(
+            repo_path=repo_path, clone_url=clone_url, revision=revision, flags=flags
+        )
+    else:
+        clone_and_checkout_git_repo(
+            repo_path=repo_path, clone_url=clone_url, revision=revision, flags=flags
+        )
 
     if settings.CLIENT_CONFIG.no_api:
         return

core/polyaxon/pkg.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 NAME = "polyaxon"
-VERSION = "1.4.5"
+VERSION = "1.4.6"
 SCHEMA_VERSION = 1.1
 DESC = "Command Line Interface (CLI) for Polyaxon."
 URL = "https://github.com/polyaxon/polyaxon"

core/polyaxon/polypod/compiler/config.py

@@ -41,7 +41,8 @@ def __init__(self, internal_auth: bool = False):
     def resolve(
         self, compiled_operation: V1CompiledOperation, agent_config: AgentConfig = None
     ):
-        agent_config = agent_config or settings.AGENT_CONFIG
+        if not agent_config and settings.AGENT_CONFIG:
+            agent_config = settings.AGENT_CONFIG.clone()
         if not agent_config:
             raise PolyaxonCompilerError(
                 "Polypod configuration not found or agent not configured."

core/polyaxon/polypod/compiler/resolver/base.py

@@ -137,7 +137,8 @@ def resolve_presets(self):
         pass
 
     def resolve_agent(self):
-        self.agent_config = settings.AGENT_CONFIG
+        if settings.AGENT_CONFIG:
+            self.agent_config = settings.AGENT_CONFIG.clone()
 
     def apply_operation_contexts(self):
         try:

core/polyaxon/schemas/base.py

@@ -201,6 +201,9 @@ def write(self, filepath: str):
         with open(filepath, "w") as config_file:
             config_file.write(self.to_dict(dump=True))
 
+    def clone(self):
+        return self.from_dict(self.to_dict())
+
     @staticmethod
     def patch_normal_merge(current_value, value, strategy: V1PatchStrategy = None):
         strategy = strategy or V1PatchStrategy.POST_MERGE

core/polyaxon/utils/cmd.py

@@ -22,13 +22,19 @@
 from psutil import Popen
 
 
-def run_command(cmd, data, location, chw):
+def run_command(cmd, data, location, chw, env=None):
+    cmd_env = None
+    if env:
+        cmd_env = os.environ.copy()
+        cmd_env.update(env)
     cwd = os.getcwd()
     if location is not None and chw is True:
         cwd = location
     elif location is not None and chw is False:
         cmd = "{0} {1}".format(cmd, location)
-    r = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE, cwd=cwd)
+    r = Popen(
+        shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE, cwd=cwd, env=cmd_env
+    )
     if data is None:
         output = r.communicate()[0].decode("utf-8")
     else: