From e6c88dad7b6a939e9d6934504f5529086312e142 Mon Sep 17 00:00:00 2001 From: arpanetus Date: Sat, 23 Mar 2024 17:09:42 +0500 Subject: [PATCH 1/2] add new args (invoker_exec, of_master_host, of_no_python) --- higgsfield/internal/cfg.py | 13 +++++++++- higgsfield/internal/experiment/builder.py | 24 +++++++++++++++++-- higgsfield/internal/util.py | 4 ++-- higgsfield/static/templates/deploy_action.j2 | 4 ++-- .../static/templates/experiment_action.j2 | 2 +- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/higgsfield/internal/cfg.py b/higgsfield/internal/cfg.py index 55aa480..9d10356 100644 --- a/higgsfield/internal/cfg.py +++ b/higgsfield/internal/cfg.py @@ -32,10 +32,14 @@ class AppConfig: user: str key: str port: int + master_host: Optional[str] + no_python: Optional[str] + invoker_exec: str = "invoker" number_of_processes_per_node: int def __init__(self, **kwargs): self.__dict__.update(kwargs) + @classmethod def from_path(cls, path: Path) -> "AppConfig": config_path = path / "src" / "config.py" @@ -66,6 +70,10 @@ def from_path(cls, path: Path) -> "AppConfig": key = get_key_from_path_or_key(os.getenv("SSH_KEY")) + master_host = module.__dict__.get("MASTER_HOST", None) + no_python = module.__dict__.get("NO_PYTHON", None) + invoker_exec = module.__dict__.get("INVOKER_EXEC", "invoker") + return AppConfig( name=name, github_repo_url=github_repo_url, @@ -74,6 +82,9 @@ def from_path(cls, path: Path) -> "AppConfig": key=key, port=port, number_of_processes_per_node=number_of_processes_per_node, + master_host=master_host, + no_python=no_python, + invoker_exec=invoker_exec, ) def get_git_origin_url(self, path) -> Optional[str]: @@ -106,7 +117,7 @@ def set_git_origin_url(self, path: Path): with open(config_path, "w") as f: # Write back the modified lines f.writelines(lines) - + if lines[-1] != "\n": f.write("\n") diff --git a/higgsfield/internal/experiment/builder.py b/higgsfield/internal/experiment/builder.py index 3ce3399..7d94532 100644 --- a/higgsfield/internal/experiment/builder.py +++ b/higgsfield/internal/experiment/builder.py @@ -105,6 +105,18 @@ def generate(self): class ExperimentBuilder(ActionBuilder): template_name = "experiment_action.j2" + def of_master_host(self) -> str: + master_host = self.app_config.master_host + if master_host is None: + return "" + return f"--master_host {master_host}" + + def of_no_python(self) -> str: + no_python = self.app_config.no_python + if no_python is None: + return "" + return f"--no_python {no_python}" + def generate(self, experiment_name: str, params: List[Param]): (self.wf_dir / f"run_{experiment_name}.yml").write_text( self.template.render( @@ -113,12 +125,14 @@ def generate(self, experiment_name: str, params: List[Param]): project_name=self.app_config.name, params=build_gh_action_inputs(params), rest=build_run_params(params), + invoker_exec=self.app_config.invoker_exec, + of_master_host=self.of_master_host(), + of_no_python=self.of_no_python(), env_gen=env_keys_as_action( self.wf_dir.parent.parent / "env", echo_indent ), ) ) - print("Updated experiment action", experiment_name) @@ -130,7 +144,13 @@ def _source_experiments(base_path: Path): the same dependencies (aka environment) as the docker container. """ for file in base_path.glob("**/*.py"): - module_name = os.path.basename(file).split(".py")[0].split(".py")[0].replace(" ", "_").replace("-", "_") + module_name = ( + os.path.basename(file) + .split(".py")[0] + .split(".py")[0] + .replace(" ", "_") + .replace("-", "_") + ) SourceFileLoader(module_name, str(file)).load_module() diff --git a/higgsfield/internal/util.py b/higgsfield/internal/util.py index 4cd1260..6ab0165 100644 --- a/higgsfield/internal/util.py +++ b/higgsfield/internal/util.py @@ -9,8 +9,8 @@ def check_name(name: str): - if len(name) < 1 or len(name) > 20: - raise ValueError("Name must be between 1 and 20 characters long") + if len(name) < 1 or len(name) > 30: + raise ValueError("Name must be between 1 and 30 characters long") if not regex.match(name): raise ValueError("Name must match regex ^[a-zA-Z_][a-zA-Z0-9_]*$") diff --git a/higgsfield/static/templates/deploy_action.j2 b/higgsfield/static/templates/deploy_action.j2 index 3dcd309..ccbefe8 100644 --- a/higgsfield/static/templates/deploy_action.j2 +++ b/higgsfield/static/templates/deploy_action.j2 @@ -37,8 +37,7 @@ jobs: host: ${{ env.HOSTS }} username: ${{ env.USER }} key: ${{ secrets.SSH_KEY }} - port: ${{ env.PORT }} - sync: true {% endraw %} + port: ${{ env.PORT }} {% endraw %} script: | mkdir -p ~/higgsfield/ cd ~/higgsfield @@ -48,5 +47,6 @@ jobs: git reset --hard origin/main && \ git pull origin main) || git clone {{ keyed_repo_url }} {{ project_name }} || rm -rf ~/higgsfield/{{ project_name }} && git clone {{ keyed_repo_url }} {{ project_name }} + cd ~/higgsfield/{{ project_name }} echo "SSH_KEY=NOTHING" > env {{ env_gen }} diff --git a/higgsfield/static/templates/experiment_action.j2 b/higgsfield/static/templates/experiment_action.j2 index c1c82ba..a38a994 100644 --- a/higgsfield/static/templates/experiment_action.j2 +++ b/higgsfield/static/templates/experiment_action.j2 @@ -68,7 +68,7 @@ jobs: cd ~/higgsfield/{% endraw %}{{ project_name }} echo "SSH_KEY=NOTHING" > env {{ env_gen }} - invoker experiment run --project_name {{ project_name }} --experiment_name {{ experiment_name }} {% raw %} --run_name ${{ env.RUN_NAME }} --port ${{ env.RUN_PORT }} --nproc_per_node ${{ env.NPROC_PER_NODE }} --hosts ${{ env.HOSTS }} {% endraw %} {{ rest }} + {{ invoker_exec }} experiment run {{ of_master_host }} {{ of_no_python }} --project_name {{ project_name }} --experiment_name {{ experiment_name }} {% raw %} --run_name ${{ env.RUN_NAME }} --port ${{ env.RUN_PORT }} --nproc_per_node ${{ env.NPROC_PER_NODE }} --hosts ${{ env.HOSTS }} {% endraw %} {{ rest }} From 1a6a10e43a0d65c552da1bca2bd65726697d2a29 Mon Sep 17 00:00:00 2001 From: arpanetus Date: Sat, 23 Mar 2024 17:14:53 +0500 Subject: [PATCH 2/2] fix deps --- poetry.lock | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index 29ca688..2c6fa5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,7 +49,6 @@ files = [ [package.dependencies] bcrypt = {version = ">=3.1.3", optional = true, markers = "extra == \"bcrypt\""} cryptography = ">=39.0" -fido2 = {version = ">=0.9.2", optional = true, markers = "extra == \"fido2\""} libnacl = {version = ">=1.4.2", optional = true, markers = "extra == \"libnacl\""} pyOpenSSL = {version = ">=23.0.0", optional = true, markers = "extra == \"pyopenssl\""} typing-extensions = ">=3.6" @@ -245,23 +244,6 @@ files = [ [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "fido2" -version = "1.1.2" -description = "FIDO2/WebAuthn library for implementing clients and servers." -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "fido2-1.1.2-py3-none-any.whl", hash = "sha256:a3b7d7d233dec3a4fa0d6178fc34d1cce17b820005a824f6ab96917a1e3be8d8"}, - {file = "fido2-1.1.2.tar.gz", hash = "sha256:6110d913106f76199201b32d262b2857562cc46ba1d0b9c51fbce30dc936c573"}, -] - -[package.dependencies] -cryptography = ">=2.6,<35 || >35,<44" - -[package.extras] -pcsc = ["pyscard (>=1.9,<3)"] - [[package]] name = "idna" version = "3.4" @@ -449,6 +431,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -508,4 +491,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8.13" -content-hash = "2d788b9b1dab389d649005cad70b2c6f9c8296a2336668028c5b829a805fd06d" +content-hash = "45a797648de5834b6fef2ea21b3fb219afe4ec9d8722132e8a69dee39bc88a92"