6 Local Training

docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM continuumio/miniconda3
+
+ADD executor.sh /home
+WORKDIR /home
+
+ENTRYPOINT ["sh", "executor.sh"]

docker/executor.sh

@@ -0,0 +1,9 @@
+mkdir project
+cd project
+
+git clone https://$OAUTH_TOKEN:x-oauth-basic@$GIT_URL .
+git checkout $COMMIT_SHA
+
+conda env create -f environment.yml
+
+conda run -n hydra $PREFIX_PARAMS python3 $MODEL_PATH

docker/local_execution.sh

@@ -0,0 +1,22 @@
+
+DIR="$( dirname "${BASH_SOURCE[0]}" )"
+LOG_NAME=$(date +'%Y_%m_%d_%H_%M_%S')
+
+cd $DIR
+#TODO Bug fix building hydra image
+docker build -t hydra_image .
+# Create logs folder if it doesn't exist
+
+docker run \
+  -e GIT_URL=$1 \
+  -e COMMIT_SHA=$2 \
+  -e OAUTH_TOKEN=$3 \
+  -e MODEL_PATH=$4 \
+  -e PREFIX_PARAMS=$5 \
+  hydra_image:latest 2>&1 | tee ${LOG_NAME}.log
+
+cd -
+pwd
+mkdir -p tmp/hydra
+
+mv ${DIR}/${LOG_NAME}.log tmp/hydra/

hydra/cli.py

@@ -1,18 +1,46 @@
+import os
+import re
+import json
 import click
+import subprocess
+from hydra.utils import check_repo
 from hydra.version import __version__
 
-
 @click.group()
 @click.version_option(__version__)
 def cli():
     pass
 
 
 @cli.command()
-@click.option('--project_name')
-@click.option('--model_name')
-@click.option('--cpu')
-@click.option('--memory')
-@click.option('--options')
-def train(project_name, model_name, cpu, memory, options):
+@click.option('-m', '--model_path', required=True, type=str)
+@click.option('-c', '--cpu', default=16, type=click.IntRange(0, 128), help='Number of CPU cores required')
+@click.option('-r', '--memory', default=8, type=click.IntRange(0, 128), help='GB of RAM required')
+@click.option('--cloud', default='local', type=click.Choice(['fast_local','local', 'aws', 'gcp', 'azure'], case_sensitive=False))
+@click.option('--github_token', envvar='GITHUB_TOKEN') # Takes either an option or environment var
+@click.option('-b', '--branch', default='master', type=str)
+@click.option('-o', '--options', default='{}', type=str)
+
+def train(model_path, cpu, memory, github_token, cloud, branch, options):
     click.echo("This is the training command")
+    click.echo("Running on {}".format(cloud))
+
+    options = json.loads(options)
+
+    prefix_params = ""
+    for key, value in options.items():
+        prefix_params += key + "=" + str(value) + " "
+
+    if cloud == 'fast_local':
+        subprocess.run([prefix_params, 'python3', model_path])
+
+    check_repo(github_token, branch)
+    git_url = subprocess.check_output("git config --get remote.origin.url", shell=True).decode("utf-8").strip()
+    # Remove https://www. prefix
+    git_url = re.compile(r"https?://(www\.)?").sub("", git_url).strip().strip('/')
+    commit_sha = subprocess.check_output("git log --pretty=tformat:'%h' -n1 .", shell=True).decode("utf-8").strip()
+
+    if cloud == 'local':
+        subprocess.run(
+            ['sh', os.path.join(os.path.dirname(__file__), '../docker/local_execution.sh'),
+             git_url, commit_sha, github_token, model_path, prefix_params])

hydra/utils.py

@@ -0,0 +1,23 @@
+import os
+import git
+
+def check_repo(github_token, branch):
+    if github_token == None:
+        raise Exception("GITHUB_TOKEN not found in environment variable or as argument")
+
+    repo = git.Repo(os.getcwd())
+    if (repo.bare):
+        raise Exception("This is not a git repo")
+
+    count_modified_files = len(repo.index.diff(None))
+    count_staged_files = len(repo.index.diff("HEAD"))
+    count_unpushed_commits = len(list(repo.iter_commits(branch+'@{u}..master')))
+
+    if count_unpushed_commits > 0:
+        raise Exception("Some commits are not pushed to master branch.")
+
+    if count_staged_files > 0:
+        raise Exception("Some staged files are not commited.")
+
+    if count_modified_files > 0:
+        raise Exception("Some modified files are not staged for commit.")

hydra/version.py

@@ -1 +1 @@
-__version__ = '0.1.0'
+__version__ = '0.1.0'

requirements.txt

@@ -1 +1 @@
-click==7.1.2
+click==7.1.2