diff --git a/modeldb/modeldb.py b/modeldb/modeldb.py index 87eb284..a22a5ca 100644 --- a/modeldb/modeldb.py +++ b/modeldb/modeldb.py @@ -30,7 +30,7 @@ def download_model(arg_tuple): # from ModelDB, but it can be overriden to come from GitHub instead. if "github" in model_run_info: # This means we should try to download the model content from - # GitHub instead of from ModelDB. + # GitHub instead of from ModelDB. (but see the local: case below.) github = model_run_info["github"] organisation = "ModelDBRepository" suffix = "" # default branch @@ -57,6 +57,27 @@ def download_model(arg_tuple): # if you need to test changes to a model that does not exist on # GitHub under the ModelDBRepository organisation. organisation = github[1:] + elif github.startswith("local:"): + # Using + # github: "local: /path/to/repository" + # in modeldb-run.yaml implies that we 'git clone /path/to/repository'. + # This is useful to tentatively explore the effect of + # model changes on results with different nrn versions + # without committing to github or before being ready to + # make a pull request + print("\nTo be cloned by runmodel", github) + return model_id, model + elif github.startswith("copy:"): + # Using + # github: "copy: /path/to/parentfolder" + # in modeldb-run.yaml implies that we + # 'cp -R /path/to/parentfolder/ ' + # A copy differs from a clone in that local changes in + # the checkout are mirrored in the copy without having to + # be committed. Note the copy leaves out the .git and + # x86_64 folders. + print("\nTo be copied by runmodel %s/%s" % (github, model_id)) + return model_id, model else: raise Exception("Invalid value for github key: {}".format(github)) url = "https://api.github.com/repos/{organisation}/{model_id}/zipball{suffix}".format( diff --git a/modeldb/modelrun.py b/modeldb/modelrun.py index 5ae7b55..5481727 100644 --- a/modeldb/modelrun.py +++ b/modeldb/modelrun.py @@ -205,15 +205,30 @@ def build_python_runfile(model): def prepare_model(model): + if "github" in model.keys() and (model['github'].startswith("local:") + or model['github'].startswith("copy")): + _prepare_model(model, None, clone=True) + return + # unzip model from cache with zipfile.ZipFile( - os.path.join(MODELS_ZIP_DIR, str(model.id) + ".zip"), "r" + os.path.join(MODELS_ZIP_DIR, str(model.id) + ".zip"), "r" ) as zip_ref: + _prepare_model(model, zip_ref, clone=False) + +def _prepare_model(model, zip_ref, clone=False): + if clone: + model_dir = os.path.join( + model.working_dir, + str(model.id), + ) + else: model_dir = os.path.join( model.working_dir, str(model.id), os.path.dirname(zip_ref.infolist()[0].filename), ) + if True: model_run_info_file = os.path.join(model_dir, str(model.id) + ".yaml") if model._inplace and os.path.isfile(model_run_info_file): with open(model_run_info_file) as run_info_file: @@ -221,7 +236,10 @@ def prepare_model(model): else: if model._clean and is_dir_non_empty(model_dir): shutil.rmtree(model_dir) - zip_ref.extractall(os.path.join(model.working_dir, str(model.id))) + if clone: + gitclone(model, model_dir) + else: + zip_ref.extractall(os.path.join(model.working_dir, str(model.id))) # set model_dir model.run_info["model_dir"] = model_dir @@ -247,6 +265,15 @@ def prepare_model(model): yaml.dump(model.run_info, run_info_file, sort_keys=True) +def gitclone(model, model_dir): + if model['github'].startswith('copy:'): + cmd = 'cp -R %s/%s %s' % (model['github'][5:], str(model.id), model_dir) + else: + cmd = 'git clone %s %s' % (model['github'][7:], model_dir) + print("\n", cmd) + a = subprocess.run(cmd, shell=True, check=True, capture_output=True) + + def run_model(model): start_time = time.perf_counter() # Some models are skipped on purpose