diff --git a/pyneuroml/archive/__init__.py b/pyneuroml/archive/__init__.py index 9f9d5d9e..2dce416c 100644 --- a/pyneuroml/archive/__init__.py +++ b/pyneuroml/archive/__init__.py @@ -8,7 +8,6 @@ import argparse import logging -import os import pathlib import shutil import typing @@ -18,6 +17,7 @@ from pyneuroml.sedml import validate_sedml_files from pyneuroml.utils import get_model_file_list from pyneuroml.utils.cli import build_namespace +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -170,20 +170,19 @@ def create_combine_archive( zipfile_name = rootfile # change to directory of rootfile - thispath = os.getcwd() - os.chdir(rootdir) - - lems_def_dir = None - if len(filelist) == 0: - lems_def_dir = get_model_file_list(rootfile, filelist, rootdir, lems_def_dir) + with chdir(rootdir): + lems_def_dir = None + if len(filelist) == 0: + lems_def_dir = get_model_file_list( + rootfile, filelist, rootdir, lems_def_dir + ) - create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir) - filelist.append("manifest.xml") + create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir) + filelist.append("manifest.xml") - with ZipFile(zipfile_name + zipfile_extension, "w") as archive: - for f in filelist + extra_files: - archive.write(f) - os.chdir(thispath) + with ZipFile(zipfile_name + zipfile_extension, "w") as archive: + for f in filelist + extra_files: + archive.write(f) if lems_def_dir is not None: logger.info(f"Removing LEMS definitions directory {lems_def_dir}") diff --git a/pyneuroml/nsgr.py b/pyneuroml/nsgr.py index 73238730..55db1a91 100644 --- a/pyneuroml/nsgr.py +++ b/pyneuroml/nsgr.py @@ -14,6 +14,7 @@ from zipfile import ZipFile from pyneuroml.runners import generate_sim_scripts_in_folder +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -126,8 +127,6 @@ def run_on_nsg( # NSG requires that the top level directory exist nsg_dir = pathlib.Path(zipfile_name.replace(".zip", "")) - cwd = pathlib.Path.cwd() - tdir = generate_sim_scripts_in_folder( engine=engine, lems_file_name=lems_file_name, @@ -139,44 +138,43 @@ def run_on_nsg( logger.info("Generating zip file") runner_file = "" - os.chdir(str(tdir)) - generated_files = os.listdir(nsg_dir) - - print(f"Generated files are {generated_files}") - - with ZipFile(zipfile_name, "w") as archive: - for f in generated_files: - if engine == "jneuroml_neuron": - if f.endswith("_nrn.py"): - runner_file = f - elif engine == "jneuroml_netpyne": - if f.endswith("_netpyne.py"): - runner_file = f - fpath = pathlib.Path(f) - moved_path = nsg_dir / fpath - archive.write(str(moved_path)) - - logger.debug("Printing testParam.properties") - nsg_sim_config_dict["filename_"] = runner_file - logger.debug(f"NSG sim config is: {nsg_sim_config_dict}") - - with open("testParam.properties", "w") as file: - for key, val in nsg_sim_config_dict.items(): - print(f"{key}={val}", file=file) - - logger.debug("Printing testInput.properties") - with open("testInput.properties", "w") as file: - print(f"infile_=@./{zipfile_name}", file=file) - - print(f"{zipfile_name} generated") - # uses argv, where the first argument is the script itself, so we must pass - # something as the 0th index of the list - if not dry_run: - if nsgr_submit(["", ".", "validate"]) == 0: - print("Attempting to submit to NSGR") - return nsgr_submit(["", ".", "run"]) - else: - print("Dry run mode enabled. Not submitting to NSG.") - - os.chdir(str(cwd)) + with chdir(str(tdir)): + generated_files = os.listdir(nsg_dir) + + print(f"Generated files are {generated_files}") + + with ZipFile(zipfile_name, "w") as archive: + for f in generated_files: + if engine == "jneuroml_neuron": + if f.endswith("_nrn.py"): + runner_file = f + elif engine == "jneuroml_netpyne": + if f.endswith("_netpyne.py"): + runner_file = f + fpath = pathlib.Path(f) + moved_path = nsg_dir / fpath + archive.write(str(moved_path)) + + logger.debug("Printing testParam.properties") + nsg_sim_config_dict["filename_"] = runner_file + logger.debug(f"NSG sim config is: {nsg_sim_config_dict}") + + with open("testParam.properties", "w") as file: + for key, val in nsg_sim_config_dict.items(): + print(f"{key}={val}", file=file) + + logger.debug("Printing testInput.properties") + with open("testInput.properties", "w") as file: + print(f"infile_=@./{zipfile_name}", file=file) + + print(f"{zipfile_name} generated") + # uses argv, where the first argument is the script itself, so we must pass + # something as the 0th index of the list + if not dry_run: + if nsgr_submit(["", ".", "validate"]) == 0: + print("Attempting to submit to NSGR") + return nsgr_submit(["", ".", "run"]) + else: + print("Dry run mode enabled. Not submitting to NSG.") + return tdir diff --git a/pyneuroml/runners.py b/pyneuroml/runners.py index 37efa376..a8ff65ac 100644 --- a/pyneuroml/runners.py +++ b/pyneuroml/runners.py @@ -30,6 +30,7 @@ import pyneuroml.utils.misc from pyneuroml import DEFAULTS, __version__ from pyneuroml.errors import UNKNOWN_ERR +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -1304,7 +1305,6 @@ def generate_sim_scripts_in_folder( if root_dir is None: root_dir = "." - cwd = Path.cwd() tdir = pyneuroml.utils.get_pyneuroml_tempdir(rootdir=run_dir, prefix="pyneuroml") os.mkdir(tdir) @@ -1315,94 +1315,89 @@ def generate_sim_scripts_in_folder( # change to root_dir, so that we're in the directory where the lems file # is - os.chdir(root_dir) + with chdir(root_dir): + logger.debug("Getting list of model files") + model_file_list = [] # type: list + lems_def_dir = None + lems_def_dir = pyneuroml.utils.get_model_file_list( + lems_file_name, model_file_list, root_dir, lems_def_dir + ) - logger.debug("Getting list of model files") - model_file_list = [] # type: list - lems_def_dir = None - lems_def_dir = pyneuroml.utils.get_model_file_list( - lems_file_name, model_file_list, root_dir, lems_def_dir - ) + logger.debug(f"Model file list is {model_file_list}") + + for model_file in model_file_list: + logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}") + # if model file has directory structures in it, recreate the dirs in + # the temporary directory + if len(model_file.split("/")) > 1: + # throw error if files in parent directories are referred to + if "../" in model_file: + raise ValueError( + """ + Cannot handle parent directories because we + cannot create these directories correctly in + the temporary location. Please re-organize + your code such that all included files are in + sub-directories of the root directory where the + main file resides. + """ + ) - logger.debug(f"Model file list is {model_file_list}") - - for model_file in model_file_list: - logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}") - # if model file has directory structures in it, recreate the dirs in - # the temporary directory - if len(model_file.split("/")) > 1: - # throw error if files in parent directories are referred to - if "../" in model_file: - raise ValueError( - """ - Cannot handle parent directories because we - cannot create these directories correctly in - the temporary location. Please re-organize - your code such that all included files are in - sub-directories of the root directory where the - main file resides. - """ - ) + model_file_path = pathlib.Path(tdir + "/" + model_file) + parent = model_file_path.parent + parent.mkdir(parents=True, exist_ok=True) + shutil.copy(model_file, tdir + "/" + model_file) + + if lems_def_dir is not None: + logger.info(f"Removing LEMS definitions directory {lems_def_dir}") + shutil.rmtree(lems_def_dir) + + with chdir(tdir): + logger.info(f"Working in {tdir}") + start_time = time.time() - 1.0 + + if engine == "jneuroml_neuron": + run_lems_with( + engine, + lems_file_name=Path(lems_file_name).name, + compile_mods=False, + only_generate_scripts=True, + *engine_args, + **engine_kwargs, + ) + elif engine == "jneuroml_netpyne": + run_lems_with( + engine, + lems_file_name=Path(lems_file_name).name, + only_generate_scripts=True, + *engine_args, + **engine_kwargs, + ) - model_file_path = pathlib.Path(tdir + "/" + model_file) - parent = model_file_path.parent - parent.mkdir(parents=True, exist_ok=True) - shutil.copy(model_file, tdir + "/" + model_file) - - if lems_def_dir is not None: - logger.info(f"Removing LEMS definitions directory {lems_def_dir}") - shutil.rmtree(lems_def_dir) - - os.chdir(tdir) - logger.info(f"Working in {tdir}") - start_time = time.time() - 1.0 - - if engine == "jneuroml_neuron": - run_lems_with( - engine, - lems_file_name=Path(lems_file_name).name, - compile_mods=False, - only_generate_scripts=True, - *engine_args, - **engine_kwargs, - ) - elif engine == "jneuroml_netpyne": - run_lems_with( - engine, - lems_file_name=Path(lems_file_name).name, - only_generate_scripts=True, - *engine_args, - **engine_kwargs, + generated_files = pyneuroml.utils.get_files_generated_after( + start_time, ignore_suffixes=["xml", "nml"] ) - generated_files = pyneuroml.utils.get_files_generated_after( - start_time, ignore_suffixes=["xml", "nml"] - ) - - # For NetPyNE, the channels are converted to NEURON mod files, but the - # network and cells are imported from the nml files. - # So we include all the model files too. - if engine == "jneuroml_netpyne": - generated_files.extend(model_file_list) + # For NetPyNE, the channels are converted to NEURON mod files, but the + # network and cells are imported from the nml files. + # So we include all the model files too. + if engine == "jneuroml_netpyne": + generated_files.extend(model_file_list) - logger.debug(f"Generated files are: {generated_files}") + logger.debug(f"Generated files are: {generated_files}") - if generated_files_dir_name is None: - generated_files_dir_name = Path(tdir).name + "_generated" - logger.debug( - f"Creating directory and moving generated files to it: {generated_files_dir_name}" - ) - - for f in generated_files: - fpath = pathlib.Path(f) - moved_path = generated_files_dir_name / fpath - # use os.renames because pathlib.Path.rename does not move - # recursively and so cannot move files within directories - os.renames(fpath, moved_path) + if generated_files_dir_name is None: + generated_files_dir_name = Path(tdir).name + "_generated" + logger.debug( + f"Creating directory and moving generated files to it: {generated_files_dir_name}" + ) - # return to original directory - # doesn't affect scripts much, but does affect our tests - os.chdir(str(cwd)) + for f in generated_files: + fpath = pathlib.Path(f) + moved_path = generated_files_dir_name / fpath + # use os.renames because pathlib.Path.rename does not move + # recursively and so cannot move files within directories + os.renames(fpath, moved_path) return tdir diff --git a/pyneuroml/utils/__init__.py b/pyneuroml/utils/__init__.py index 62b25f1d..54b877d0 100644 --- a/pyneuroml/utils/__init__.py +++ b/pyneuroml/utils/__init__.py @@ -713,17 +713,18 @@ def get_model_file_list( filelist.append(str(fullrootfile_rel)) if str(rootfile_name).endswith(".nml"): - print(f"Processing NML file: {fullrootfile_rel}") + logger.info(f"Processing NML file: {fullrootfile_rel}") rootdoc = read_neuroml2_file(fullrootfile_rel) logger.debug(f"Has includes: {rootdoc.includes}") for inc in rootdoc.includes: - logger.debug(f"Processing includes: {inc.href} in {str(rootdirpath)}") + logger.debug(f"NML: Processing includes: {inc.href} in {str(rootdirpath)}") lems_def_dir = get_model_file_list( inc.href, filelist, str(rootdirpath_rel), lems_def_dir ) elif str(rootfile_name).endswith(".xml"): + logger.info(f"Processing LEMS file: {fullrootfile_rel}") # extract the standard NeuroML2 LEMS definitions into a directory # so that the LEMS parser can find them if lems_def_dir is None: @@ -734,17 +735,25 @@ def get_model_file_list( model.import_from_file(fullrootfile_rel) for inc in model.included_files: - # `inc` includes the folder name, but we want to keep the file name - # and the directory in which it is located separtely as the - # directory may have to be passed on recursively to other included - # files. So, we separate the name out. - incfile = pathlib.Path(inc).name - logger.debug(f"Processing include file {incfile} ({inc})") + # `inc` includes the complete path relative to the current + # directory, which may repeat information such as the "rootdirpath" + # that we're tracking outselves. So, we need to do some massaging + # here to get the path to inc relative to the rootdirpath_rel + # again + incfile_path = pathlib.Path(inc) + incfile = incfile_path.name + + if incfile_path.is_relative_to(rootdirpath_rel): + incfile_path_rel = incfile_path.relative_to(rootdirpath_rel) + else: + incfile_path_rel = incfile_path + + logger.debug(f"LEMS: Processing include file {incfile} ({inc})") if incfile in STANDARD_LEMS_FILES: logger.debug(f"Ignoring NeuroML2 standard LEMS file: {inc}") continue lems_def_dir = get_model_file_list( - incfile, filelist, str(rootdirpath_rel), lems_def_dir + str(incfile_path_rel), filelist, str(rootdirpath_rel), lems_def_dir ) elif str(rootfile_name).endswith(".sedml"): @@ -754,6 +763,7 @@ def get_model_file_list( logger.error("Please install optional dependencies to use SED-ML features:") logger.error("pip install pyneuroml[combine]") + logger.info(f"Processing SED-ML file: {fullrootfile_rel}") rootdoc = libsedml.readSedMLFromFile(fullrootfile_rel) # there should only be one model diff --git a/pyneuroml/utils/misc.py b/pyneuroml/utils/misc.py index 507fdcdd..2afd481c 100644 --- a/pyneuroml/utils/misc.py +++ b/pyneuroml/utils/misc.py @@ -24,3 +24,23 @@ def get_path_to_jnml_jar() -> str: "jNeuroML-%s-jar-with-dependencies.jar" % JNEUROML_VERSION, ) return jar_path + + +try: + from contextlib import chdir # Python 3.11+ +except ImportError: + from contextlib import contextmanager + + @contextmanager + def chdir(path): + """chdir context manager for python < 3.11 + + :param path: path to change to + :type path: str or os.PathLike + """ + prev_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev_cwd) diff --git a/tests/archive/test_archive.py b/tests/archive/test_archive.py index 0a4161d6..3afff42c 100644 --- a/tests/archive/test_archive.py +++ b/tests/archive/test_archive.py @@ -17,6 +17,7 @@ get_model_file_list, ) from pyneuroml.runners import run_jneuroml +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -44,6 +45,18 @@ def test_get_model_file_list_2(self): ) self.assertEqual(5, len(filelist)) + def test_get_model_file_list_2a(self): + # a LEMS file in the examples directory + thispath = pathlib.Path(__file__) + dirname = str(thispath.parent.parent.parent) + + with chdir(dirname + "/examples"): + filelist = [] + get_model_file_list( + "LEMS_NML2_Ex5_DetCell.xml", filelist, dirname + "/examples" + ) + self.assertEqual(5, len(filelist)) + def test_get_model_file_list_3(self): thispath = pathlib.Path(__file__) # a SEDML file in the examples directory diff --git a/tests/test_biosimulations.py b/tests/test_biosimulations.py index 978104c6..3e91b1fa 100644 --- a/tests/test_biosimulations.py +++ b/tests/test_biosimulations.py @@ -8,13 +8,13 @@ """ import logging -import os import pathlib from pyneuroml.biosimulations import ( get_simulator_versions, submit_simulation, ) +from pyneuroml.utils.misc import chdir from . import BaseTestCase @@ -53,20 +53,18 @@ def test_submit_simulation(self): dry_run = True thispath = pathlib.Path(__file__) dirname = str(thispath.parent.parent) - cwd = os.getcwd() - os.chdir(dirname + "/examples") - sim_dict = { - "name": "PyNeuroML test simulation", - "simulator": "neuron", - "simulatorVersion": "latest", - "maxTime": "20", - } + with chdir(dirname + "/examples"): + sim_dict = { + "name": "PyNeuroML test simulation", + "simulator": "neuron", + "simulatorVersion": "latest", + "maxTime": "20", + } - resdict = submit_simulation( - "LEMS_NML2_Ex5_DetCell.xml", sim_dict=sim_dict, dry_run=dry_run - ) - response = resdict["response"] - os.chdir(cwd) + resdict = submit_simulation( + "LEMS_NML2_Ex5_DetCell.xml", sim_dict=sim_dict, dry_run=dry_run + ) + response = resdict["response"] if dry_run: pass diff --git a/tests/test_pynml.py b/tests/test_pynml.py index 3435f9a4..9fe5953c 100644 --- a/tests/test_pynml.py +++ b/tests/test_pynml.py @@ -22,6 +22,7 @@ run_jneuroml, validate_neuroml2, ) +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -73,10 +74,9 @@ def test_exposure_listing(self): def test_exposure_listing_2(self): """Test listing of exposures in NeuroML documents.""" - os.chdir("tests/") - exps = list_exposures("HH_example_net.nml") - print(exps) - os.chdir("../") + with chdir("tests/"): + exps = list_exposures("HH_example_net.nml") + print(exps) def test_recording_path_listing(self): """Test listing of recording paths in NeuroML documents.""" @@ -89,12 +89,11 @@ def test_recording_path_listing(self): def test_recording_path_listing_2(self): """Test listing of recording paths in NeuroML documents.""" - os.chdir("tests/") - paths = list_recording_paths_for_exposures( - "HH_example_net.nml", "hh_cell", "single_hh_cell_network" - ) - print("\n".join(paths)) - os.chdir("../") + with chdir("tests/"): + paths = list_recording_paths_for_exposures( + "HH_example_net.nml", "hh_cell", "single_hh_cell_network" + ) + print("\n".join(paths)) def test_execute_command_in_dir(self): """Test execute_command_in_dir function.""" @@ -152,19 +151,18 @@ def test_run_jneuroml(self): def test_validate_neuroml2(self): """Test validate_neuroml2""" - os.chdir("tests/") - retval = None - retval = validate_neuroml2("HH_example_k_channel.nml") - self.assertTrue(retval) - - retval = None - retstring = None - retval, retstring = validate_neuroml2( - "HH_example_k_channel.nml", return_string=True - ) - self.assertTrue(retval) - self.assertIn("Valid against schema and all tests", retstring) - os.chdir("../") + with chdir("tests/"): + retval = None + retval = validate_neuroml2("HH_example_k_channel.nml") + self.assertTrue(retval) + + retval = None + retstring = None + retval, retstring = validate_neuroml2( + "HH_example_k_channel.nml", return_string=True + ) + self.assertTrue(retval) + self.assertIn("Valid against schema and all tests", retstring) retval = None retstring = None