From edb8b9ebd958a538f98518e1c512a4755a7c9321 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 6 Aug 2025 16:18:02 +0100 Subject: [PATCH 1/5] chore(archive): add logging --- pyneuroml/utils/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyneuroml/utils/__init__.py b/pyneuroml/utils/__init__.py index 62b25f1d..386476fd 100644 --- a/pyneuroml/utils/__init__.py +++ b/pyneuroml/utils/__init__.py @@ -713,17 +713,18 @@ def get_model_file_list( filelist.append(str(fullrootfile_rel)) if str(rootfile_name).endswith(".nml"): - print(f"Processing NML file: {fullrootfile_rel}") + logger.info(f"Processing NML file: {fullrootfile_rel}") rootdoc = read_neuroml2_file(fullrootfile_rel) logger.debug(f"Has includes: {rootdoc.includes}") for inc in rootdoc.includes: - logger.debug(f"Processing includes: {inc.href} in {str(rootdirpath)}") + logger.debug(f"NML: Processing includes: {inc.href} in {str(rootdirpath)}") lems_def_dir = get_model_file_list( inc.href, filelist, str(rootdirpath_rel), lems_def_dir ) elif str(rootfile_name).endswith(".xml"): + logger.info(f"Processing LEMS file: {fullrootfile_rel}") # extract the standard NeuroML2 LEMS definitions into a directory # so that the LEMS parser can find them if lems_def_dir is None: @@ -739,7 +740,7 @@ def get_model_file_list( # directory may have to be passed on recursively to other included # files. So, we separate the name out. incfile = pathlib.Path(inc).name - logger.debug(f"Processing include file {incfile} ({inc})") + logger.debug(f"LEMS: Processing include file {incfile} ({inc})") if incfile in STANDARD_LEMS_FILES: logger.debug(f"Ignoring NeuroML2 standard LEMS file: {inc}") continue @@ -754,6 +755,7 @@ def get_model_file_list( logger.error("Please install optional dependencies to use SED-ML features:") logger.error("pip install pyneuroml[combine]") + logger.info(f"Processing SED-ML file: {fullrootfile_rel}") rootdoc = libsedml.readSedMLFromFile(fullrootfile_rel) # there should only be one model From d20cca12a29fa9f2edcb8867e13ce67273f82f9d Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 6 Aug 2025 16:58:31 +0100 Subject: [PATCH 2/5] test(archive): add test case This is when the user specifies a rootdir that is not the current folder --- tests/archive/test_archive.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/archive/test_archive.py b/tests/archive/test_archive.py index 0a4161d6..4cf3af33 100644 --- a/tests/archive/test_archive.py +++ b/tests/archive/test_archive.py @@ -7,6 +7,7 @@ Copyright 2023 NeuroML contributors """ +import contextlib import logging import pathlib import unittest @@ -44,6 +45,18 @@ def test_get_model_file_list_2(self): ) self.assertEqual(5, len(filelist)) + def test_get_model_file_list_2a(self): + # a LEMS file in the examples directory + thispath = pathlib.Path(__file__) + dirname = str(thispath.parent.parent.parent) + + with contextlib.chdir(dirname + "/examples"): + filelist = [] + get_model_file_list( + "LEMS_NML2_Ex5_DetCell.xml", filelist, dirname + "/examples" + ) + self.assertEqual(5, len(filelist)) + def test_get_model_file_list_3(self): thispath = pathlib.Path(__file__) # a SEDML file in the examples directory From 80ca4e1cb43a17c3ee19c14b6a833f1b648d84a2 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 6 Aug 2025 17:01:07 +0100 Subject: [PATCH 3/5] fix(archive): handle lems.model modifying the include path It always adds the path from the current working directory. So we need to strip that back out to be relative to the current root dir --- pyneuroml/utils/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pyneuroml/utils/__init__.py b/pyneuroml/utils/__init__.py index 386476fd..54b877d0 100644 --- a/pyneuroml/utils/__init__.py +++ b/pyneuroml/utils/__init__.py @@ -735,17 +735,25 @@ def get_model_file_list( model.import_from_file(fullrootfile_rel) for inc in model.included_files: - # `inc` includes the folder name, but we want to keep the file name - # and the directory in which it is located separtely as the - # directory may have to be passed on recursively to other included - # files. So, we separate the name out. - incfile = pathlib.Path(inc).name + # `inc` includes the complete path relative to the current + # directory, which may repeat information such as the "rootdirpath" + # that we're tracking outselves. So, we need to do some massaging + # here to get the path to inc relative to the rootdirpath_rel + # again + incfile_path = pathlib.Path(inc) + incfile = incfile_path.name + + if incfile_path.is_relative_to(rootdirpath_rel): + incfile_path_rel = incfile_path.relative_to(rootdirpath_rel) + else: + incfile_path_rel = incfile_path + logger.debug(f"LEMS: Processing include file {incfile} ({inc})") if incfile in STANDARD_LEMS_FILES: logger.debug(f"Ignoring NeuroML2 standard LEMS file: {inc}") continue lems_def_dir = get_model_file_list( - incfile, filelist, str(rootdirpath_rel), lems_def_dir + str(incfile_path_rel), filelist, str(rootdirpath_rel), lems_def_dir ) elif str(rootfile_name).endswith(".sedml"): From 7051d55dca163c79b79bdc3f8030940333196c5e Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 6 Aug 2025 17:23:55 +0100 Subject: [PATCH 4/5] feat(utils): add `chdir` context manager --- pyneuroml/utils/misc.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyneuroml/utils/misc.py b/pyneuroml/utils/misc.py index 507fdcdd..2afd481c 100644 --- a/pyneuroml/utils/misc.py +++ b/pyneuroml/utils/misc.py @@ -24,3 +24,23 @@ def get_path_to_jnml_jar() -> str: "jNeuroML-%s-jar-with-dependencies.jar" % JNEUROML_VERSION, ) return jar_path + + +try: + from contextlib import chdir # Python 3.11+ +except ImportError: + from contextlib import contextmanager + + @contextmanager + def chdir(path): + """chdir context manager for python < 3.11 + + :param path: path to change to + :type path: str or os.PathLike + """ + prev_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev_cwd) From 1ffa6062ac51bdb05b0ad230420840c4fade1aa3 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Wed, 6 Aug 2025 17:29:38 +0100 Subject: [PATCH 5/5] feat: use new `chdir` context manager --- pyneuroml/archive/__init__.py | 25 +++--- pyneuroml/nsgr.py | 82 +++++++++--------- pyneuroml/runners.py | 157 ++++++++++++++++------------------ tests/archive/test_archive.py | 4 +- tests/test_biosimulations.py | 26 +++--- tests/test_pynml.py | 44 +++++----- 6 files changed, 163 insertions(+), 175 deletions(-) diff --git a/pyneuroml/archive/__init__.py b/pyneuroml/archive/__init__.py index 9f9d5d9e..2dce416c 100644 --- a/pyneuroml/archive/__init__.py +++ b/pyneuroml/archive/__init__.py @@ -8,7 +8,6 @@ import argparse import logging -import os import pathlib import shutil import typing @@ -18,6 +17,7 @@ from pyneuroml.sedml import validate_sedml_files from pyneuroml.utils import get_model_file_list from pyneuroml.utils.cli import build_namespace +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -170,20 +170,19 @@ def create_combine_archive( zipfile_name = rootfile # change to directory of rootfile - thispath = os.getcwd() - os.chdir(rootdir) - - lems_def_dir = None - if len(filelist) == 0: - lems_def_dir = get_model_file_list(rootfile, filelist, rootdir, lems_def_dir) + with chdir(rootdir): + lems_def_dir = None + if len(filelist) == 0: + lems_def_dir = get_model_file_list( + rootfile, filelist, rootdir, lems_def_dir + ) - create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir) - filelist.append("manifest.xml") + create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir) + filelist.append("manifest.xml") - with ZipFile(zipfile_name + zipfile_extension, "w") as archive: - for f in filelist + extra_files: - archive.write(f) - os.chdir(thispath) + with ZipFile(zipfile_name + zipfile_extension, "w") as archive: + for f in filelist + extra_files: + archive.write(f) if lems_def_dir is not None: logger.info(f"Removing LEMS definitions directory {lems_def_dir}") diff --git a/pyneuroml/nsgr.py b/pyneuroml/nsgr.py index 73238730..55db1a91 100644 --- a/pyneuroml/nsgr.py +++ b/pyneuroml/nsgr.py @@ -14,6 +14,7 @@ from zipfile import ZipFile from pyneuroml.runners import generate_sim_scripts_in_folder +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -126,8 +127,6 @@ def run_on_nsg( # NSG requires that the top level directory exist nsg_dir = pathlib.Path(zipfile_name.replace(".zip", "")) - cwd = pathlib.Path.cwd() - tdir = generate_sim_scripts_in_folder( engine=engine, lems_file_name=lems_file_name, @@ -139,44 +138,43 @@ def run_on_nsg( logger.info("Generating zip file") runner_file = "" - os.chdir(str(tdir)) - generated_files = os.listdir(nsg_dir) - - print(f"Generated files are {generated_files}") - - with ZipFile(zipfile_name, "w") as archive: - for f in generated_files: - if engine == "jneuroml_neuron": - if f.endswith("_nrn.py"): - runner_file = f - elif engine == "jneuroml_netpyne": - if f.endswith("_netpyne.py"): - runner_file = f - fpath = pathlib.Path(f) - moved_path = nsg_dir / fpath - archive.write(str(moved_path)) - - logger.debug("Printing testParam.properties") - nsg_sim_config_dict["filename_"] = runner_file - logger.debug(f"NSG sim config is: {nsg_sim_config_dict}") - - with open("testParam.properties", "w") as file: - for key, val in nsg_sim_config_dict.items(): - print(f"{key}={val}", file=file) - - logger.debug("Printing testInput.properties") - with open("testInput.properties", "w") as file: - print(f"infile_=@./{zipfile_name}", file=file) - - print(f"{zipfile_name} generated") - # uses argv, where the first argument is the script itself, so we must pass - # something as the 0th index of the list - if not dry_run: - if nsgr_submit(["", ".", "validate"]) == 0: - print("Attempting to submit to NSGR") - return nsgr_submit(["", ".", "run"]) - else: - print("Dry run mode enabled. Not submitting to NSG.") - - os.chdir(str(cwd)) + with chdir(str(tdir)): + generated_files = os.listdir(nsg_dir) + + print(f"Generated files are {generated_files}") + + with ZipFile(zipfile_name, "w") as archive: + for f in generated_files: + if engine == "jneuroml_neuron": + if f.endswith("_nrn.py"): + runner_file = f + elif engine == "jneuroml_netpyne": + if f.endswith("_netpyne.py"): + runner_file = f + fpath = pathlib.Path(f) + moved_path = nsg_dir / fpath + archive.write(str(moved_path)) + + logger.debug("Printing testParam.properties") + nsg_sim_config_dict["filename_"] = runner_file + logger.debug(f"NSG sim config is: {nsg_sim_config_dict}") + + with open("testParam.properties", "w") as file: + for key, val in nsg_sim_config_dict.items(): + print(f"{key}={val}", file=file) + + logger.debug("Printing testInput.properties") + with open("testInput.properties", "w") as file: + print(f"infile_=@./{zipfile_name}", file=file) + + print(f"{zipfile_name} generated") + # uses argv, where the first argument is the script itself, so we must pass + # something as the 0th index of the list + if not dry_run: + if nsgr_submit(["", ".", "validate"]) == 0: + print("Attempting to submit to NSGR") + return nsgr_submit(["", ".", "run"]) + else: + print("Dry run mode enabled. Not submitting to NSG.") + return tdir diff --git a/pyneuroml/runners.py b/pyneuroml/runners.py index 37efa376..a8ff65ac 100644 --- a/pyneuroml/runners.py +++ b/pyneuroml/runners.py @@ -30,6 +30,7 @@ import pyneuroml.utils.misc from pyneuroml import DEFAULTS, __version__ from pyneuroml.errors import UNKNOWN_ERR +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -1304,7 +1305,6 @@ def generate_sim_scripts_in_folder( if root_dir is None: root_dir = "." - cwd = Path.cwd() tdir = pyneuroml.utils.get_pyneuroml_tempdir(rootdir=run_dir, prefix="pyneuroml") os.mkdir(tdir) @@ -1315,94 +1315,89 @@ def generate_sim_scripts_in_folder( # change to root_dir, so that we're in the directory where the lems file # is - os.chdir(root_dir) + with chdir(root_dir): + logger.debug("Getting list of model files") + model_file_list = [] # type: list + lems_def_dir = None + lems_def_dir = pyneuroml.utils.get_model_file_list( + lems_file_name, model_file_list, root_dir, lems_def_dir + ) - logger.debug("Getting list of model files") - model_file_list = [] # type: list - lems_def_dir = None - lems_def_dir = pyneuroml.utils.get_model_file_list( - lems_file_name, model_file_list, root_dir, lems_def_dir - ) + logger.debug(f"Model file list is {model_file_list}") + + for model_file in model_file_list: + logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}") + # if model file has directory structures in it, recreate the dirs in + # the temporary directory + if len(model_file.split("/")) > 1: + # throw error if files in parent directories are referred to + if "../" in model_file: + raise ValueError( + """ + Cannot handle parent directories because we + cannot create these directories correctly in + the temporary location. Please re-organize + your code such that all included files are in + sub-directories of the root directory where the + main file resides. + """ + ) - logger.debug(f"Model file list is {model_file_list}") - - for model_file in model_file_list: - logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}") - # if model file has directory structures in it, recreate the dirs in - # the temporary directory - if len(model_file.split("/")) > 1: - # throw error if files in parent directories are referred to - if "../" in model_file: - raise ValueError( - """ - Cannot handle parent directories because we - cannot create these directories correctly in - the temporary location. Please re-organize - your code such that all included files are in - sub-directories of the root directory where the - main file resides. - """ - ) + model_file_path = pathlib.Path(tdir + "/" + model_file) + parent = model_file_path.parent + parent.mkdir(parents=True, exist_ok=True) + shutil.copy(model_file, tdir + "/" + model_file) + + if lems_def_dir is not None: + logger.info(f"Removing LEMS definitions directory {lems_def_dir}") + shutil.rmtree(lems_def_dir) + + with chdir(tdir): + logger.info(f"Working in {tdir}") + start_time = time.time() - 1.0 + + if engine == "jneuroml_neuron": + run_lems_with( + engine, + lems_file_name=Path(lems_file_name).name, + compile_mods=False, + only_generate_scripts=True, + *engine_args, + **engine_kwargs, + ) + elif engine == "jneuroml_netpyne": + run_lems_with( + engine, + lems_file_name=Path(lems_file_name).name, + only_generate_scripts=True, + *engine_args, + **engine_kwargs, + ) - model_file_path = pathlib.Path(tdir + "/" + model_file) - parent = model_file_path.parent - parent.mkdir(parents=True, exist_ok=True) - shutil.copy(model_file, tdir + "/" + model_file) - - if lems_def_dir is not None: - logger.info(f"Removing LEMS definitions directory {lems_def_dir}") - shutil.rmtree(lems_def_dir) - - os.chdir(tdir) - logger.info(f"Working in {tdir}") - start_time = time.time() - 1.0 - - if engine == "jneuroml_neuron": - run_lems_with( - engine, - lems_file_name=Path(lems_file_name).name, - compile_mods=False, - only_generate_scripts=True, - *engine_args, - **engine_kwargs, - ) - elif engine == "jneuroml_netpyne": - run_lems_with( - engine, - lems_file_name=Path(lems_file_name).name, - only_generate_scripts=True, - *engine_args, - **engine_kwargs, + generated_files = pyneuroml.utils.get_files_generated_after( + start_time, ignore_suffixes=["xml", "nml"] ) - generated_files = pyneuroml.utils.get_files_generated_after( - start_time, ignore_suffixes=["xml", "nml"] - ) - - # For NetPyNE, the channels are converted to NEURON mod files, but the - # network and cells are imported from the nml files. - # So we include all the model files too. - if engine == "jneuroml_netpyne": - generated_files.extend(model_file_list) + # For NetPyNE, the channels are converted to NEURON mod files, but the + # network and cells are imported from the nml files. + # So we include all the model files too. + if engine == "jneuroml_netpyne": + generated_files.extend(model_file_list) - logger.debug(f"Generated files are: {generated_files}") + logger.debug(f"Generated files are: {generated_files}") - if generated_files_dir_name is None: - generated_files_dir_name = Path(tdir).name + "_generated" - logger.debug( - f"Creating directory and moving generated files to it: {generated_files_dir_name}" - ) - - for f in generated_files: - fpath = pathlib.Path(f) - moved_path = generated_files_dir_name / fpath - # use os.renames because pathlib.Path.rename does not move - # recursively and so cannot move files within directories - os.renames(fpath, moved_path) + if generated_files_dir_name is None: + generated_files_dir_name = Path(tdir).name + "_generated" + logger.debug( + f"Creating directory and moving generated files to it: {generated_files_dir_name}" + ) - # return to original directory - # doesn't affect scripts much, but does affect our tests - os.chdir(str(cwd)) + for f in generated_files: + fpath = pathlib.Path(f) + moved_path = generated_files_dir_name / fpath + # use os.renames because pathlib.Path.rename does not move + # recursively and so cannot move files within directories + os.renames(fpath, moved_path) return tdir diff --git a/tests/archive/test_archive.py b/tests/archive/test_archive.py index 4cf3af33..3afff42c 100644 --- a/tests/archive/test_archive.py +++ b/tests/archive/test_archive.py @@ -7,7 +7,6 @@ Copyright 2023 NeuroML contributors """ -import contextlib import logging import pathlib import unittest @@ -18,6 +17,7 @@ get_model_file_list, ) from pyneuroml.runners import run_jneuroml +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -50,7 +50,7 @@ def test_get_model_file_list_2a(self): thispath = pathlib.Path(__file__) dirname = str(thispath.parent.parent.parent) - with contextlib.chdir(dirname + "/examples"): + with chdir(dirname + "/examples"): filelist = [] get_model_file_list( "LEMS_NML2_Ex5_DetCell.xml", filelist, dirname + "/examples" diff --git a/tests/test_biosimulations.py b/tests/test_biosimulations.py index 978104c6..3e91b1fa 100644 --- a/tests/test_biosimulations.py +++ b/tests/test_biosimulations.py @@ -8,13 +8,13 @@ """ import logging -import os import pathlib from pyneuroml.biosimulations import ( get_simulator_versions, submit_simulation, ) +from pyneuroml.utils.misc import chdir from . import BaseTestCase @@ -53,20 +53,18 @@ def test_submit_simulation(self): dry_run = True thispath = pathlib.Path(__file__) dirname = str(thispath.parent.parent) - cwd = os.getcwd() - os.chdir(dirname + "/examples") - sim_dict = { - "name": "PyNeuroML test simulation", - "simulator": "neuron", - "simulatorVersion": "latest", - "maxTime": "20", - } + with chdir(dirname + "/examples"): + sim_dict = { + "name": "PyNeuroML test simulation", + "simulator": "neuron", + "simulatorVersion": "latest", + "maxTime": "20", + } - resdict = submit_simulation( - "LEMS_NML2_Ex5_DetCell.xml", sim_dict=sim_dict, dry_run=dry_run - ) - response = resdict["response"] - os.chdir(cwd) + resdict = submit_simulation( + "LEMS_NML2_Ex5_DetCell.xml", sim_dict=sim_dict, dry_run=dry_run + ) + response = resdict["response"] if dry_run: pass diff --git a/tests/test_pynml.py b/tests/test_pynml.py index 3435f9a4..9fe5953c 100644 --- a/tests/test_pynml.py +++ b/tests/test_pynml.py @@ -22,6 +22,7 @@ run_jneuroml, validate_neuroml2, ) +from pyneuroml.utils.misc import chdir logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -73,10 +74,9 @@ def test_exposure_listing(self): def test_exposure_listing_2(self): """Test listing of exposures in NeuroML documents.""" - os.chdir("tests/") - exps = list_exposures("HH_example_net.nml") - print(exps) - os.chdir("../") + with chdir("tests/"): + exps = list_exposures("HH_example_net.nml") + print(exps) def test_recording_path_listing(self): """Test listing of recording paths in NeuroML documents.""" @@ -89,12 +89,11 @@ def test_recording_path_listing(self): def test_recording_path_listing_2(self): """Test listing of recording paths in NeuroML documents.""" - os.chdir("tests/") - paths = list_recording_paths_for_exposures( - "HH_example_net.nml", "hh_cell", "single_hh_cell_network" - ) - print("\n".join(paths)) - os.chdir("../") + with chdir("tests/"): + paths = list_recording_paths_for_exposures( + "HH_example_net.nml", "hh_cell", "single_hh_cell_network" + ) + print("\n".join(paths)) def test_execute_command_in_dir(self): """Test execute_command_in_dir function.""" @@ -152,19 +151,18 @@ def test_run_jneuroml(self): def test_validate_neuroml2(self): """Test validate_neuroml2""" - os.chdir("tests/") - retval = None - retval = validate_neuroml2("HH_example_k_channel.nml") - self.assertTrue(retval) - - retval = None - retstring = None - retval, retstring = validate_neuroml2( - "HH_example_k_channel.nml", return_string=True - ) - self.assertTrue(retval) - self.assertIn("Valid against schema and all tests", retstring) - os.chdir("../") + with chdir("tests/"): + retval = None + retval = validate_neuroml2("HH_example_k_channel.nml") + self.assertTrue(retval) + + retval = None + retstring = None + retval, retstring = validate_neuroml2( + "HH_example_k_channel.nml", return_string=True + ) + self.assertTrue(retval) + self.assertIn("Valid against schema and all tests", retstring) retval = None retstring = None