Skip to content

Fix archiving again: correct logic for paths, use a context manager #459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 6, 2025
Merged

Fix archiving again: correct logic for paths, use a context manager #459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
edb8b9e
chore(archive): add logging
sanjayankur31 Aug 6, 2025
d20cca1
test(archive): add test case
sanjayankur31 Aug 6, 2025
80ca4e1
fix(archive): handle lems.model modifying the include path
sanjayankur31 Aug 6, 2025
7051d55
feat(utils): add `chdir` context manager
sanjayankur31 Aug 6, 2025
1ffa606
feat: use new `chdir` context manager
sanjayankur31 Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 12 additions & 13 deletions pyneuroml/archive/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

import argparse
import logging
import os
import pathlib
import shutil
import typing
Expand All @@ -18,6 +17,7 @@
from pyneuroml.sedml import validate_sedml_files
from pyneuroml.utils import get_model_file_list
from pyneuroml.utils.cli import build_namespace
from pyneuroml.utils.misc import chdir

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
Expand Down Expand Up @@ -170,20 +170,19 @@ def create_combine_archive(
zipfile_name = rootfile

# change to directory of rootfile
thispath = os.getcwd()
os.chdir(rootdir)

lems_def_dir = None
if len(filelist) == 0:
lems_def_dir = get_model_file_list(rootfile, filelist, rootdir, lems_def_dir)
with chdir(rootdir):
lems_def_dir = None
if len(filelist) == 0:
lems_def_dir = get_model_file_list(
rootfile, filelist, rootdir, lems_def_dir
)

create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir)
filelist.append("manifest.xml")
create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir)
filelist.append("manifest.xml")

with ZipFile(zipfile_name + zipfile_extension, "w") as archive:
for f in filelist + extra_files:
archive.write(f)
os.chdir(thispath)
with ZipFile(zipfile_name + zipfile_extension, "w") as archive:
for f in filelist + extra_files:
archive.write(f)

if lems_def_dir is not None:
logger.info(f"Removing LEMS definitions directory {lems_def_dir}")
Expand Down
82 changes: 40 additions & 42 deletions pyneuroml/nsgr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from zipfile import ZipFile

from pyneuroml.runners import generate_sim_scripts_in_folder
from pyneuroml.utils.misc import chdir

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
Expand Down Expand Up @@ -126,8 +127,6 @@ def run_on_nsg(
# NSG requires that the top level directory exist
nsg_dir = pathlib.Path(zipfile_name.replace(".zip", ""))

cwd = pathlib.Path.cwd()

tdir = generate_sim_scripts_in_folder(
engine=engine,
lems_file_name=lems_file_name,
Expand All @@ -139,44 +138,43 @@ def run_on_nsg(
logger.info("Generating zip file")
runner_file = ""

os.chdir(str(tdir))
generated_files = os.listdir(nsg_dir)

print(f"Generated files are {generated_files}")

with ZipFile(zipfile_name, "w") as archive:
for f in generated_files:
if engine == "jneuroml_neuron":
if f.endswith("_nrn.py"):
runner_file = f
elif engine == "jneuroml_netpyne":
if f.endswith("_netpyne.py"):
runner_file = f
fpath = pathlib.Path(f)
moved_path = nsg_dir / fpath
archive.write(str(moved_path))

logger.debug("Printing testParam.properties")
nsg_sim_config_dict["filename_"] = runner_file
logger.debug(f"NSG sim config is: {nsg_sim_config_dict}")

with open("testParam.properties", "w") as file:
for key, val in nsg_sim_config_dict.items():
print(f"{key}={val}", file=file)

logger.debug("Printing testInput.properties")
with open("testInput.properties", "w") as file:
print(f"infile_=@./{zipfile_name}", file=file)

print(f"{zipfile_name} generated")
# uses argv, where the first argument is the script itself, so we must pass
# something as the 0th index of the list
if not dry_run:
if nsgr_submit(["", ".", "validate"]) == 0:
print("Attempting to submit to NSGR")
return nsgr_submit(["", ".", "run"])
else:
print("Dry run mode enabled. Not submitting to NSG.")

os.chdir(str(cwd))
with chdir(str(tdir)):
generated_files = os.listdir(nsg_dir)

print(f"Generated files are {generated_files}")

with ZipFile(zipfile_name, "w") as archive:
for f in generated_files:
if engine == "jneuroml_neuron":
if f.endswith("_nrn.py"):
runner_file = f
elif engine == "jneuroml_netpyne":
if f.endswith("_netpyne.py"):
runner_file = f
fpath = pathlib.Path(f)
moved_path = nsg_dir / fpath
archive.write(str(moved_path))

logger.debug("Printing testParam.properties")
nsg_sim_config_dict["filename_"] = runner_file
logger.debug(f"NSG sim config is: {nsg_sim_config_dict}")

with open("testParam.properties", "w") as file:
for key, val in nsg_sim_config_dict.items():
print(f"{key}={val}", file=file)

logger.debug("Printing testInput.properties")
with open("testInput.properties", "w") as file:
print(f"infile_=@./{zipfile_name}", file=file)

print(f"{zipfile_name} generated")
# uses argv, where the first argument is the script itself, so we must pass
# something as the 0th index of the list
if not dry_run:
if nsgr_submit(["", ".", "validate"]) == 0:
print("Attempting to submit to NSGR")
return nsgr_submit(["", ".", "run"])
else:
print("Dry run mode enabled. Not submitting to NSG.")

return tdir
157 changes: 76 additions & 81 deletions pyneuroml/runners.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import pyneuroml.utils.misc
from pyneuroml import DEFAULTS, __version__
from pyneuroml.errors import UNKNOWN_ERR
from pyneuroml.utils.misc import chdir

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
Expand Down Expand Up @@ -1304,7 +1305,6 @@ def generate_sim_scripts_in_folder(
if root_dir is None:
root_dir = "."

cwd = Path.cwd()
tdir = pyneuroml.utils.get_pyneuroml_tempdir(rootdir=run_dir, prefix="pyneuroml")
os.mkdir(tdir)

Expand All @@ -1315,94 +1315,89 @@ def generate_sim_scripts_in_folder(

# change to root_dir, so that we're in the directory where the lems file
# is
os.chdir(root_dir)
with chdir(root_dir):
logger.debug("Getting list of model files")
model_file_list = [] # type: list
lems_def_dir = None
lems_def_dir = pyneuroml.utils.get_model_file_list(
lems_file_name, model_file_list, root_dir, lems_def_dir
)

logger.debug("Getting list of model files")
model_file_list = [] # type: list
lems_def_dir = None
lems_def_dir = pyneuroml.utils.get_model_file_list(
lems_file_name, model_file_list, root_dir, lems_def_dir
)
logger.debug(f"Model file list is {model_file_list}")

for model_file in model_file_list:
logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}")
# if model file has directory structures in it, recreate the dirs in
# the temporary directory
if len(model_file.split("/")) > 1:
# throw error if files in parent directories are referred to
if "../" in model_file:
raise ValueError(
"""
Cannot handle parent directories because we
cannot create these directories correctly in
the temporary location. Please re-organize
your code such that all included files are in
sub-directories of the root directory where the
main file resides.
"""
)

logger.debug(f"Model file list is {model_file_list}")

for model_file in model_file_list:
logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}")
# if model file has directory structures in it, recreate the dirs in
# the temporary directory
if len(model_file.split("/")) > 1:
# throw error if files in parent directories are referred to
if "../" in model_file:
raise ValueError(
"""
Cannot handle parent directories because we
cannot create these directories correctly in
the temporary location. Please re-organize
your code such that all included files are in
sub-directories of the root directory where the
main file resides.
"""
)
model_file_path = pathlib.Path(tdir + "/" + model_file)
parent = model_file_path.parent
parent.mkdir(parents=True, exist_ok=True)
shutil.copy(model_file, tdir + "/" + model_file)

if lems_def_dir is not None:
logger.info(f"Removing LEMS definitions directory {lems_def_dir}")
shutil.rmtree(lems_def_dir)

with chdir(tdir):
logger.info(f"Working in {tdir}")
start_time = time.time() - 1.0

if engine == "jneuroml_neuron":
run_lems_with(
engine,
lems_file_name=Path(lems_file_name).name,
compile_mods=False,
only_generate_scripts=True,
*engine_args,
**engine_kwargs,
)
elif engine == "jneuroml_netpyne":
run_lems_with(
engine,
lems_file_name=Path(lems_file_name).name,
only_generate_scripts=True,
*engine_args,
**engine_kwargs,
)

model_file_path = pathlib.Path(tdir + "/" + model_file)
parent = model_file_path.parent
parent.mkdir(parents=True, exist_ok=True)
shutil.copy(model_file, tdir + "/" + model_file)

if lems_def_dir is not None:
logger.info(f"Removing LEMS definitions directory {lems_def_dir}")
shutil.rmtree(lems_def_dir)

os.chdir(tdir)
logger.info(f"Working in {tdir}")
start_time = time.time() - 1.0

if engine == "jneuroml_neuron":
run_lems_with(
engine,
lems_file_name=Path(lems_file_name).name,
compile_mods=False,
only_generate_scripts=True,
*engine_args,
**engine_kwargs,
)
elif engine == "jneuroml_netpyne":
run_lems_with(
engine,
lems_file_name=Path(lems_file_name).name,
only_generate_scripts=True,
*engine_args,
**engine_kwargs,
generated_files = pyneuroml.utils.get_files_generated_after(
start_time, ignore_suffixes=["xml", "nml"]
)

generated_files = pyneuroml.utils.get_files_generated_after(
start_time, ignore_suffixes=["xml", "nml"]
)

# For NetPyNE, the channels are converted to NEURON mod files, but the
# network and cells are imported from the nml files.
# So we include all the model files too.
if engine == "jneuroml_netpyne":
generated_files.extend(model_file_list)
# For NetPyNE, the channels are converted to NEURON mod files, but the
# network and cells are imported from the nml files.
# So we include all the model files too.
if engine == "jneuroml_netpyne":
generated_files.extend(model_file_list)

logger.debug(f"Generated files are: {generated_files}")
logger.debug(f"Generated files are: {generated_files}")

if generated_files_dir_name is None:
generated_files_dir_name = Path(tdir).name + "_generated"
logger.debug(
f"Creating directory and moving generated files to it: {generated_files_dir_name}"
)

for f in generated_files:
fpath = pathlib.Path(f)
moved_path = generated_files_dir_name / fpath
# use os.renames because pathlib.Path.rename does not move
# recursively and so cannot move files within directories
os.renames(fpath, moved_path)
if generated_files_dir_name is None:
generated_files_dir_name = Path(tdir).name + "_generated"
logger.debug(
f"Creating directory and moving generated files to it: {generated_files_dir_name}"
)

# return to original directory
# doesn't affect scripts much, but does affect our tests
os.chdir(str(cwd))
for f in generated_files:
fpath = pathlib.Path(f)
moved_path = generated_files_dir_name / fpath
# use os.renames because pathlib.Path.rename does not move
# recursively and so cannot move files within directories
os.renames(fpath, moved_path)

return tdir

Expand Down
28 changes: 19 additions & 9 deletions pyneuroml/utils/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,17 +713,18 @@ def get_model_file_list(
filelist.append(str(fullrootfile_rel))

if str(rootfile_name).endswith(".nml"):
print(f"Processing NML file: {fullrootfile_rel}")
logger.info(f"Processing NML file: {fullrootfile_rel}")
rootdoc = read_neuroml2_file(fullrootfile_rel)
logger.debug(f"Has includes: {rootdoc.includes}")

for inc in rootdoc.includes:
logger.debug(f"Processing includes: {inc.href} in {str(rootdirpath)}")
logger.debug(f"NML: Processing includes: {inc.href} in {str(rootdirpath)}")
lems_def_dir = get_model_file_list(
inc.href, filelist, str(rootdirpath_rel), lems_def_dir
)

elif str(rootfile_name).endswith(".xml"):
logger.info(f"Processing LEMS file: {fullrootfile_rel}")
# extract the standard NeuroML2 LEMS definitions into a directory
# so that the LEMS parser can find them
if lems_def_dir is None:
Expand All @@ -734,17 +735,25 @@ def get_model_file_list(
model.import_from_file(fullrootfile_rel)

for inc in model.included_files:
# `inc` includes the folder name, but we want to keep the file name
# and the directory in which it is located separtely as the
# directory may have to be passed on recursively to other included
# files. So, we separate the name out.
incfile = pathlib.Path(inc).name
logger.debug(f"Processing include file {incfile} ({inc})")
# `inc` includes the complete path relative to the current
# directory, which may repeat information such as the "rootdirpath"
# that we're tracking outselves. So, we need to do some massaging
# here to get the path to inc relative to the rootdirpath_rel
# again
incfile_path = pathlib.Path(inc)
incfile = incfile_path.name

if incfile_path.is_relative_to(rootdirpath_rel):
incfile_path_rel = incfile_path.relative_to(rootdirpath_rel)
else:
incfile_path_rel = incfile_path

logger.debug(f"LEMS: Processing include file {incfile} ({inc})")
if incfile in STANDARD_LEMS_FILES:
logger.debug(f"Ignoring NeuroML2 standard LEMS file: {inc}")
continue
lems_def_dir = get_model_file_list(
incfile, filelist, str(rootdirpath_rel), lems_def_dir
str(incfile_path_rel), filelist, str(rootdirpath_rel), lems_def_dir
)

elif str(rootfile_name).endswith(".sedml"):
Expand All @@ -754,6 +763,7 @@ def get_model_file_list(
logger.error("Please install optional dependencies to use SED-ML features:")
logger.error("pip install pyneuroml[combine]")

logger.info(f"Processing SED-ML file: {fullrootfile_rel}")
rootdoc = libsedml.readSedMLFromFile(fullrootfile_rel)

# there should only be one model
Expand Down
Loading
Loading