diff --git a/examples/example_PyGromosPP.ipynb b/examples/developer_examples/dev_example_PyGromosPP.ipynb similarity index 100% rename from examples/example_PyGromosPP.ipynb rename to examples/developer_examples/dev_example_PyGromosPP.ipynb diff --git a/examples/example_gromos_files.ipynb b/examples/developer_examples/dev_example_gromos_files.ipynb similarity index 100% rename from examples/example_gromos_files.ipynb rename to examples/developer_examples/dev_example_gromos_files.ipynb diff --git a/examples/tutorial_gromos_pipeline_old.ipynb b/examples/developer_examples/dev_example_gromos_pipeline_low_level.ipynb similarity index 100% rename from examples/tutorial_gromos_pipeline_old.ipynb rename to examples/developer_examples/dev_example_gromos_pipeline_low_level.ipynb diff --git a/examples/example_gromos_simulation.ipynb b/examples/developer_examples/dev_example_gromos_simulation.ipynb similarity index 100% rename from examples/example_gromos_simulation.ipynb rename to examples/developer_examples/dev_example_gromos_simulation.ipynb diff --git a/examples/example_gromos_trajectories.ipynb b/examples/developer_examples/dev_example_gromos_trajectories.ipynb similarity index 100% rename from examples/example_gromos_trajectories.ipynb rename to examples/developer_examples/dev_example_gromos_trajectories.ipynb diff --git a/examples/example_submission_systems.ipynb b/examples/developer_examples/dev_example_submission_systems.ipynb similarity index 100% rename from examples/example_submission_systems.ipynb rename to examples/developer_examples/dev_example_submission_systems.ipynb diff --git a/examples/example_topology_creation.ipynb b/examples/developer_examples/dev_example_topology_creation.ipynb similarity index 100% rename from examples/example_topology_creation.ipynb rename to examples/developer_examples/dev_example_topology_creation.ipynb diff --git a/examples/example_gromos_new_submission_system.ipynb b/examples/example_gromos_on_HPC_Clusters.ipynb similarity index 100% rename from examples/example_gromos_new_submission_system.ipynb rename to examples/example_gromos_on_HPC_Clusters.ipynb diff --git a/examples/example_hvap.ipynb b/examples/example_hvap.ipynb index 819c9c54..325a60f9 100644 --- a/examples/example_hvap.ipynb +++ b/examples/example_hvap.ipynb @@ -17,7 +17,7 @@ "source": [ "from pygromos.files.gromos_system.ff.forcefield_system import forcefield_system\n", "from pygromos.files.gromos_system.gromos_system import Gromos_System\n", - "from pygromos.simulations.hvap_calculation.hvap_calculation import Hvap_calculation\n", + "from pygromos.simulations.approaches.hvap_calculation.hvap_calculation import Hvap_calculation\n", "from pygromos.data.simulation_parameters_templates import template_sd" ] }, @@ -27,7 +27,9 @@ "metadata": {}, "outputs": [], "source": [ - "pygro_env={'SHELL': '/bin/bash', 'LIBGL_ALWAYS_INDIRECT': '1', 'CONDA_EXE': '/home/mlehner/anaconda3/bin/conda', '_CE_M': '', 'WSL_DISTRO_NAME': 'Ubuntu-20.04', 'NAME': 'MarcSurface', 'GSETTINGS_SCHEMA_DIR': '/home/mlehner/anaconda3/envs/pygro/share/glib-2.0/schemas', 'LOGNAME': 'mlehner', 'CONDA_PREFIX': '/home/mlehner/anaconda3/envs/pygro', 'GSETTINGS_SCHEMA_DIR_CONDA_BACKUP': '', 'HOME': '/home/mlehner', 'LANG': 'C.UTF-8', 'WSL_INTEROP': '/run/WSL/451_interop', 'CONDA_PROMPT_MODIFIER': '(pygro) ', 'PERL5LIB': '/home/mlehner/anaconda3/envs/pygro/lib/perl/mm_pbsa', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'TERM': 'xterm-256color', '_CE_CONDA': '', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'USER': 'mlehner', 'CONDA_SHLVL': '2', 'AMBERHOME': '/home/mlehner/anaconda3/envs/pygro', 'DISPLAY': '172.21.240.1:0', 'SHLVL': '1', 'CONDA_PYTHON_EXE': '/home/mlehner/anaconda3/bin/python', 'CONDA_DEFAULT_ENV': 'pygro', 'WSLENV': '', 'XDG_DATA_DIRS': '/usr/local/share:/usr/share:/var/lib/snapd/desktop', 'PATH': '/home/mlehner/anaconda3/envs/pygro/bin:/home/mlehner/anaconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/mnt/c/Program Files/WindowsApps/CanonicalGroupLimited.Ubuntu20.04onWindows_2004.2021.222.0_x64__79rhkp1fndgsc:/mnt/c/WINDOWS/system32:/mnt/c/WINDOWS:/mnt/c/WINDOWS/System32/Wbem:/mnt/c/WINDOWS/System32/WindowsPowerShell/v1.0:/mnt/c/WINDOWS/System32/OpenSSH:/mnt/c/Program Files/NVIDIA Corporation/NVIDIA NvDLISR:/mnt/c/Program Files (x86)/NVIDIA Corporation/PhysX/Common:/mnt/c/Program Files/Wolfram Research/WolframScript:/mnt/c/Users/thier/AppData/Local/Microsoft/WindowsApps:/mnt/c/Users/thier/AppData/Local/Programs/Microsoft VS Code/bin:/mnt/c/Users/thier/AppData/Local/GitHubDesktop/bin:/snap/bin:/usr/local/gromacs/bin/GMXRC', 'HOSTTYPE': 'x86_64', 'CONDA_PREFIX_1': '/home/mlehner/anaconda3', '_': '/home/mlehner/anaconda3/envs/pygro/bin/python3'}" + "work_dir=os.getcwd()+\"/example_files/Hvap_files/\"\n", + "gromosXX_path=None \n", + "gromosPP_path=None" ] }, { @@ -36,9 +38,8 @@ "metadata": {}, "outputs": [], "source": [ - "work_dir=os.getcwd()+\"/example_files/Hvap_files/\"\n", - "gromosXX_path=None #\"/home/mlehner/gromosXX/gromosXX/BUILD/program/\"\n", - "gromosPP_path=None# \"/home/mlehner/gromosPlsPls/gromos++/BUILD/programs/\"" + "ff = forcefield_system(name=\"54A7\")\n", + "ff.mol_name = \"H2O\"" ] }, { @@ -47,8 +48,9 @@ "metadata": {}, "outputs": [], "source": [ - "ff = forcefield_system(name=\"54A7\")\n", - "ff.mol_name = \"H2O\"" + "groSys = Gromos_System(work_folder=work_dir+\"init/\", system_name=\"Hvap_test\", in_smiles=\"O\",\n", + " auto_convert=True, Forcefield=ff, in_imd_path=template_sd,\n", + " in_gromosPP_bin_dir=gromosPP_path, in_gromosXX_bin_dir=gromosXX_path)" ] }, { @@ -57,7 +59,7 @@ "metadata": {}, "outputs": [], "source": [ - "groSys = Gromos_System(work_folder=work_dir+\"init/\", system_name=\"Hvap_test\", in_smiles=\"O\", auto_convert=True, Forcefield=ff, in_imd_path=template_sd, in_gromosPP_bin_dir=gromosPP_path, in_gromosXX_bin_dir=gromosXX_path)" + "hvap_sys=Hvap_calculation(input_system=groSys, work_folder=work_dir+\"hvap\", forcefield=ff, system_name=\"test\", useGromosPlsPls=True)" ] }, { @@ -66,7 +68,7 @@ "metadata": {}, "outputs": [], "source": [ - "hvap_sys=Hvap_calculation(input_system=groSys, work_folder=work_dir+\"hvap\", forcefield=ff, system_name=\"test\")" + "hvap_sys.create_liq()" ] }, { @@ -75,7 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "hvap_sys.submissonSystem._enviroment = pygro_env" + "hvap_sys.run_gas()" ] }, { @@ -84,7 +86,7 @@ "metadata": {}, "outputs": [], "source": [ - "hvap_sys.create_liq()" + "hvap_sys.run_liq()" ] }, { @@ -93,16 +95,14 @@ "metadata": {}, "outputs": [], "source": [ - "hvap_sys.run_gas()" + "hvap_sys.calc_hvap()\n" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "hvap_sys.run_liq()" + "## How to deactivate binary checks:" ] }, { @@ -111,7 +111,17 @@ "metadata": {}, "outputs": [], "source": [ - "hvap_sys.calc_hvap()" + "#GromosLayer\n", + "from pygromos.gromos.gromosXX import GromosXX\n", + "\n", + "grom = GromosXX(_dont_check_binary=True)\n", + "\n", + "\n", + "#System Layer\n", + "from pygromos.files.gromos_system.gromos_system import Gromos_System\n", + "\n", + "Gromos_System._gromos_noBinary_checks = True\n", + "sys = Gromos_System()" ] }, { @@ -123,8 +133,25 @@ } ], "metadata": { + "interpreter": { + "hash": "b1b7b2ea43b8e767316eee98e01335d045804d2d47db68b6a5827e187ee91a7e" + }, + "kernelspec": { + "display_name": "Python 3.9.7 ('pygro2')", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" } }, "nbformat": 4, diff --git a/examples/tutorial_gromos_pipeline.ipynb b/examples/tutorial_gromos_pipeline.ipynb index 5b8a3a24..7d885466 100644 --- a/examples/tutorial_gromos_pipeline.ipynb +++ b/examples/tutorial_gromos_pipeline.ipynb @@ -921,6 +921,7 @@ "# Check simulation params\n", "in_eminBox_system.imd = template_emin #Here we use template simulation parameters, The blocks are the same as above in the vacuum case with slight deviations.\n", "in_eminBox_system.imd.INITIALISE.NTISHI = 1\n", + "\n", "in_eminBox_system.prepare_for_simulation()\n", "\n", "in_eminBox_system" @@ -1565,8 +1566,8 @@ "source": [ "from matplotlib import pyplot as plt\n", "\n", - "plt.plot(temperatures.TIMESTEP_time, temperatures.bath1, label=\"TBath1\")\n", - "plt.plot(temperatures.TIMESTEP_time, temperatures.bath2, label=\"TBath2\")\n", + "plt.plot(temperatures.index, temperatures.bath1, label=\"TBath1\")\n", + "plt.plot(temperatures.index, temperatures.bath2, label=\"TBath2\")\n", "\n", "plt.legend()\n", "plt.ylabel(\"$T~[K]$\")\n", @@ -1774,7 +1775,7 @@ "source": [ "#Next we plot some data of the energies to se their development.\n", "\n", - "time_axis = energy_traj.database.TIMESTEP_time\n", + "time_axis = energy_traj.time\n", "total_energies.totene.plot(x=time_axis, xlabel=\"t [ps]\", ylabel=\"V [kJ/mol]\", title=\"Simulation Data\", legend=True)\n", "total_energies.totpot.plot(legend=True)\n", "total_energies.totkin.plot(legend=True)\n" @@ -1815,7 +1816,7 @@ "outputs": [], "source": [ "#First lets get the Force Group Energy contributions:\n", - "forceGroupNonbondedContributions = energy_traj.get_nonbondedForceGroupContributions()\n", + "forceGroupNonbondedContributions = energy_traj.get_nonbondedContributions()\n", "\n", "#Here we give each Force Group contribution category nice names:\n", "peptide_peptide_nonbonded = forceGroupNonbondedContributions[1][1]\n", @@ -1894,13 +1895,6 @@ "coordinate_traj = out_md_system.trc\n" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### RMSD" - ] - }, { "cell_type": "code", "execution_count": null, @@ -1982,8 +1976,25 @@ } ], "metadata": { + "interpreter": { + "hash": "e7d5b70120806d05daeaf98b799438b40f063b8cf99696cad03a40d92a237582" + }, + "kernelspec": { + "display_name": "Python 3.9.10 ('pygromosDev')", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" } }, "nbformat": 4, diff --git a/pygromos/files/_basics/_general_gromos_file.py b/pygromos/files/_basics/_general_gromos_file.py index e17661ff..d151b891 100644 --- a/pygromos/files/_basics/_general_gromos_file.py +++ b/pygromos/files/_basics/_general_gromos_file.py @@ -8,7 +8,7 @@ import copy import inspect import warnings -from typing import List, Dict, Callable +from typing import List, Dict, Callable, Union from pygromos.files.blocks import all_blocks @@ -30,7 +30,7 @@ class _general_gromos_file: _block_order: List[str] = [] _future_file: bool - def __init__(self, in_value: (str or dict or None or __class__), _future_file: bool = False): + def __init__(self, in_value: Union[str, dict, None], _future_file: bool = False): self._future_file = _future_file if isinstance(in_value, str): self.path = self._orig_file_path = in_value @@ -43,10 +43,12 @@ def __init__(self, in_value: (str or dict or None or __class__), _future_file: b elif isinstance(type(in_value), __class__): raise NotImplementedError("This variant is not implemented") + elif in_value is None: self.path = None self._orig_file_path = None # print("Empty class") + else: raise ValueError("The given type of input could not be translated in " + str(__class__) + ".__init__") @@ -302,11 +304,9 @@ def write(self, out_path: str) -> str: str out_path """ - os.makedirs(os.path.dirname(out_path), exist_ok=True) - file = open(out_path, "w") - file.write(str(self)) - file.close() - self.path = out_path + self._write_to_file(out_path=out_path, content_str=str(self)) + self.path = os.path.abspath(out_path) + self._future_file = False return out_path diff --git a/pygromos/files/blocks/_general_blocks.py b/pygromos/files/blocks/_general_blocks.py index 2e2cd130..fbd494e7 100644 --- a/pygromos/files/blocks/_general_blocks.py +++ b/pygromos/files/blocks/_general_blocks.py @@ -88,14 +88,18 @@ def read_content_from_str(self, content: str): def block_to_string(self) -> str: result = self.name + self.line_seperator - if isinstance(self.content, list) and len(self.content) > 0 and all([isinstance(x, str) for x in self.content]): - result += self.field_seperator.join(self.content) + self.line_seperator + if ( + isinstance(self.content, list) + and len(self.content) > 0 + and all([isinstance(x, (str, Number)) for x in self.content]) + ): + result += self.field_seperator.join(map(str, self.content)) + self.line_seperator elif ( isinstance(self.content, list) and len(self.content) > 0 - and all([isinstance(x, list) and all([isinstance(y, str) for y in x]) for x in self.content]) + and all([isinstance(x, list) and all([isinstance(y, (str, Number)) for y in x]) for x in self.content]) ): - result += self.line_seperator.join(map(lambda x: self.field_seperator.join(x), self.content)) + result += self.line_seperator.join(map(lambda x: self.field_seperator.join(map(str, x)), self.content)) elif isinstance(self.content, (str, Number)): result += self.field_seperator + str(self.content) + self.line_seperator else: diff --git a/pygromos/files/blocks/topology_blocks.py b/pygromos/files/blocks/topology_blocks.py index be0fcb3c..26a7c1ea 100644 --- a/pygromos/files/blocks/topology_blocks.py +++ b/pygromos/files/blocks/topology_blocks.py @@ -3027,34 +3027,68 @@ def block_to_string(self) -> str: return result -class SOLUTEMOLECULES(_topology_block): +class _generic_topology_groups(_topology_block): + NSM: int + NSP: List[int] + def __init__( self, - content: (str or dict or None or __class__), + content: Union[str, dict] = None, FORCEFIELD: FORCEFIELD = None, MAKETOPVERSION: MAKETOPVERSION = None, + NSM: int = None, + NSP: List[int] = None, ): - super().__init__(FORCEFIELD=FORCEFIELD, MAKETOPVERSION=MAKETOPVERSION, content=content) + if NSP is not None: + if NSM is not None: + if len(NSP) == NSM: + self.NSM = NSM + else: + raise ValueError("FUn") + else: + self.NSM = len(NSP) + self.NSP = list(map(int, NSP)) + super().__init__( + FORCEFIELD=FORCEFIELD, MAKETOPVERSION=MAKETOPVERSION, content=[str(NSM)] + list(map(str, NSP)) + ) + else: + super().__init__(FORCEFIELD=FORCEFIELD, MAKETOPVERSION=MAKETOPVERSION, content=content) + + if len(self.content) == 1 and len(self.content[0]) - 1 == int(self.content[0][0]): + self.NSM = int(self.content[0][0]) + self.NSP = [int(x) for x in self.content[0][1:]] + elif len(self.content) > 1: + self.NSM = int(self.content[0][0]) + self.NSP = [] + [ + self.NSP.extend(list(map(int, t))) if (isinstance(t, list)) else self.NSP.extend([int(t)]) + for t in self.content[1:] + ] + else: + raise ValueError("SOLUTEMOLECULES has not the correct number of fields.") + # Clean COntent + self.content = [[self.NSM]] + self.content.extend([[x] for x in self.NSP]) -class TEMPERATUREGROUPS(_topology_block): - def __init__( - self, - content: (str or dict or None or __class__), - FORCEFIELD: FORCEFIELD = None, - MAKETOPVERSION: MAKETOPVERSION = None, - ): - super().__init__(FORCEFIELD=FORCEFIELD, MAKETOPVERSION=MAKETOPVERSION, content=content) + def block_to_string(self) -> str: + # Clean COntent + self.content = [[self.NSM]] + self.content.extend([[x] for x in self.NSP]) + return super().block_to_string() -class PRESSUREGROUPS(_topology_block): - def __init__( - self, - content: (str or dict or None or __class__), - FORCEFIELD: FORCEFIELD = None, - MAKETOPVERSION: MAKETOPVERSION = None, - ): - super().__init__(FORCEFIELD=FORCEFIELD, MAKETOPVERSION=MAKETOPVERSION, content=content) + +class SOLUTEMOLECULES(_generic_topology_groups): + pass + + +class TEMPERATUREGROUPS(_generic_topology_groups): + pass + + +class PRESSUREGROUPS(_generic_topology_groups): + pass class LJEXCEPTIONS(_topology_table_block): diff --git a/pygromos/files/coord/cnf.py b/pygromos/files/coord/cnf.py index f9723684..39d64812 100644 --- a/pygromos/files/coord/cnf.py +++ b/pygromos/files/coord/cnf.py @@ -1,5 +1,4 @@ import copy -import os from collections import namedtuple, defaultdict from typing import Dict, List, Tuple, Union @@ -64,7 +63,7 @@ class Cnf(_general_gromos_file): def __init__( self, - in_value: (str or dict or None or __class__), + in_value: Union[str, dict, None], clean_resiNumbers_by_Name=False, verbose: bool = False, _future_file: bool = False, @@ -102,14 +101,6 @@ def read_file(self) -> Dict[str, any]: """ return parser.read_cnf(self._orig_file_path) - def write(self, out_path: str) -> str: - # write out - out_file = open(out_path, "w") - out_file.write(self.__str__()) - out_file.close() - self.path = os.path.abspath(out_path) - return out_path - """ manipulate/analysis of coordinates """ diff --git a/pygromos/files/gromos_system/ff/openforcefield2gromos.py b/pygromos/files/gromos_system/ff/openforcefield2gromos.py index ae224f87..edb4bf28 100644 --- a/pygromos/files/gromos_system/ff/openforcefield2gromos.py +++ b/pygromos/files/gromos_system/ff/openforcefield2gromos.py @@ -10,6 +10,7 @@ # imports import importlib from pygromos.files.topology.top import Top +from pygromos.files.blocks import topology_blocks as blocks from pygromos.files.gromos_system.ff.forcefield_system import forcefield_system @@ -283,17 +284,20 @@ def convertVdW(self): def convert_other_stuff(self): if not hasattr(self.gromosTop, "SOLUTEMOLECULES"): - self.gromosTop.add_block(blocktitle="SOLUTEMOLECULES", content=["1", str(self.openFFmolecule.n_atoms)]) + self.gromosTop.add_block(block=blocks.SOLUTEMOLECULES(NSM=1, NSP=[self.openFFmolecule.n_atoms])) else: - self.gromosTop.SOLUTEMOLECULES.content = [["1"], [str(self.openFFmolecule.n_atoms)]] + self.gromosTop.SOLUTEMOLECULES.NSM = 1 + self.gromosTop.SOLUTEMOLECULES.NSP = [self.openFFmolecule.n_atoms] if not hasattr(self.gromosTop, "TEMPERATUREGROUPS"): - self.gromosTop.add_block(blocktitle="TEMPERATUREGROUPS", content=["1", str(self.openFFmolecule.n_atoms)]) + self.gromosTop.add_block(block=blocks.TEMPERATUREGROUPS(NSM=1, NSP=[self.openFFmolecule.n_atoms])) else: - self.gromosTop.TEMPERATUREGROUPS.content = [["1"], [str(self.openFFmolecule.n_atoms)]] + self.gromosTop.TEMPERATUREGROUPS.NSM = 1 + self.gromosTop.TEMPERATUREGROUPS.NSP = [self.openFFmolecule.n_atoms] if not hasattr(self.gromosTop, "PRESSUREGROUPS"): - self.gromosTop.add_block(blocktitle="PRESSUREGROUPS", content=["1", str(self.openFFmolecule.n_atoms)]) + self.gromosTop.add_block(block=blocks.PRESSUREGROUPS(NSM=1, NSP=[self.openFFmolecule.n_atoms])) else: - self.gromosTop.PRESSUREGROUPS.content = [["1"], [str(self.openFFmolecule.n_atoms)]] + self.gromosTop.PRESSUREGROUPS.NSM = 1 + self.gromosTop.PRESSUREGROUPS.NSP = [self.openFFmolecule.n_atoms] if not hasattr(self.gromosTop, "LJEXCEPTIONS"): self.gromosTop.add_block(blocktitle="LJEXCEPTIONS", content=["0", ""]) if not hasattr(self.gromosTop, "SOLVENTATOM"): diff --git a/pygromos/files/gromos_system/gromos_system.py b/pygromos/files/gromos_system/gromos_system.py index 63766c9d..f7e6cf0f 100644 --- a/pygromos/files/gromos_system/gromos_system.py +++ b/pygromos/files/gromos_system/gromos_system.py @@ -13,14 +13,16 @@ """ # imports +import io import os import copy +import pickle import inspect import functools import importlib import warnings from pygromos.data.ff import Gromos54A7 -from typing import Dict, Union, List, Callable +from typing import Dict, Union, List from pygromos.files._basics._general_gromos_file import _general_gromos_file from pygromos.files.blocks import imd_blocks @@ -39,8 +41,6 @@ from pygromos.gromos import GromosXX, GromosPP from pygromos.utils import bash, utils -import pickle -import io if importlib.util.find_spec("rdkit") is not None: from rdkit import Chem @@ -68,12 +68,7 @@ "non_ligand_info": cnf.non_ligand_infos, "solvent_info": cnf.solvent_infos, } -exclude_pickle = { - "GromosPP": GromosPP, - "GromosXX": GromosXX, - "_gromosPP": GromosPP, - "_gromosXX": GromosXX, -} +exclude_pickle = {} class Gromos_System: @@ -91,13 +86,19 @@ class Gromos_System: protein_info: cnf.protein_infos non_ligand_info: cnf.non_ligand_infos solvent_info: cnf.solvent_infos - checkpoint_path: str + + # privates + _checkpoint_path: str + _last_jobID: int # hpcScheduling ID of the last submitted job. _future_promise: bool # for interest if multiple jobs shall be chained. _future_promised_files: list _single_multibath: bool _single_energy_group: bool + # if this class Variable is set to True, all binary checks will be removed from the systems. + _gromos_binary_checks: bool = True + _gromosPP_bin_dir: Union[None, str] _gromosXX_bin_dir: Union[None, str] _gromosPP: GromosPP @@ -131,6 +132,10 @@ def __init__( With this class all files can be read-in or the files can be automatically generated from smiles. Additionally to that can all gromos++ functions be used from the Gromos System, so system generation can be easily accomplished. + if you want to remove all binary checks, do the following: + >>> from pygromos.files.gromos_system.gromos_system import Gromos_System + >>> Gromos_System._gromos_noBinary_checks = True + >>> sys = Gromos_System() Parameters ---------- @@ -188,7 +193,8 @@ def __init__( self.Forcefield = Forcefield self.mol = Chem.Mol() self.in_mol2_file = in_mol2_file - self.checkpoint_path = None + self._checkpoint_path = None + self._last_jobID = None self.adapt_imd_automatically = adapt_imd_automatically self.verbose = verbose @@ -262,13 +268,14 @@ def __init__( if self.hasData: self.auto_convert() else: - raise Warning("auto_convert active but no data provided -> auto_convert NOT done!") + warnings.warn("auto_convert active but no data provided -> auto_convert NOT done!") if in_cnf_path is None and type(self.mol) == Chem.rdchem.Mol and self.mol.GetNumAtoms() >= 1: self.cnf = Cnf(in_value=self.mol) # TODO: fix ugly workaround for cnf from rdkit with GROMOS FFs + if self.Forcefield.name == "2016H66" or self.Forcefield.name == "54A7": - if self.gromosPP is not None and self.gromosPP._found_binary["pdb2g96"]: + if self.gromosPP is not None and self.gromosPP._check_binary("pdb2g96"): try: from pygromos.files.blocks.coord_blocks import atomP @@ -286,10 +293,10 @@ def __init__( ] self.cnf.POSITION = new_pos self.cnf.write_pdb(self.work_folder + "/tmp.pdb") - self.pdb2gromos(self.work_folder + "/tmp.pdb") + self.pdb2gromos(in_pdb_path=self.work_folder + "/tmp.pdb") self.add_hydrogens() except IOError: - raise Warning("Could not convert cnf from rdkit to gromos, will use rdkit cnf") + warnings.warn("Could not convert cnf from rdkit to gromos, will use rdkit cnf") # decide if the imd should be adapted (force groups etc.) # assert if the respective option is activated and cnf/imd files do actually exist @@ -307,7 +314,7 @@ def __str__(self) -> str: msg += "GROMOS SYSTEM: " + self.name + "\n" msg += utils.spacer msg += "WORKDIR: " + self._work_folder + "\n" - msg += "LAST CHECKPOINT: " + str(self.checkpoint_path) + "\n" + msg += "LAST CHECKPOINT: " + str(self._checkpoint_path) + "\n" msg += "\n" msg += "GromosXX_bin: " + str(self.gromosXX_bin_dir) + "\n" msg += "GromosPP_bin: " + str(self.gromosPP_bin_dir) + "\n" @@ -407,12 +414,11 @@ def __getstate__(self): attribute_dict = self.__dict__ new_dict = {} for key in attribute_dict.keys(): - if not isinstance(attribute_dict[key], Callable) and key not in skip and key not in exclude_pickle: + if not callable(attribute_dict[key]) and key not in skip and key not in exclude_pickle: new_dict.update({key: attribute_dict[key]}) elif attribute_dict[key] is not None and key in skip and key not in exclude_pickle: new_dict.update({key: attribute_dict[key]._asdict()}) else: - print("STUPID ", key) new_dict.update({key: None}) return new_dict @@ -428,8 +434,8 @@ def __setstate__(self, state): self._all_files = copy.copy(self.required_files) self._all_files.update(copy.copy(self.optional_files)) - self._gromosPP = GromosPP(self._gromosPP_bin_dir) - self._gromosXX = GromosXX(self._gromosXX_bin_dir) + self._gromosPP = GromosPP(self._gromosPP_bin_dir, _check_binary_paths=self._gromos_binary_checks) + self._gromosXX = GromosXX(self._gromosXX_bin_dir, _check_binary_paths=self._gromos_binary_checks) self.__bind_gromosPPFuncs() @@ -705,10 +711,11 @@ def gromosXX(self) -> GromosXX: @gromosXX.setter def gromosXX(self, input_value: Union[str, GromosXX]): if isinstance(input_value, str) or input_value is None: - self._gromosXX = GromosXX(gromosXX_bin_dir=input_value) + self._gromosXX = GromosXX(gromosXX_bin_dir=input_value, _check_binary_paths=self._gromos_binary_checks) self._gromosXX_bin_dir = input_value elif isinstance(input_value, GromosXX): self._gromosXX = input_value + self._gromosXX._check_binary_paths = self._gromos_binary_checks self._gromosXX_bin_dir = input_value.bin else: raise ValueError(f"Could not parse input type: {str(type(input_value))} {str(input_value)}") @@ -720,10 +727,11 @@ def gromosPP(self) -> GromosPP: @gromosPP.setter def gromosPP(self, input_value: Union[str, GromosPP]): if isinstance(input_value, str) or input_value is None: - self._gromosPP = GromosPP(gromosPP_bin_dir=input_value) + self._gromosPP = GromosPP(gromosPP_bin_dir=input_value, _check_binary_paths=self._gromos_binary_checks) self._gromosPP_bin_dir = input_value elif isinstance(input_value, GromosPP): self._gromosPP = input_value + self._gromosPP._check_binary_paths = self._gromos_binary_checks self._gromosPP_bin_dir = input_value.bin else: raise ValueError(f"Could not parse input type: {str(type(input_value))} {str(input_value)}") @@ -741,6 +749,7 @@ def get_script_generation_command(self, var_name: str = None, var_prefixes: str var_name = var_prefixes + self.__class__.__name__.lower() gen_cmd = "#Generate " + self.__class__.__name__ + "\n" + gen_cmd = "\n" gen_cmd += ( "from " + self.__module__ @@ -748,14 +757,19 @@ def get_script_generation_command(self, var_name: str = None, var_prefixes: str + self.__class__.__name__ + " as " + self.__class__.__name__ - + "_obj" + + "_class" + "\n" ) + if hasattr(self, "_gromos_binary_checks") and not self._gromos_binary_checks: + gen_cmd += ( + self.__class__.__name__ + "_class._gromos_binary_checks = " + str(self._gromos_binary_checks) + "\n" + ) + gen_cmd += ( var_name + " = " + __class__.__name__ - + '_obj(work_folder="' + + '_class(work_folder="' + self.work_folder + '", system_name="' + self.name @@ -764,6 +778,13 @@ def get_script_generation_command(self, var_name: str = None, var_prefixes: str for arg, path in self.all_file_paths.items(): gen_cmd += str(var_name) + "." + str(arg) + ' = "' + str(path) + '"\n' + + if self.gromosXX_bin_dir is not None: + gen_cmd += str(var_name) + ".gromosXX_bin_dir = '" + str(self.gromosXX_bin_dir) + "'\n" + + if self.gromosPP_bin_dir is not None: + gen_cmd += str(var_name) + ".gromosPP_bin_dir = '" + str(self.gromosXX_bin_dir) + "'\n" + gen_cmd += "\n" return gen_cmd @@ -859,7 +880,6 @@ def auto_convert(self): # create topology if self.Forcefield.name == "2016H66" or self.Forcefield.name == "54A7": # set parameters for make_top - out = self.work_folder + "/make_top.top" mtb_temp = self.Forcefield.mtb_path if hasattr(self.Forcefield, "mtb_orga_path"): mtb_temp += " " + self.Forcefield.mtb_orga_path @@ -868,19 +888,16 @@ def auto_convert(self): name = self.rdkit2GromosName() else: name = self.Forcefield.mol_name + # make top - if bash.command_exists(f"{self.gromosPP_bin_dir}/make_top"): - self.gromosPP.make_top( - out_top_path=out, + if "make_top" in self.gromosPP._found_binary and self.gromosPP._found_binary["make_top"]: + self.make_top( in_building_block_lib_path=mtb_temp, in_parameter_lib_path=ifp_temp, in_sequence=name, ) - self.top = Top(in_value=out) else: - warnings.warn( - "could not find a gromosPP version. Please provide a valid version for Gromos auto system generation" - ) + warnings.warn("Could not use make_top! no binary was found!") elif ( self.Forcefield.name == "smirnoff" @@ -1182,7 +1199,7 @@ def save(self, path: Union[str, io.FileIO] = None, safe: bool = True) -> str: if not safe_skip: pickle.dump(obj=self, file=bufferdWriter) bufferdWriter.close() - self.checkpoint_path = path + self._checkpoint_path = path return path @classmethod @@ -1208,7 +1225,7 @@ def load(cls, path: Union[str, io.FileIO] = None) -> object: obj.non_ligand_info, obj.solvent_info, ) = obj._cnf.get_system_information() - obj.checkpoint_path = path + obj._checkpoint_path = path return obj """ @@ -1234,8 +1251,8 @@ def __SystemConstructionAttributeFinder(self, func: callable) -> callable: """ @functools.wraps(func) - def findGromosSystemAttributes(*args, **kwargs): - # print(func.__name__, args, kwargs) + def _findGromosSystemAttributes(*args, **kwargs): + # print("attribute finder", func.__name__, args, kwargs) # collect input parameters present in system/ replace them with tmp_files = [] @@ -1257,7 +1274,10 @@ def findGromosSystemAttributes(*args, **kwargs): grom_obj.write(grom_obj.path) # make sure filestatus is good :) kwargs.update({k: grom_obj.path}) + args = list(filter(lambda x: isinstance(x, self.gromosPP.__class__), args)) + # execute function + # print("attibuteFinder 2:", func.__name__, args, kwargs) r = func(self.gromosPP, *args, **kwargs) # remove tmp_files @@ -1275,9 +1295,9 @@ def findGromosSystemAttributes(*args, **kwargs): else: red_params.append(par) red_sig = sig.replace(parameters=red_params) - findGromosSystemAttributes.__signature__ = red_sig + _findGromosSystemAttributes.__signature__ = red_sig - return findGromosSystemAttributes + return _findGromosSystemAttributes def __SystemConstructionUpdater(self, func: callable) -> callable: """ @@ -1297,8 +1317,8 @@ def __SystemConstructionUpdater(self, func: callable) -> callable: """ @functools.wraps(func) - def updateGromosSystem(*args, **kwargs): - # rint(func.__name__, args, kwargs) + def _updateGromosSystem(*args, **kwargs): + # print("updater", func.__name__, args, kwargs) # collect out_paths update_dict = {} @@ -1309,7 +1329,9 @@ def updateGromosSystem(*args, **kwargs): update_dict.update({k: attr_key}) # execute function + # print("updater2", func.__name__, args, kwargs) r = func(*args, **kwargs) + # print("updater3", func.__name__, args, kwargs) # update attribute states and remove tmp files. for k in update_dict: @@ -1328,9 +1350,9 @@ def updateGromosSystem(*args, **kwargs): else: red_params.append(par) red_sig = sig.replace(parameters=red_params) - updateGromosSystem.__signature__ = red_sig + _updateGromosSystem.__signature__ = red_sig - return updateGromosSystem + return _updateGromosSystem def __ionDecorator(self, func: callable) -> callable: """ @@ -1373,7 +1395,7 @@ def generate_ion_top( out_top_path=top_cl, ) self.top += Top(top_cl) - bash.remove_file(top_cl) + # bash.remove_file(top_cl) return r diff --git a/pygromos/files/topology/top.py b/pygromos/files/topology/top.py index 1d9b7d4d..f0f7a3c6 100644 --- a/pygromos/files/topology/top.py +++ b/pygromos/files/topology/top.py @@ -230,16 +230,16 @@ def _add_top(self, top: Union[TopType, None], solvFrom1: bool = True, verbose: b ) # add SOLUTEMOLECULES - for solmol in top.SOLUTEMOLECULES.content[1:]: - retTop.add_new_SOLUTEMOLECULES(number=str(int(solmol[0]) + atnmShift)) + for solmol in top.SOLUTEMOLECULES.NSP: + retTop.add_new_SOLUTEMOLECULES(number=solmol + atnmShift) # add TEMPERATUREGROUPS - for solmol in top.TEMPERATUREGROUPS.content[1:]: - retTop.add_new_TEMPERATUREGROUPS(number=str(int(solmol[0]) + atnmShift)) + for solmol in top.TEMPERATUREGROUPS.NSP: + retTop.add_new_TEMPERATUREGROUPS(number=solmol + atnmShift) # add PRESSUREGROUPS - for solmol in top.PRESSUREGROUPS.content[1:]: - retTop.add_new_PRESSUREGROUPS(number=str(int(solmol[0]) + atnmShift)) + for solmol in top.PRESSUREGROUPS.NSP: + retTop.add_new_PRESSUREGROUPS(number=solmol + atnmShift) return retTop @@ -250,7 +250,7 @@ def multiply_top(self, n_muliplication: int, unifyGroups: bool = False, verbose= # catch simple cases and create return top if n_muliplication == 0: - return TopType(in_value=None) + return self.__class__(in_value=None) retTop = deepcopy(self) if n_muliplication == 1: return retTop @@ -363,33 +363,23 @@ def multiply_top(self, n_muliplication: int, unifyGroups: bool = False, verbose= retTop.DIHEDRALH.content.append(deepcopy(angle)) if hasattr(top, "SOLUTEMOLECULES"): - if unifyGroups and int(top.SOLUTEMOLECULES.content[0][0]) == 1: - retTop.SOLUTEMOLECULES.content[1][0] = str(int(retTop.SOLUTEMOLECULES.content[1][0]) * n_muliplication) + if unifyGroups and top.SOLUTEMOLECULES.NSM == 1: + retTop.SOLUTEMOLECULES.NSM = 1 + retTop.SOLUTEMOLECULES.NSP = [sum(top.SOLUTEMOLECULES.NSP) * n_muliplication] else: - retTop.SOLUTEMOLECULES.content[0][0] = str(int(top.SOLUTEMOLECULES.content[0][0]) * n_muliplication) + retTop.SOLUTEMOLECULES.NSM = top.SOLUTEMOLECULES.NSM * n_muliplication for i in range(n_loops): - groups = [str(int(i) + atnmShift) for i in top.SOLUTEMOLECULES.content[1]] - retTop.SOLUTEMOLECULES.content.append(groups) + groups = [j + atnmShift * (i + 1) for j in top.SOLUTEMOLECULES.NSP] + retTop.SOLUTEMOLECULES.NSP.extend(groups) + # So far there was no reason to destinguish between SOLUTEMOLECULES and the following blocks if hasattr(top, "TEMPERATUREGROUPS"): - if unifyGroups and int(top.TEMPERATUREGROUPS.content[0][0]) == 1: - retTop.TEMPERATUREGROUPS.content[1][0] = str( - int(retTop.TEMPERATUREGROUPS.content[1][0]) * n_muliplication - ) - else: - retTop.TEMPERATUREGROUPS.content[0][0] = str(int(top.TEMPERATUREGROUPS.content[0][0]) * n_muliplication) - for i in range(n_loops): - groups = [str(int(i) + atnmShift) for i in top.TEMPERATUREGROUPS.content[1]] - retTop.TEMPERATUREGROUPS.content.append(groups) + retTop.TEMPERATUREGROUPS.NSM = retTop.SOLUTEMOLECULES.NSM + retTop.TEMPERATUREGROUPS.NSP = retTop.SOLUTEMOLECULES.NSP if hasattr(top, "PRESSUREGROUPS"): - if unifyGroups and int(top.PRESSUREGROUPS.content[0][0]) == 1: - retTop.PRESSUREGROUPS.content[1][0] = str(int(retTop.PRESSUREGROUPS.content[1][0]) * n_muliplication) - else: - retTop.PRESSUREGROUPS.content[0][0] = str(int(top.PRESSUREGROUPS.content[0][0]) * n_muliplication) - for i in range(n_loops): - groups = [str(int(i) + atnmShift) for i in top.PRESSUREGROUPS.content[1]] - retTop.PRESSUREGROUPS.content.append(groups) + retTop.PRESSUREGROUPS.NSM = retTop.SOLUTEMOLECULES.NSM + retTop.PRESSUREGROUPS.NSP = retTop.SOLUTEMOLECULES.NSP # return everything return retTop @@ -1036,39 +1026,27 @@ def add_new_CONSTRAINT(self, IC: int, JC: int, ICC: float, verbose=False): def add_new_TEMPERATUREGROUPS(self, number: str, verbose=False): if not hasattr(self, "TEMPERATUREGROUPS"): - defaultContent = ["0", "Dummy"] + defaultContent = ["0", 0] self.add_block(blocktitle="TEMPERATUREGROUPS", content=defaultContent, verbose=verbose) - self.TEMPERATUREGROUPS.content.append([number]) - self.TEMPERATUREGROUPS.content.remove(["Dummy"]) - else: - if len(self.TEMPERATUREGROUPS.content) < 1: - self.TEMPERATUREGROUPS.content.append(["0"]) - self.TEMPERATUREGROUPS.content.append([number]) - self.TEMPERATUREGROUPS.content[0][0] = str(int(self.TEMPERATUREGROUPS.content[0][0]) + 1) + self.TEMPERATUREGROUPS.NSP.remove([0]) + self.TEMPERATUREGROUPS.NSM += 1 + self.TEMPERATUREGROUPS.NSP.append(number) def add_new_SOLUTEMOLECULES(self, number: str, verbose=False): if not hasattr(self, "SOLUTEMOLECULES"): defaultContent = ["0", "Dummy"] self.add_block(blocktitle="SOLUTEMOLECULES", content=defaultContent, verbose=verbose) - self.SOLUTEMOLECULES.content.append([number]) self.SOLUTEMOLECULES.content.remove(["Dummy"]) - else: - if len(self.SOLUTEMOLECULES.content) < 1: - self.SOLUTEMOLECULES.content.append(["0"]) - self.SOLUTEMOLECULES.content.append([number]) - self.SOLUTEMOLECULES.content[0][0] = str(int(self.SOLUTEMOLECULES.content[0][0]) + 1) + self.SOLUTEMOLECULES.NSM += 1 + self.SOLUTEMOLECULES.NSP.append(number) def add_new_PRESSUREGROUPS(self, number: str, verbose=False): if not hasattr(self, "PRESSUREGROUPS"): - defaultContent = ["0", "Dummy"] + defaultContent = ["0", 0] self.add_block(blocktitle="PRESSUREGROUPS", content=defaultContent, verbose=verbose) - self.PRESSUREGROUPS.content.append([number]) - self.PRESSUREGROUPS.content.remove(["Dummy"]) - else: - if len(self.PRESSUREGROUPS.content) < 1: - self.PRESSUREGROUPS.content.append(["0"]) - self.PRESSUREGROUPS.content.append([number]) - self.PRESSUREGROUPS.content[0][0] = str(int(self.PRESSUREGROUPS.content[0][0]) + 1) + self.PRESSUREGROUPS.NSP.remove([0]) + self.PRESSUREGROUPS.NSM += 1 + self.PRESSUREGROUPS.NSP.append(number) def get_mass(self) -> float: """ diff --git a/pygromos/files/trajectory/tre.py b/pygromos/files/trajectory/tre.py index 9227e4f9..4c330d19 100644 --- a/pygromos/files/trajectory/tre.py +++ b/pygromos/files/trajectory/tre.py @@ -22,7 +22,7 @@ import pygromos.files.trajectory._general_trajectory as traj from pygromos.files.trajectory.tre_field_libs.ene_fields import ( - gromos_2020_tre_block_names_table, + gromos_2021_tre_block_names_table, gromos_tre_block_names_table, ) from pygromos.analysis import energy_analysis as ea @@ -65,7 +65,7 @@ def __init__( auto_save=True, stride: int = 1, skip: int = 0, - _ene_ana_names: gromos_tre_block_names_table = gromos_2020_tre_block_names_table, + _ene_ana_names: gromos_tre_block_names_table = gromos_2021_tre_block_names_table, ): """ Build a Gromos energy trajectory file (.tre) @@ -107,10 +107,11 @@ def get_totals(self) -> pd.DataFrame: """ # print(self.database["totals"][0].shape, self.database["totals"][0]) if not hasattr(self, "totals"): + totals_data = np.stack(self.database["totals"].to_numpy()) self.totals = pd.DataFrame( - data=np.stack(self.database["totals"].to_numpy()), + data=totals_data, index=self.database.time, - columns=self.tre_block_name_table.totals_subblock_names, + columns=self.tre_block_name_table.totals_subblock_names[: totals_data.shape[1]], ) else: pass @@ -446,12 +447,12 @@ def get_density(self) -> pd.DataFrame: def get_Hvap(self, gas_traj, nMolecules=1, temperature=None) -> float: gas_totpot_energy = 0 if type(gas_traj) == type(self): - gas_totpot_energy = gas_traj.get_totals_totpot().mean() + gas_totpot_energy = gas_traj.get_totpot().mean() elif type(gas_traj) == float: gas_totpot_energy = gas_traj else: raise TypeError("Did not understand the type of gas. Allowed are float (E_gas) or Tre (gas_trajectory)") - liq_totpot_energy = self.get_totals_totpot().mean() + liq_totpot_energy = self.get_totpot().mean() # get temperature from liq trajectory if not given if temperature is None: diff --git a/pygromos/gromos/_gromosClass.py b/pygromos/gromos/_gromosClass.py index a2588c76..aa4e06b5 100644 --- a/pygromos/gromos/_gromosClass.py +++ b/pygromos/gromos/_gromosClass.py @@ -1,60 +1,80 @@ +from typing import Callable +from pygromos.utils.compiledProgram import _compiled_program + import os import functools -from typing import Union from pygromos.files._basics import _general_gromos_file -from pygromos.utils.compiledProgram import _compiled_program class _gromosClass(_compiled_program): - def __init__(self, in_bin_dir: str, dummy: bool = False) -> Union[str, None]: - - for func in dir(self): - print(func, end="") - if callable(getattr(self, func)) and not func.startswith("__"): - print("\t check") - setattr(self, func, gromosTypeConverter(self, getattr(self, func))) - pass - - super().__init__(in_bin_dir, dummy) - - -def gromosTypeConverter(self, func) -> callable: - """ - This decorator can be used to automatically convert - Parameters - ---------- - func - - Returns - ------- - callable - - """ - - @functools.wraps(func) - def convert_pyGromos_types(*args, **kwargs): - # no key-word parameters - nargs = [] - for v in args: - if isinstance(v, _general_gromos_file._general_gromos_file): - if v.path is None or not os.path.exists(v.path): - raise IOError( - "please write out the " - + str(v.__name__) - + " first to use the function " - + str(func.__name__) - + "" - ) - v = v.path - nargs.append(v) - - # key-value - parameters - for k, v in kwargs.items(): - if isinstance(v, _general_gromos_file._general_gromos_file): - if v.path is None or not os.path.exists(v.path): - raise IOError("please write out the " + k + " first to use the function " + str(func.__name__) + "") - kwargs[k] = v.path - - return func(*nargs, **kwargs) - - return convert_pyGromos_types + def __init__(self, in_bin_dir: str, dummy: bool = False, _check_binary_paths: bool = True) -> str: + """ + This parent class contains wrappers for gromos functionalities. + E.g. gromosTypeConverter converts a passed gromos obj (Cnf, Top etc.) to a string path, such it can be passed to the comand line tools. + + Parameters + ---------- + in_bin_dir : Union[str, None] + directory containing binaries of the gromos program. + dummy : bool, optional + For dummy executions, will not throw errors on binary check fails, by default False + _dont_check_binary : bool, optional + This flag removes the checks of the binary presence for this obj. This can make sense if system access is slow!, by default False - checks will be made + + Returns + ------- + Union[str, None] + _description_ + """ + super().__init__(in_bin_dir, dummy, _check_binary_paths=_check_binary_paths) + + def _gromosTypeConverter(func: Callable) -> Callable: + """ + This decorator can be used to automatically convert gromos files to the str path, where this obj, was written to. + + Parameters + ---------- + func: Callable + function to be decorated + + Returns + ------- + Callable + decorated function + + """ + + @functools.wraps(func) + def convert_pyGromos_types(self, *args, **kwargs): + # print("Converter1: ", func.__name__, args, kwargs) + + # no key-word parameters + nargs = [] + for v in args: + if isinstance(v, _general_gromos_file._general_gromos_file): + if v.path is None or not os.path.exists(v.path): + raise IOError( + "please write out the " + + str(v.__name__) + + " first to use the function " + + str(func.__name__) + + "" + ) + v = v.path + nargs.append(v) + + nargs = list(filter(lambda x: x != self, nargs)) # avoid double selfing + + # key-value - parameters + for k, v in kwargs.items(): + if isinstance(v, _general_gromos_file._general_gromos_file): + if v.path is None or not os.path.exists(v.path): + raise IOError( + "please write out the " + k + " first to use the function " + str(func.__name__) + "" + ) + kwargs[k] = v.path + + # print("Converter2: ", func.__name__, self, nargs, kwargs) + return func(self, *nargs, **kwargs) + + return convert_pyGromos_types diff --git a/pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/analysis_tools/__init__.py b/pygromos/gromos/compile_gromos.py similarity index 100% rename from pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/analysis_tools/__init__.py rename to pygromos/gromos/compile_gromos.py diff --git a/pygromos/gromos/gromosPP.py b/pygromos/gromos/gromosPP.py index f9ebfcf1..4117419a 100644 --- a/pygromos/gromos/gromosPP.py +++ b/pygromos/gromos/gromosPP.py @@ -34,21 +34,27 @@ class _gromosPPbase(_gromosClass): _isValid: bool = False - def __init__(self, gromosPP_bin_dir: str = None, verbose: bool = False): + def __init__( + self, gromosPP_bin_dir: Union[str, None] = None, _check_binary_paths: bool = True, verbose: bool = False + ): """ Constructing a gromosPP object. Parameters ---------- - bin : str, optional + bin : Union[str, None], optional This is the path to the folder containing the binaries of gromosXX. If None, the bash enviroment variables will be used. + _dont_check_binary : bool, optional + This flag removes the checks of the binary presence for this obj. This can make sense if system access is slow!, by default False - checks will be made """ # lazy me - doc text for functions: functions_text = "\n Methods:\n ---------\n" + "\n".join( ["\t\t" + x for x in dir(self) if (not x.startswith("_") and callable(getattr(self, x)))] ) self.__doc__ = self.__doc__ + functions_text - super().__init__(in_bin_dir=gromosPP_bin_dir) # initialises the binary checks + super().__init__( + in_bin_dir=gromosPP_bin_dir, _check_binary_paths=_check_binary_paths + ) # initialises the binary checks def __str__(self): return self.__doc__ @@ -60,6 +66,7 @@ def __repr__(self): GromosPP Programms """ + @_gromosClass._gromosTypeConverter def amber2gromos( self, ambertop: str, @@ -108,6 +115,7 @@ def amber2gromos( print(command) bash.execute(command, catch_STD=out_path, verbose=verbose) + @_gromosClass._gromosTypeConverter def pdb2gromos( self, in_pdb_path: str, @@ -158,6 +166,7 @@ def pdb2gromos( return out_cnf_path + @_gromosClass._gromosTypeConverter def pdb2seq( self, in_pdb_path: str, @@ -227,6 +236,7 @@ def pdb2seq( bash.execute(command) return out_path + @_gromosClass._gromosTypeConverter def make_top( self, out_top_path: str, @@ -278,6 +288,7 @@ def make_top( bash.execute(command, catch_STD=out_top_path) return out_top_path + @_gromosClass._gromosTypeConverter def com_top( self, in_topo_paths: (str or List[str]), @@ -330,6 +341,7 @@ def com_top( bash.execute(command, catch_STD=out_top_path) return out_top_path + @_gromosClass._gromosTypeConverter def dfmult( self, in_endstate_file_paths: List[str], @@ -389,6 +401,7 @@ def dfmult( return out_file_path + @_gromosClass._gromosTypeConverter def frameout( self, in_top_path: str, @@ -522,6 +535,7 @@ def frameout( return out_file_path + @_gromosClass._gromosTypeConverter def ene_ana( self, in_ene_ana_library_path: str, @@ -725,6 +739,7 @@ def ene_ana( else: return result_files + @_gromosClass._gromosTypeConverter def gch( self, in_cnf_path: str, @@ -770,6 +785,7 @@ def gch( return out_cnf_path + @_gromosClass._gromosTypeConverter def add_hydrogens( self, in_cnf_path: str, @@ -798,16 +814,24 @@ def add_hydrogens( out_cnf_path """ - self.gch( - in_cnf_path=in_cnf_path, - in_top_path=in_top_path, - out_cnf_path=out_cnf_path, - tolerance=tolerance, - periodic_boundary_condition=periodic_boundary_condition, - gathering=gathering, - _binary_name=_binary_name, + command = ( + self._bin + + _binary_name + + " @topo " + + in_top_path + + " @pos " + + in_cnf_path + + " @tol " + + str(tolerance) + + " " + "@pbc " + periodic_boundary_condition + " " + gathering ) + bash.execute(command, catch_STD=out_cnf_path) + + return out_cnf_path + + @_gromosClass._gromosTypeConverter def sim_box( self, in_top_path: str, @@ -904,6 +928,7 @@ def sim_box( print(p.stderr) return out_cnf_path + @_gromosClass._gromosTypeConverter def ran_box( self, in_top_path: str, @@ -967,6 +992,7 @@ def ran_box( else: return command + @_gromosClass._gromosTypeConverter def build_box( self, in_top_path: str, @@ -1010,6 +1036,7 @@ def build_box( else: return command + @_gromosClass._gromosTypeConverter def tser( self, in_trc_path: str, @@ -1082,6 +1109,7 @@ def tser( bash.execute(command) return out_csv_path + @_gromosClass._gromosTypeConverter def red_top(self, in_top_path: str, atom_selection: str, out_top_path: str, _binary_name: str = "red_top") -> str: """ red_top is a gromos tool to reduce a gromos tool to a certain selection. @@ -1109,6 +1137,7 @@ def red_top(self, in_top_path: str, atom_selection: str, out_top_path: str, _bin bash.execute(command) return out_top_path + @_gromosClass._gromosTypeConverter def prep_eds( self, in_top_paths: List[str], @@ -1170,6 +1199,7 @@ def prep_eds( return out_top, out_ptp + @_gromosClass._gromosTypeConverter def prep_noe( self, in_top_path: str, @@ -1249,6 +1279,7 @@ def prep_noe( return out_path + @_gromosClass._gromosTypeConverter def rmsf( self, in_top_path: str, @@ -1306,6 +1337,7 @@ def rmsf( bash.execute(command, catch_STD=out_file_path) return out_file_path + @_gromosClass._gromosTypeConverter def rmsd( self, in_top_path: str, @@ -1354,6 +1386,7 @@ def rmsd( bash.execute(command, catch_STD=out_file_path) return out_file_path + @_gromosClass._gromosTypeConverter def cog( self, in_top_path: str, @@ -1430,6 +1463,7 @@ def cog( return out_file_path + @_gromosClass._gromosTypeConverter def noe( self, in_top_path: str, @@ -1495,6 +1529,7 @@ def noe( return out_path + @_gromosClass._gromosTypeConverter def jval( self, in_top_path: str, @@ -1586,6 +1621,7 @@ def jval( return out_path + @_gromosClass._gromosTypeConverter def ion( self, in_top_path: str, @@ -1893,5 +1929,5 @@ class GromosPP(_gromosPPbase): This is the path to the folder containing the binaries of gromosPP If None, the bash enviroment variables will be used. """ - def __init__(self, gromosPP_bin_dir: str = None, verbose: bool = False): - super().__init__(gromosPP_bin_dir=gromosPP_bin_dir, verbose=verbose) + def __init__(self, gromosPP_bin_dir: str = None, _check_binary_paths: bool = True, verbose: bool = False): + super().__init__(gromosPP_bin_dir=gromosPP_bin_dir, verbose=verbose, _check_binary_paths=_check_binary_paths) diff --git a/pygromos/gromos/gromosPlsPls b/pygromos/gromos/gromosPlsPls new file mode 160000 index 00000000..d459d15e --- /dev/null +++ b/pygromos/gromos/gromosPlsPls @@ -0,0 +1 @@ +Subproject commit d459d15ee421786a1c3c5b852bf2b4c7b609bd62 diff --git a/pygromos/gromos/gromosXX b/pygromos/gromos/gromosXX new file mode 160000 index 00000000..4395064f --- /dev/null +++ b/pygromos/gromos/gromosXX @@ -0,0 +1 @@ +Subproject commit 4395064f6114b2d04ec12d5267f1ad2740182a33 diff --git a/pygromos/gromos/gromosXX.py b/pygromos/gromos/gromosXX.py index 6ef7de35..f343fe23 100644 --- a/pygromos/gromos/gromosXX.py +++ b/pygromos/gromos/gromosXX.py @@ -27,7 +27,11 @@ class _GromosXX(_gromosClass): This is the path to the folder containing the binaries of gromosXX. If None, the bash enviroment variables will be used. """ - def __init__(self, gromosXX_bin_dir: str = None): + def __init__( + self, + gromosXX_bin_dir: str = None, + _check_binary_paths: bool = True, + ): """ Constructing a gromosXX object. @@ -35,6 +39,8 @@ def __init__(self, gromosXX_bin_dir: str = None): ---------- gromosXX_bin_dir : str, optional This is the path to the folder containing the binaries of gromosXX. If None, the bash enviroment variables will be used. + _dont_check_binary : bool, optional + This flag removes the checks of the binary presence for this obj. This can make sense if system access is slow!, by default False - checks will be made """ # lazy me - doc text for functions: functions_text = "\n Methods:\n ---------\n" + "\n".join( @@ -42,7 +48,9 @@ def __init__(self, gromosXX_bin_dir: str = None): ) self.__doc__ = self.__doc__ + functions_text - super().__init__(in_bin_dir=gromosXX_bin_dir) # initialises the binary checks + super().__init__( + in_bin_dir=gromosXX_bin_dir, _check_binary_paths=_check_binary_paths + ) # initialises the binary checks def __str__(self): return self.__doc__ @@ -54,6 +62,7 @@ def __repr__(self): GromosXX Programms """ + @_gromosClass._gromosTypeConverter def md_run( self, in_topo_path: str, @@ -216,6 +225,12 @@ def md_run( log_file.write("\tRUN:\tSUCESSFUL\n") else: log_file.write("\tRUN:\tFAILED\n") + + omd_file_content = open(log_file_path, "r").read_lines() + if len(omd_file_content) > 0: + print("\t" + "\n\t".join(omd_file_content)) + else: + print("\t None") failed = True log_file.write("\tTIME:\n\tstart: " + str(start_time) + "\tend: " + str(end_time) + "\n") @@ -228,6 +243,7 @@ def md_run( return log_file_path + @_gromosClass._gromosTypeConverter def repex_run( self, in_topo_path: str, @@ -414,5 +430,5 @@ class GromosXX(_GromosXX): This is the path to the folder containing the binaries of gromosXX. If None, the bash enviroment variables will be used. """ - def __init__(self, gromosXX_bin_dir: str = None): - super().__init__(gromosXX_bin_dir=gromosXX_bin_dir) + def __init__(self, gromosXX_bin_dir: str = None, _check_binary_paths: bool = True): + super().__init__(gromosXX_bin_dir=gromosXX_bin_dir, _check_binary_paths=_check_binary_paths) diff --git a/pygromos/simulations/approaches/hvap_calculation/hvap_calculation.py b/pygromos/simulations/approaches/hvap_calculation/hvap_calculation.py index f9c71fac..ba51ebf4 100644 --- a/pygromos/simulations/approaches/hvap_calculation/hvap_calculation.py +++ b/pygromos/simulations/approaches/hvap_calculation/hvap_calculation.py @@ -24,30 +24,35 @@ import time from pygromos.gromos.pyGromosPP.ran_box import ran_box -from pygromos.gromos.pyGromosPP.com_top import com_top from pygromos.files.gromos_system.gromos_system import Gromos_System from pygromos.simulations.approaches.hvap_calculation import hvap_input_files from pygromos.files.gromos_system.ff.forcefield_system import forcefield_system -from pygromos.simulations.hpc_queuing.submission_systems.local import LOCAL as subSys +from pygromos.simulations.hpc_queuing.submission_systems import get_submission_system, _submission_system from pygromos.simulations.modules.general_simulation_modules import simulation from pygromos.simulations.hpc_queuing.job_scheduling.workers.analysis_workers import simulation_analysis -from pygromos.files.coord.cnf import Cnf from pygromos.files.simulation_parameters.imd import Imd from pygromos.files.topology.top import Top from pygromos.utils.utils import time_wait_s_for_filesystem +# automatically get Local or LSF submission system (depending on hostname) +subSystem = get_submission_system() + class Hvap_calculation: + dens_modifier: float = 0.7 + submissonSystem_gas: _submission_system + submissonSystem_liq: _submission_system + def __init__( self, input_system: Gromos_System or str or Chem.rdchem.Mol, work_folder: str, system_name: str = "dummy", forcefield: forcefield_system = forcefield_system(name="54A7"), - gromosXX: str = None, - gromosPP: str = None, + in_gromosXX_bin_dir: str = None, + in_gromosPP_bin_dir: str = None, useGromosPlsPls: bool = True, verbose: bool = True, ) -> None: @@ -61,11 +66,17 @@ def __init__( # system variables if type(input_system) is Gromos_System: self.groSys_gas = input_system + if in_gromosXX_bin_dir is not None: + self.groSys_gas.gromosXX = in_gromosXX_bin_dir + if in_gromosPP_bin_dir is not None: + self.groSys_gas.gromosPP = in_gromosPP_bin_dir elif (type(input_system) is str) or (type(input_system) is Chem.rdchem.Mol): self.groSys_gas = Gromos_System( work_folder=work_folder, system_name=system_name, in_smiles=input_system, + in_gromosXX_bin_dir=in_gromosXX_bin_dir, + in_gromosPP_bin_dir=in_gromosPP_bin_dir, Forcefield=forcefield, in_imd_path=hvap_input_files.imd_hvap_gas_sd, verbose=verbose, @@ -74,6 +85,12 @@ def __init__( self.work_folder = work_folder self.system_name = system_name + self.submissonSystem_gas = subSystem(job_duration="4:00") + # give the liquid simulation as much cores as possible to speedup the long liquid simulation. + # if you use a mpi version of gromos use nmpi=4 (or more) + # the attribute can be overwritten by the user at runtime after the class is initiated + self.submissonSystem_liq = subSystem(nomp=4, job_duration="24:00") + # create folders and structure try: os.mkdir(path=work_folder) @@ -88,8 +105,6 @@ def __init__( self.groSys_liq.work_folder = work_folder + "/" + system_name + "_liq" self.groSys_liq.rebase_files() - self.submissonSystem = subSys() - self.gromosXX = self.groSys_gas.gromosXX self.gromosPP = self.groSys_gas.gromosPP @@ -106,7 +121,7 @@ def __init__( # used to multiply the single molecule system # made for small molecule Hvap calculation self.num_molecules = 512 - self.density = 1000 + self.density = 700 self.temperature = 298.15 self.groSys_gas_final = None @@ -136,21 +151,11 @@ def create_liq(self): time.sleep(time_wait_s_for_filesystem) # wait for file to write and close self.groSys_liq.top = tempTop except Exception as e: - self.groSys_liq.top = com_top( - top1=self.groSys_gas.top, - top2=self.groSys_gas.top, - topo_multiplier=[self.num_molecules, 0], - verbose=False, - ) + self.groSys_liq.top = self.groSys_gas.top * self.num_molecules if self.verbose: print(e) else: - self.groSys_liq.top = com_top( - top1=self.groSys_gas.top, - top2=self.groSys_gas.top, - topo_multiplier=[self.num_molecules, 0], - verbose=False, - ) + self.groSys_liq.top = self.groSys_gas.top * self.num_molecules # create liq cnf if self.useGromosPlsPls: @@ -159,8 +164,8 @@ def create_liq(self): in_cnf_path=self.groSys_gas.cnf.path, out_cnf_path=self.work_folder + "/temp.cnf", nmolecule=self.num_molecules, - dens=self.density, - threshold=0.1, + dens=self.dens_modifier * self.density, + threshold=0.12, layer=True, ) else: @@ -169,48 +174,48 @@ def create_liq(self): in_cnf_path=self.groSys_gas.cnf.path, out_cnf_path=self.work_folder + "/temp.cnf", nmolecule=self.num_molecules, - dens=self.density, + dens=self.dens_modifier * self.density, ) time.sleep(time_wait_s_for_filesystem) # wait for file to write and close - self.groSys_liq.cnf = Cnf(in_value=self.work_folder + "/temp.cnf") + self.groSys_liq.cnf = self.work_folder + "/temp.cnf" # reset liq system self.groSys_liq.rebase_files() def run_gas(self): - self.groSys_gas.rebase_files() # min - print(self.groSys_gas.work_folder) - sys_emin_gas, jobID = simulation( + self.groSys_gas.imd = self.imd_gas_min + self.groSys_gas.prepare_for_simulation() + sys_emin_gas = simulation( in_gromos_simulation_system=self.groSys_gas, override_project_dir=self.groSys_gas.work_folder, step_name="1_emin", - in_imd_path=self.imd_gas_min, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_gas, analysis_script=simulation_analysis.do, verbose=self.verbose, ) - print(self.groSys_gas.work_folder) # eq - sys_eq_gas, jobID = simulation( + sys_emin_gas.imd = self.imd_gas_eq + sys_emin_gas.prepare_for_simulation() + sys_eq_gas = simulation( in_gromos_simulation_system=sys_emin_gas, override_project_dir=self.groSys_gas.work_folder, step_name="2_eq", - in_imd_path=self.imd_gas_eq, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_gas, analysis_script=simulation_analysis.do, verbose=self.verbose, ) # sd - sys_sd_gas, jobID = simulation( + sys_eq_gas.imd = self.imd_gas_eq + sys_eq_gas.prepare_for_simulation() + sys_sd_gas = simulation( in_gromos_simulation_system=sys_eq_gas, override_project_dir=self.groSys_gas.work_folder, step_name="3_sd", - in_imd_path=self.imd_gas_sd, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_gas, analysis_script=simulation_analysis.do, verbose=self.verbose, ) @@ -218,37 +223,39 @@ def run_gas(self): self.groSys_gas_final = sys_sd_gas def run_liq(self): - self.groSys_liq.rebase_files() # minsys_emin_liq, jobID - sys_emin_liq, jobID = simulation( + self.groSys_liq.imd = self.imd_liq_min + self.groSys_liq.prepare_for_simulation() + sys_emin_liq = simulation( in_gromos_simulation_system=self.groSys_liq, override_project_dir=self.groSys_liq.work_folder, step_name="1_emin", - in_imd_path=self.imd_liq_min, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_liq, analysis_script=simulation_analysis.do, verbose=self.verbose, ) # eq - sys_eq_liq, jobID = simulation( + sys_emin_liq.imd = self.imd_liq_eq + sys_emin_liq.prepare_for_simulation() + sys_eq_liq = simulation( in_gromos_simulation_system=sys_emin_liq, override_project_dir=self.groSys_liq.work_folder, step_name="2_eq", - in_imd_path=self.imd_liq_eq, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_liq, analysis_script=simulation_analysis.do, verbose=self.verbose, ) # md - sys_md_liq, jobID = simulation( + sys_eq_liq.imd = self.imd_liq_md + sys_eq_liq.prepare_for_simulation() + sys_md_liq = simulation( in_gromos_simulation_system=sys_eq_liq, override_project_dir=self.groSys_liq.work_folder, step_name="3_sd", - in_imd_path=self.imd_liq_md, - submission_system=self.submissonSystem, + submission_system=self.submissonSystem_liq, analysis_script=simulation_analysis.do, verbose=self.verbose, ) diff --git a/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/scheduler_functions.py b/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/scheduler_functions.py index cd79f027..2179c018 100644 --- a/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/scheduler_functions.py +++ b/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/scheduler_functions.py @@ -1,7 +1,5 @@ import glob import os -import warnings - import pandas as pd from pygromos.files.coord.cnf import Cnf @@ -26,7 +24,7 @@ def do_skip_job( # Check if job with same name is already in the queue! if (verbose) and verbose_lvl >= 2: print("Checking if jobs was already submitted or done") - if job_submission_system._block_double_submission: # can we find an job with this name in the queue? + if job_submission_system.block_double_submission: # can we find an job with this name in the queue? if (verbose) and verbose_lvl >= 2: print("Checking for jobs with name: " + tmp_jobname) queued_job_ids = job_submission_system.search_queue_for_jobname(job_name=tmp_jobname) @@ -204,14 +202,8 @@ def chain_submission( md_args += "-nomp " + str(job_submission_system.nomp) + "\n" md_args += "-initialize_first_run " + str(initialize_first_run) + "\n" md_args += "-reinitialize_every_run " + str(reinitialize_every_run) + "\n" - if simSystem.gromosXX is not None: - md_args += "-gromosXX_bin_dir " + str(simSystem.gromosXX.bin) + "\n" - else: - md_args += "-gromosXX_bin_dir None \n" - if verbose: - warnings.warn( - "gromosXX_bin_dir is None \n If you want to simulate something please add a existing gromos bin\n" - ) + md_args += "-gromosXX_bin_dir " + str(simSystem.gromosXX.bin) + "\n" + md_args += "-gromosXX_check_binary_paths " + str(simSystem.gromosXX._check_binary_paths) + "\n" if work_dir is not None: md_args += "-work_dir " + str(work_dir) + "\n" diff --git a/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/simulation_scheduler.py b/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/simulation_scheduler.py index 11cbcfde..0fc21a91 100644 --- a/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/simulation_scheduler.py +++ b/pygromos/simulations/hpc_queuing/job_scheduling/schedulers/simulation_scheduler.py @@ -26,16 +26,17 @@ def do( out_dir_path: str, simulation_run_num: int, equilibration_run_num: int = 0, - work_dir: str = None, initialize_first_run=False, reinitialize_every_run=False, analysis_script_path: str = None, submission_system: _SubmissionSystem = LSF(), previous_job_ID: int = None, - no_double_submit: bool = False, + _no_double_submit_check: bool = False, + _work_dir: str = None, verbose: bool = True, verbose_lvl: int = 1, ): + """ Parameters @@ -45,7 +46,7 @@ def do( simulation_run_num equilibration_run_num gromos_bin_dir - work_dir + _work_dir analysis_script_path submission_system previous_job_ID @@ -60,6 +61,7 @@ def do( ------- """ + submission_system.block_double_submission = _no_double_submit_check job_verb = True if (verbose and verbose_lvl > 2) else False # prepare @@ -74,12 +76,12 @@ def do( bash.make_folder(out_dir_path) # final output_folder # workdir: - if not isinstance(work_dir, type(None)) and work_dir != "None": + if not isinstance(_work_dir, type(None)) and _work_dir != "None": if verbose and verbose_lvl > 2: - print("\t -> Generating given workdir: " + work_dir) - bash.make_folder(work_dir, "-p") - os.chdir(work_dir) - prepared_imd = work_dir + "/" + os.path.basename(in_simSystem.imd.path) # noqa: F841 + print("\t -> Generating given workdir: " + _work_dir) + bash.make_folder(_work_dir, "-p") + os.chdir(_work_dir) + prepared_imd = _work_dir + "/" + os.path.basename(in_simSystem.imd.path) # noqa: F841 else: if verbose and verbose_lvl > 2: print("\t -> Using on node workdir") @@ -87,17 +89,17 @@ def do( # sim vars logs out_prefix = in_simSystem.name - slave_script = workerScript.__file__ + worker_script = workerScript.__file__ # CHECK PATH DEPENDENCIES - all Files present? # needed variables check_path_dependencies_paths = [ - slave_script, + worker_script, out_dir_path, ] # Coord file is used by repex in_imd_path prepared_im # variable paths - if work_dir is not None: - check_path_dependencies_paths.append(work_dir) + if _work_dir is not None and _work_dir != "out_dir": + check_path_dependencies_paths.append(_work_dir) if not in_simSystem.top._future_file: check_path_dependencies_paths.append(in_simSystem.top.path) @@ -116,8 +118,6 @@ def do( if in_simSystem.qmmm is not None: check_path_dependencies_paths.append(in_simSystem.qmmm.path) - # prepared_imd = bash.copy_file(in_simSystem.imd.path, prepared_imd) #Todo: Remove? @bschroed - bash.check_path_dependencies(check_path_dependencies_paths, verbose=job_verb) except Exception as err: @@ -158,7 +158,7 @@ def do( start_run_index=1, prefix_command="", previous_job_ID=previous_job_ID, - work_dir=work_dir, + work_dir=_work_dir, initialize_first_run=initialize_first_run, reinitialize_every_run=reinitialize_every_run, verbose=job_verb, @@ -179,7 +179,7 @@ def do( job_submission_system=submission_system, prefix_command="", previous_job_ID=previous_job_ID, - work_dir=None, + work_dir=_work_dir, initialize_first_run=initialize_first_run, reinitialize_every_run=reinitialize_every_run, verbose=job_verb, diff --git a/pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/simulation_analysis.py b/pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/simulation_analysis.py index f8559e74..a459b784 100644 --- a/pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/simulation_analysis.py +++ b/pygromos/simulations/hpc_queuing/job_scheduling/workers/analysis_workers/simulation_analysis.py @@ -6,7 +6,6 @@ import math from typing import Dict, Union, List from collections import OrderedDict -from pygromos.gromos import gromosPP from pygromos.files.coord import cnf from pygromos.files.trajectory import trc, tre, trg from pygromos.utils import bash @@ -38,7 +37,6 @@ def do( in_simulation_dir: str, out_analysis_dir: str, sim_prefix: str, # in_system:Gromos_System, - gromosPP_bin_dir: str = None, n_processes: int = 1, control_dict: dict = None, verbose: bool = True, @@ -59,11 +57,6 @@ def do( ------- """ - # data = {} - if gromosPP_bin_dir is not None: - gromos = gromosPP.GromosPP(gromosPP_bin_dir) - bash.command_exists(gromos.make_top()) - if not os.path.exists(out_analysis_dir) and not os.path.isdir(out_analysis_dir): bash.make_folder(out_analysis_dir) if not isinstance(control_dict, dict): @@ -226,34 +219,6 @@ def project_concatenation( out_trg_file += tmp_trg out_trg_file.write(output_path=out_traj_path) - """ - - if (control_dict["convert_trcs"]): - print("\tStart Trc Conversion") - # wait for async job creating the trcs. - if (submitted_trc_job): - p_trc.wait() - - # get files: - final_trc_files = list( - sorted(glob.glob(out_folder + "/*.trc*"), key=lambda x: int(x.split("_")[-1].split(".")[0]))) - - if (n_processes > 1): - out_dcd = manager.dict() - distributed_jobs = [ - (n, range(n, num_replicas, n_processes), final_trc_files, in_topology_path, gromosPP_bin_dir, - out_dcd, - fit_traj_to_mol, verbose) for - n in range(n_processes)] - p_conv = p.starmap_async(_thread_worker_conv_trc, distributed_jobs) - else: - out_dcd = {} - _thread_worker_conv_trc(job=-1, replica_range=range(num_replicas), trc_files=final_trc_files, - in_topology_path=in_topology_path, - gromos_path=gromosPP_bin_dir, out_traj=out_dcd, fit_traj_to_mol=1, - verbose=verbose, - boundary_conditions=boundary_conditions) - """ if verbose: print("all jobs finished") return out_cnf # in_simSystem diff --git a/pygromos/simulations/hpc_queuing/job_scheduling/workers/simulation_workers/simulation_run_worker.py b/pygromos/simulations/hpc_queuing/job_scheduling/workers/simulation_workers/simulation_run_worker.py index ebfa3e3d..e9cce807 100644 --- a/pygromos/simulations/hpc_queuing/job_scheduling/workers/simulation_workers/simulation_run_worker.py +++ b/pygromos/simulations/hpc_queuing/job_scheduling/workers/simulation_workers/simulation_run_worker.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import os import sys +import glob import time import math +import traceback from pygromos.gromos import gromosXX as mdGromosXX from pygromos.files.simulation_parameters import imd @@ -39,9 +41,11 @@ def work( reinitialize_every_run: bool = False, initialize_first_run: bool = True, gromosXX_bin_dir: str = None, + gromosXX_check_binary_paths: bool = True, work_dir: str = None, zip_trajectories: bool = True, - **kwargs + _gromos_noBinary_checks: bool = False, + **kwargs, ): """ Executed by repex_EDS_long_production_run as workers @@ -89,14 +93,25 @@ def work( """ time.sleep(time_wait_s_for_filesystem) # WORKDIR SetUP - if (work_dir is None or work_dir == "None") and "TMPDIR" in os.environ: - work_dir = os.environ["TMPDIR"] - print("using TmpDir") + if work_dir is None or work_dir == "None": + if "TMPDIR" in os.environ: + work_dir = os.environ["TMPDIR"] + else: + print("Could not find TMPDIR!\n Switched to outdir for work") + work_dir = out_dir + if not os.path.isdir(work_dir): + bash.make_folder(work_dir) + elif isinstance(work_dir, str) and work_dir != "None": + if work_dir == "out_dir": + work_dir = out_dir + if not os.path.isdir(work_dir): + bash.make_folder(work_dir) + else: + if not os.path.isdir(work_dir): + bash.make_folder(work_dir) else: - print("Could not find TMPDIR!\n Switched to outdir for work") - work_dir = out_dir - if not os.path.isdir(work_dir): - bash.make_folder(work_dir) + raise ValueError(f"work_dir is not a valid path, work_dir: {work_dir}") + print("workDIR: " + str(work_dir)) # Check if the calculation is running on multiple nodes: @@ -145,7 +160,7 @@ def work( if hasattr(imd_file, "MULTIBATH"): imd_file.INITIALISE.NTINHT = 0 if (imd_file.MULTIBATH.ALGORITHM <= 1) else 1 - imd_file.INITIALISE.NTISHI = 0 if (hasattr(cnf_file, "LATTICESHIFT")) else 1 + imd_file.INITIALISE.NTISHI = 0 if (hasattr(cnf_file, "LATTICESHIFTS")) else 1 imd_file.INITIALISE.NTIRTC = 0 imd_file.INITIALISE.NTICOM = 0 @@ -165,13 +180,17 @@ def work( imd_file.INITIALISE.NTISTI = 0 if is_stochastic_dynamics_sim or is_vacuum: - imd_file.INITIALISE.NTISHI = 1 + imd_file.INITIALISE.NTISTI = 1 # Write out: tmp_imd_path = imd_file.write(tmp_imd_path) # RUN - gromosXX = mdGromosXX.GromosXX(gromosXX_bin_dir=gromosXX_bin_dir) + print("input: ", type(gromosXX_bin_dir), gromosXX_bin_dir) + gromosXX = mdGromosXX.GromosXX(gromosXX_bin_dir=gromosXX_bin_dir, _check_binary_paths=gromosXX_check_binary_paths) + print("gromosXX._bin: ", type(gromosXX._bin), gromosXX._bin) + print("gromosXX.bin: ", type(gromosXX.bin), gromosXX.bin) + try: print(spacer + "\n start MD " + str(os.path.basename(tmp_imd_path)) + "\n") @@ -208,6 +227,8 @@ def work( except Exception as err: print("Failed! process returned: \n Err: \n" + "\n".join(err.args)) md_failed = True + traceback.print_exc() + raise err # zip the files after the simulation. n_cpu_zip = nmpi if nmpi >= nomp else nomp @@ -232,6 +253,16 @@ def work( print("\nFailed during simulations: ", file=sys.stderr) print(type(err), file=sys.stderr) print(err.args, file=sys.stderr) + print("GROMOS OUTPUT:") + + omd_outs = glob.glob(out_dir + "/" + tmp_prefix + "*.omd") + if len(omd_outs) > 1: + omd_file_content = open(omd_outs[0], "r").read_lines() + if len(omd_file_content) > 1: + print("\t" + "\n\t".join(omd_file_content)) + else: + print("\t None") + exit(1) if md_failed: print("\nFailed during simulations: \n Checkout: \n " + str(tmp_prefix) + ".omd", file=sys.stderr) diff --git a/pygromos/simulations/hpc_queuing/submission_systems/_submission_system.py b/pygromos/simulations/hpc_queuing/submission_systems/_submission_system.py index 123fec76..d9d15094 100644 --- a/pygromos/simulations/hpc_queuing/submission_systems/_submission_system.py +++ b/pygromos/simulations/hpc_queuing/submission_systems/_submission_system.py @@ -1,29 +1,35 @@ -from typing import List, Union +import inspect import pandas as pd +from typing import List, Union + from pygromos.simulations.hpc_queuing.submission_systems.submission_job import Submission_job class _SubmissionSystem: verbose: bool submission: bool - - _job_duration: str = "24:00" - _nmpi: int - _nomp: int - _max_storage: float + nmpi: int + nomp: int + max_storage: float + job_duration: str + environment: dict + block_double_submission: bool + chain_prefix: str + begin_mail: bool + end_mail: bool job_queue_list: pd.DataFrame # contains all jobs in the queue (from this users) - _zip_trajectories: bool + zip_trajectories: bool def __init__( self, - submission: bool = False, + submission: bool = True, nmpi: int = 1, nomp: int = 1, max_storage: float = 1000, job_duration: str = "24:00", verbose: bool = False, - enviroment=None, + environment=None, block_double_submission: bool = True, chain_prefix: str = "done", begin_mail: bool = False, @@ -47,7 +53,7 @@ def __init__( the duration of the job as str("HHH:MM") (default: "24:00") verbose : bool, optional let me write you a book! (default: False) - enviroment: dict, optional + environment: dict, optional here you can pass environment variables as dict{varname: value} (default: None) block_double_submission: bool, optional if a job with the same name is already in the queue, it will not be submitted again. (default: True) @@ -61,18 +67,18 @@ def __init__( zip_trajectories: bool, optional determines if output trajectories are compressed or not """ - self.verbose = verbose - self.submission = submission - self._job_duration = job_duration + self._submission = submission self._nmpi = nmpi self._nomp = nomp self._max_storage = max_storage - self._enviroment = enviroment + self._job_duration = job_duration + self.verbose = verbose + self._environment = environment self._block_double_submission = block_double_submission - self.chain_prefix = chain_prefix - self.begin_mail = begin_mail - self.end_mail = end_mail + self._chain_prefix = chain_prefix + self._begin_mail = begin_mail + self._end_mail = end_mail self._zip_trajectories = zip_trajectories def submit_to_queue(self, sub_job: Submission_job) -> int: @@ -100,24 +106,25 @@ def get_script_generation_command(self, var_name: str = None, var_prefixes: str if var_name is None: var_name = var_prefixes + name + params = [] + for key in inspect.signature(self.__init__).parameters: + if hasattr(self, key): + param = getattr(self, key) + if isinstance(param, str): + params.append(key + "='" + str(getattr(self, key)) + "'") + else: + params.append(key + "=" + str(getattr(self, key))) + elif hasattr(self, "_" + key): + param = getattr(self, key) + if isinstance(param, str): + params.append(key + "='" + str(getattr(self, "_" + key)) + "'") + else: + params.append(key + "=" + str(getattr(self, "_" + key))) + parameters_str = ", ".join(params) + gen_cmd = "#Generate " + name + "\n" - gen_cmd += "from " + self.__module__ + " import " + name + " as " + name + "_obj" + "\n" - gen_cmd += ( - var_name - + " = " - + name - + "_obj(submission=" - + str(self.submission) - + ", verbose=" - + str(self.verbose) - + ", nmpi=" - + str(self.nmpi) - + ", nomp=" - + str(self.nomp) - + ', job_duration="' - + str(self.job_duration) - + '")\n\n' - ) + gen_cmd += "from " + self.__module__ + " import " + name + " as " + name + "_class" + "\n" + gen_cmd += var_name + " = " + name + "_class(" + parameters_str + ")\n\n" return gen_cmd def get_jobs_from_queue(self, job_text: str, **kwargs) -> List[int]: @@ -209,42 +216,92 @@ def kill_jobs(self, job_name: str = None, regex: bool = False, job_ids: Union[Li """ raise NotImplementedError("kill_jobs is not implemented for: " + self.__class__.__name__) + @property + def submission(self): + return self._submission + + @submission.setter + def submission(self, submission_value): + self._submission = submission_value + @property def nmpi(self) -> int: return self._nmpi @nmpi.setter - def nmpi(self, nmpi: int): - self._nmpi = int(nmpi) + def nmpi(self, nmpi_value: int): + self._nmpi = int(nmpi_value) @property def nomp(self) -> int: return self._nomp @nomp.setter - def nomp(self, nomp: int): - self._nomp = int(nomp) + def nomp(self, nomp_value: int): + self._nomp = int(nomp_value) + + @property + def max_storage(self) -> str: + return self._max_storage + + @max_storage.setter + def max_storage(self, max_storage_value: float): + self._max_storage = float(max_storage_value) @property def job_duration(self) -> str: return self._job_duration @job_duration.setter - def job_duration(self, job_duration: str): - self._job_duration = str(job_duration) + def job_duration(self, job_duration_value: str): + self._job_duration = str(job_duration_value) + + # no verbose setter since verbose is not private @property - def max_storage(self) -> str: - return self._max_storage + def environment(self) -> str: + return self._environment + + @environment.setter + def environment(self, environment_value: str): + self._environment = str(environment_value) + + @property + def block_double_submission(self) -> bool: + return self._block_double_submission + + @block_double_submission.setter + def block_double_submission(self, block_double_submission_value: bool): + self._block_double_submission = bool(block_double_submission_value) + + @property + def chain_prefix(self) -> str: + return self._chain_prefix + + @chain_prefix.setter + def chain_prefix(self, chain_prefix_value: str): + self._chain_prefix = str(chain_prefix_value) + + @property + def begin_mail(self) -> str: + return self._begin_mail + + @begin_mail.setter + def begin_mail(self, begin_mail_value: str): + self._begin_mail = str(begin_mail_value) + + @property + def end_mail(self) -> str: + return self._end_mail + + @end_mail.setter + def end_mail(self, end_mail_value: str): + self._end_mail = str(end_mail_value) @property def zip_trajectories(self) -> bool: return self._zip_trajectories @zip_trajectories.setter - def zip_trajectories(self, zip_trajectories: bool): - self._zip_trajectories = zip_trajectories - - @max_storage.setter - def max_storage(self, max_storage: float): - self._max_storage = float(max_storage) + def zip_trajectories(self, zip_trajectories_value: bool): + self._zip_trajectories = zip_trajectories_value diff --git a/pygromos/simulations/hpc_queuing/submission_systems/dummy.py b/pygromos/simulations/hpc_queuing/submission_systems/dummy.py index f2a9537a..7c42c3fa 100644 --- a/pygromos/simulations/hpc_queuing/submission_systems/dummy.py +++ b/pygromos/simulations/hpc_queuing/submission_systems/dummy.py @@ -17,7 +17,7 @@ def __init__( nmpi: int = 1, job_duration: str = "24:00", submission: bool = True, - enviroment=None, + environment=None, ): super().__init__( verbose=verbose, @@ -25,7 +25,7 @@ def __init__( nomp=nomp, job_duration=job_duration, submission=submission, - enviroment=enviroment, + environment=environment, ) def submit_to_queue(self, sub_job: Submission_job) -> Union[int, None]: diff --git a/pygromos/simulations/hpc_queuing/submission_systems/local.py b/pygromos/simulations/hpc_queuing/submission_systems/local.py index 18468ff1..fa3195bf 100644 --- a/pygromos/simulations/hpc_queuing/submission_systems/local.py +++ b/pygromos/simulations/hpc_queuing/submission_systems/local.py @@ -21,7 +21,7 @@ def __init__( nmpi: int = 1, job_duration: str = "24:00", verbose: bool = False, - enviroment=None, + environment=None, zip_trajectories: bool = True, ): super().__init__( @@ -30,7 +30,7 @@ def __init__( nomp=nomp, job_duration=job_duration, submission=submission, - enviroment=enviroment, + environment=environment, zip_trajectories=zip_trajectories, ) @@ -49,8 +49,8 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: sub_job.command = sub_job.command.strip() # remove trailing linebreaks - if self.nomp >= 1: - command = "export OMP_NUM_THREADS=" + str(self.nomp) + ";\n " + sub_job.command + "" + if self._nomp >= 1: + command = "export OMP_NUM_THREADS=" + str(self._nomp) + ";\n " + sub_job.command + "" else: command = sub_job.command @@ -60,7 +60,7 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: command_file.write(command.replace("&& ", ";\n") + ";\n") command_file.close() command = command_file_path - bash.execute("chmod +x " + command_file_path, env=self._enviroment) + bash.execute("chmod +x " + command_file_path, env=self.environment) # finalize string @@ -68,7 +68,7 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: print("Submission Command: \t", " ".join(command)) if self.submission: try: - process = bash.execute(command=command, catch_STD=True, env=self._enviroment) + process = bash.execute(command=command, catch_STD=True, env=self.environment) std_out_buff = map(str, process.stdout.readlines()) std_out = "\t" + "\n\t".join(std_out_buff) @@ -100,8 +100,8 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: if isinstance(sub_job.submit_from_dir, str) and os.path.isdir(sub_job.submit_from_dir): submission_string += "cd " + sub_job.submit_from_dir + " && " - if self.nomp > 1: - command = submission_string + " export OMP_NUM_THREADS=" + str(self.nomp) + " && " + sub_job.command + if self._nomp > 1: + command = submission_string + " export OMP_NUM_THREADS=" + str(self._nomp) + " && " + sub_job.command else: command = submission_string + sub_job.command @@ -111,7 +111,7 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: try: for jobID in range(sub_job.start_job, sub_job.end_job + 1): std_out_buff = bash.execute( - command="export JOBID=" + str(jobID) + " && " + command, env=self._enviroment + command="export JOBID=" + str(jobID) + " && " + command, env=self.environment ) std_out = "\n".join(std_out_buff.readlines()) if self.verbose: diff --git a/pygromos/simulations/hpc_queuing/submission_systems/lsf.py b/pygromos/simulations/hpc_queuing/submission_systems/lsf.py index 4c428314..76b5d612 100644 --- a/pygromos/simulations/hpc_queuing/submission_systems/lsf.py +++ b/pygromos/simulations/hpc_queuing/submission_systems/lsf.py @@ -28,7 +28,7 @@ def __init__( job_duration: str = "24:00", max_storage: float = 1000, verbose: bool = False, - enviroment=None, + environment=None, block_double_submission: bool = True, bjobs_only_same_host: bool = False, chain_prefix: str = "done", @@ -44,7 +44,7 @@ def __init__( job_duration=job_duration, max_storage=max_storage, submission=submission, - enviroment=enviroment, + environment=environment, block_double_submission=block_double_submission, chain_prefix=chain_prefix, begin_mail=begin_mail, @@ -72,7 +72,7 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: submission_string = "" # QUEUE checking to not double submit - if self._block_double_submission and self.submission: + if self._block_double_submission and self._submission: if self.verbose: print("check queue") ids = list(self.search_queue_for_jobname(sub_job.jobName).index) @@ -95,7 +95,7 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: submission_string += "bsub " submission_string += " -J" + sub_job.jobName + " " - submission_string += " -W " + str(self.job_duration) + " " + submission_string += " -W " + str(self._job_duration) + " " if not isinstance(sub_job.post_execution_command, type(None)): submission_string += '-Ep "' + sub_job.post_execution_command + '" ' @@ -109,28 +109,28 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: if isinstance(sub_job.errLog, str): submission_string += " -e " + sub_job.errLog - nCPU = self.nmpi * self.nomp + nCPU = self._nmpi * self._nomp submission_string += " -n " + str(nCPU) + " " # TODO: add GPU support # add_string = "" # add_string= "-R \"select[model==XeonGold_5118 || model==XeonGold_6150 || model==XeonE3_1585Lv5 || model==XeonE3_1284Lv4 || model==XeonE7_8867v3 || model == XeonGold_6140 || model==XeonGold_6150 ]\"" - if isinstance(self.max_storage, int): - submission_string += " -R rusage[mem=" + str(self.max_storage) + "] " + if isinstance(self._max_storage, int): + submission_string += " -R rusage[mem=" + str(self._max_storage) + "] " if isinstance(sub_job.queue_after_jobID, (int, str)) and ( sub_job.queue_after_jobID != 0 or sub_job.queue_after_jobID != "0" ): - submission_string += ' -w "' + self.chain_prefix + "(" + str(sub_job.queue_after_jobID) + ')" ' + submission_string += ' -w "' + self._chain_prefix + "(" + str(sub_job.queue_after_jobID) + ')" ' - if self.begin_mail: + if self._begin_mail: submission_string += " -B " - if self.end_mail: + if self._end_mail: submission_string += " -N " sub_job.command = sub_job.command.strip() # remove trailing line breaks - if self.nomp >= 1: - command = '"export OMP_NUM_THREADS=' + str(self.nomp) + ";\n " + sub_job.command + '"' + if self._nomp >= 1: + command = "export OMP_NUM_THREADS=" + str(self._nomp) + ";\n " + sub_job.command + " " else: command = "\n " + sub_job.command + "" @@ -143,16 +143,16 @@ def submit_to_queue(self, sub_job: Submission_job) -> int: command_file.close() command = command_file_path - bash.execute("chmod +x " + command_file_path, env=self._enviroment) + bash.execute("chmod +x " + command_file_path, env=self._environment) # finalize string submission_string = list(map(lambda x: x.strip(), submission_string.split())) + [command] if self.verbose: print("Submission Command: \t", " ".join(submission_string)) - if self.submission and not self._dummy: + if self._submission and not self._dummy: try: - out_process = bash.execute(command=submission_string, catch_STD=True, env=self._enviroment) + out_process = bash.execute(command=submission_string, catch_STD=True, env=self._environment) std_out = "\n".join(map(str, out_process.stdout.readlines())) # next sopt_job is queued with id: @@ -191,7 +191,7 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: """ # QUEUE checking to not double submit - if self.submission and self._block_double_submission: + if self._submission and self._block_double_submission: if self.verbose: print("check queue") ids = self.search_queue_for_jobname(sub_job.jobName) @@ -216,7 +216,7 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: jobName = str(sub_job.jobName) + "[" + str(sub_job.start_job) + "-" + str(sub_job.end_job) + "]%" + str(jobLim) - submission_string += 'bsub -J " ' + jobName + ' " -W "' + str(self.job_duration) + '" ' + submission_string += 'bsub -J " ' + jobName + ' " -W "' + str(self._job_duration) + '" ' if isinstance(sub_job.jobGroup, str): submission_string += " -g " + sub_job.jobGroup + " " @@ -230,33 +230,33 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: if isinstance(sub_job.errLog, str): submission_string += " -eo " + sub_job.errLog - nCPU = self.nmpi * self.nomp + nCPU = self._nmpi * self._nomp submission_string += " -n " + str(nCPU) + " " if isinstance(self.max_storage, int): - submission_string += ' -R "rusage[mem=' + str(self.max_storage) + ']" ' + submission_string += ' -R "rusage[mem=' + str(self._max_storage) + ']" ' if isinstance(sub_job.queue_after_jobID, (int, str)): - submission_string += " -w " + self.chain_prefix + "(" + str(sub_job.queue_after_jobID) + ')" ' + submission_string += " -w " + self._chain_prefix + "(" + str(sub_job.queue_after_jobID) + ')" ' - if self.begin_mail: + if self._begin_mail: submission_string += " -B " - if self.end_mail: + if self._end_mail: submission_string += " -N " - if self.nomp > 1: - command = ' " export OMP_NUM_THREADS=' + str(self.nomp) + " && " + sub_job.command + '"' + if self._nomp > 1: + command = " export OMP_NUM_THREADS=" + str(self._nomp) + " && " + sub_job.command + " " else: - command = ' "' + sub_job.command + '"' + command = " " + sub_job.command + " " # finalize string submission_string = list(map(lambda x: x.strip(), submission_string.split())) + [command] if self.verbose: print("Submission Command: \t", " ".join(submission_string)) - if self.submission and not self._dummy: + if self._submission and not self._dummy: try: - std_out_buff = bash.execute(command=submission_string, env=self._enviroment) + std_out_buff = bash.execute(command=submission_string, env=self._environment) std_out = "\n".join(std_out_buff.readlines()) # next sopt_job is queued with id: @@ -276,33 +276,6 @@ def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int: sub_job.jobID = job_id return int(job_id) - def get_script_generation_command(self, var_name: str = None, var_prefixes: str = "") -> str: - name = self.__class__.__name__ - if var_name is None: - var_name = var_prefixes + name - - gen_cmd = "#Generate " + name + "\n" - gen_cmd += "from " + self.__module__ + " import " + name + " as " + name + "_obj" + "\n" - gen_cmd += ( - var_name - + " = " - + name - + "_obj(submission=" - + str(self.submission) - + ", verbose=" - + str(self.verbose) - + ", nmpi=" - + str(self.nmpi) - + ", nomp=" - + str(self.nomp) - + ", max_storage=" - + str(self.max_storage) - + ', job_duration="' - + str(self.job_duration) - + '")\n\n' - ) - return gen_cmd - """ Job Queue Managment """ @@ -321,27 +294,27 @@ def get_queued_jobs(self) -> pd.DataFrame: if hasattr(self, "_job_queue_time_stamp"): last_update = datetime.now() - self._job_queue_time_stamp check_job_list = last_update.seconds > self._refresh_job_queue_list_all_s - if not self.submission: # shortcut to reduce queue calls! - self.job_queue_list = pd.DataFrame( + if not self._submission: # shortcut to reduce queue calls! + self._job_queue_list = pd.DataFrame( columns=["JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME".split()] ) - return self.job_queue_list + return self._job_queue_list if check_job_list: # try getting the lsf queue if not self._dummy: try: # get all running and pending jobs if self.bjobs_only_same_host: - out_process = bash.execute("bjobs -w", catch_STD=True) - else: out_process = bash.execute("bjobs -w | grep '$HOSTNAME|JOBID'", catch_STD=True) + else: + out_process = bash.execute("bjobs -w", catch_STD=True) job_list_str = list(map(lambda x: x.decode("utf-8"), out_process.stdout.readlines())) # get all finished jobs if self.bjobs_only_same_host: - out_process = bash.execute("bjobs -wd", catch_STD=True) - else: out_process = bash.execute("bjobs -wd | grep '$HOSTNAME|JOBID'", catch_STD=True) + else: + out_process = bash.execute("bjobs -wd", catch_STD=True) job_list_finished_str = list(map(lambda x: x.decode("utf-8"), out_process.stdout.readlines())) self._job_queue_time_stamp = datetime.now() except Exception as err: @@ -372,9 +345,9 @@ def get_queued_jobs(self) -> pd.DataFrame: values = [jobID, user, status, queue, from_host, exec_host, job_name, submit_time] jobs_dict.update({jobID: {key: value for key, value in zip(header, values)}}) - self.job_queue_list = pd.DataFrame(jobs_dict, index=None).T + self._job_queue_list = pd.DataFrame(jobs_dict, index=None).T else: - self.job_queue_list = pd.DataFrame( + self._job_queue_list = pd.DataFrame( columns=[ "JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME".split() ] @@ -383,11 +356,11 @@ def get_queued_jobs(self) -> pd.DataFrame: if self.verbose: print("Skipping refresh of job list, as the last update is " + str(last_update) + "s ago") pass - return self.job_queue_list + return self._job_queue_list def search_queue_for_jobid(self, job_id: int) -> pd.DataFrame: self.get_queued_jobs() - return self.job_queue_list.where(self.job_queue_list.JOBID == job_id).dropna() + return self._job_queue_list.where(self._job_queue_list.JOBID == job_id).dropna() def search_queue_for_jobname(self, job_name: str, regex: bool = False) -> pd.DataFrame: """search_queue_for_jobname @@ -407,9 +380,9 @@ def search_queue_for_jobname(self, job_name: str, regex: bool = False) -> pd.Dat self.get_queued_jobs() if regex: - return self.job_queue_list.where(self.job_queue_list.JOB_NAME.str.match(job_name)).dropna() + return self._job_queue_list.where(self._job_queue_list.JOB_NAME.str.match(job_name)).dropna() else: - return self.job_queue_list.where(self.job_queue_list.JOB_NAME == job_name).dropna() + return self._job_queue_list.where(self._job_queue_list.JOB_NAME == job_name).dropna() """ kill jobs diff --git a/pygromos/simulations/modules/general_simulation_modules.py b/pygromos/simulations/modules/general_simulation_modules.py index fc5a9b0d..d5ae71db 100644 --- a/pygromos/simulations/modules/general_simulation_modules.py +++ b/pygromos/simulations/modules/general_simulation_modules.py @@ -1,5 +1,6 @@ import os import sys +import glob import traceback import time import warnings @@ -33,6 +34,8 @@ def simulation( reinitialize_every_run=False, analysis_script: callable = simulation_analysis.do, analysis_control_dict: dict = None, + _no_double_submit_check: bool = False, + _work_dir: str = None, verbose: bool = True, verbose_lvl: int = 1, _template_imd_path: str = None, @@ -80,6 +83,8 @@ def simulation( try: try: gromos_system = deepcopy(in_gromos_simulation_system) + if previous_simulation_run is not None: + gromos_system._last_jobID = previous_simulation_run # check if override dir is given and set project to correct location if override_project_dir is not None: @@ -171,10 +176,14 @@ def simulation( "analysis_script_path": in_analysis_script_path, "initialize_first_run": initialize_first_run, "reinitialize_every_run": reinitialize_every_run, + "previous_job_ID": gromos_system._last_jobID, + "_no_double_submit_check": _no_double_submit_check, + "_work_dir": _work_dir, "verbose": verbose, "verbose_lvl": verbose_lvl, } ) + try: in_scheduler_script_path = utils.write_job_script( # noqa: F841 out_script_path=step_dir + "/schedule_MD_job.py", @@ -195,26 +204,21 @@ def simulation( # warnings.warn("Skipping active submission, as result CNF was found: \n"+out_analysis_cnf) last_jobID = 0 else: - last_jobID = simulation_scheduler.do( - in_simSystem=gromos_system, - out_dir_path=out_simulation_dir, - simulation_run_num=simulation_runs, - equilibration_run_num=equilibration_runs, - submission_system=submission_system, - previous_job_ID=previous_simulation_run, - initialize_first_run=initialize_first_run, - reinitialize_every_run=reinitialize_every_run, - analysis_script_path=in_analysis_script_path, - verbose=verbose, - verbose_lvl=verbose_lvl, - ) + last_jobID = simulation_scheduler.do(**MD_job_vars) except Exception as err: traceback.print_exception(*sys.exc_info()) raise Exception("Could not submit the commands\n\t" + "\n\t".join(map(str, err.args))) time.sleep(time_wait_s_for_filesystem) + # Return the promise final system - if os.path.exists(out_analysis_cnf): + if in_analysis_script_path is None and os.path.exists(out_simulation_dir): + cnfs = list( + sorted(glob.glob(out_simulation_dir + "/*/*cnf"), key=lambda x: int(x.split("_")[-1].split(".")[0])) + ) + last_cnf = cnfs[-1] + gromos_system.cnf = cnf.Cnf(last_cnf) + elif os.path.exists(out_analysis_cnf): gromos_system.cnf = cnf.Cnf(out_analysis_cnf) else: gromos_system.cnf = cnf.Cnf(in_value=None) diff --git a/pygromos/simulations/modules/preset_simulation_modules.py b/pygromos/simulations/modules/preset_simulation_modules.py index 69e4261b..fb548e32 100644 --- a/pygromos/simulations/modules/preset_simulation_modules.py +++ b/pygromos/simulations/modules/preset_simulation_modules.py @@ -24,6 +24,7 @@ def emin( equilibration_runs: int = 0, previous_simulation_run: int = None, _template_imd_path: str = template_emin, + _no_double_submit_check: bool = False, initialize_first_run=False, analysis_script: callable = simulation_analysis.do, verbose: bool = True, @@ -54,6 +55,7 @@ def emin( analysis_control_dict=template_emin_control_dict, analysis_script=analysis_script, _template_imd_path=_template_imd_path, + _no_double_submit_check=_no_double_submit_check, verbose=verbose, ) @@ -70,6 +72,7 @@ def md( reinitialize_every_run=False, previous_simulation_run: int = None, _template_imd_path: str = template_md, + _no_double_submit_check: bool = False, analysis_script: callable = simulation_analysis.do, verbose: bool = True, ) -> Tuple[Gromos_System, int]: @@ -85,6 +88,7 @@ def md( equilibration_runs=equilibration_runs, analysis_script=analysis_script, _template_imd_path=_template_imd_path, + _no_double_submit_check=_no_double_submit_check, verbose=verbose, ) @@ -101,6 +105,7 @@ def sd( reinitialize_every_run=False, previous_simulation_run: int = None, _template_imd_path: str = template_sd, + _no_double_submit_check: bool = False, analysis_script: callable = simulation_analysis.do, verbose: bool = True, ) -> Tuple[Gromos_System, int]: @@ -116,6 +121,7 @@ def sd( equilibration_runs=equilibration_runs, analysis_script=analysis_script, _template_imd_path=_template_imd_path, + _no_double_submit_check=_no_double_submit_check, verbose=verbose, ) @@ -131,6 +137,7 @@ def thermalisation( equilibration_runs: int = 0, previous_simulation_run: int = None, _template_imd_path: str = template_sd, + _no_double_submit_check: bool = False, analysis_script: callable = simulation_analysis.do, verbose: bool = True, ) -> Tuple[Gromos_System, int]: @@ -158,6 +165,7 @@ def thermalisation( equilibration_runs=equilibration_runs, analysis_script=analysis_script, _template_imd_path=_template_imd_path, + _no_double_submit_check=_no_double_submit_check, verbose=verbose, ) @@ -173,5 +181,6 @@ def thermalisation( equilibration_runs=equilibration_runs, analysis_script=analysis_script, _template_imd_path=_template_imd_path, + _no_double_submit_check=_no_double_submit_check, verbose=verbose, ) diff --git a/pygromos/utils/bash.py b/pygromos/utils/bash.py index 377cba78..85c63c9c 100644 --- a/pygromos/utils/bash.py +++ b/pygromos/utils/bash.py @@ -259,8 +259,8 @@ def compress_tar( FileIO process return log. """ - - if out_path is not None: + print(out_path) + if out_path is None: out_path = in_path if not out_path.endswith(".tar.gz") and gunzip_compression: out_path += ".tar.gz" diff --git a/pygromos/utils/compiledProgram.py b/pygromos/utils/compiledProgram.py index 78f0e35d..40b27864 100644 --- a/pygromos/utils/compiledProgram.py +++ b/pygromos/utils/compiledProgram.py @@ -3,7 +3,7 @@ """ import inspect import functools -from typing import Union, Dict +from typing import Union, Dict, Callable from pygromos.utils import bash @@ -15,22 +15,72 @@ class _compiled_program: """ _bin: str - _dont_check_bin: bool + _force_bin_present: bool + _check_binary_paths: bool _found_binary_dir: Dict[str, bool] # found? binary dir _found_binary: Dict[str, bool] # found? binary _found_binary_paths: Dict[str, str] # the found binary paths. - def __init__(self, in_bin_dir: str, dummy: bool = False) -> Union[str, None]: + def __init__( + self, in_bin_dir: Union[str, None], _force_bin_present: bool = True, _check_binary_paths: bool = True + ) -> Union[str, None]: + """ + The _compiled_program parent class can be used, to ensure on runtime, that certain binaries are present. + + Parameters + ---------- + in_bin_dir : Union[str, None] + directory that should contain the binaries. If None, the assumption is made, that the path is part of the PATH variable. + _force_bin_present : bool, optional + if True, the check_binary or check_binary_folder will throw errors, if they don't find the targets, by default True + _dont_check_binary : bool, optional + This is a kill switch for all checks of this class., by default False + + Returns + ------- + Union[str, None] + _description_ + """ # init structures - self._dont_check_bin = dummy + self._force_bin_present = _force_bin_present + self._found_binary_dir = {} + self._found_binary = {} + self._found_binary_paths = {} + self._function_binary = {} + self._check_binary_paths = _check_binary_paths + + # Check initial status of binaries + if in_bin_dir is None or in_bin_dir == "None" or in_bin_dir == "": + self._bin = "" + elif in_bin_dir.endswith("/"): + self._bin = in_bin_dir + else: + self._bin = in_bin_dir + "/" + + self._check_binary_dir(in_bin_dir=self._bin) + self._check_all_binaries() + self.__wrap_programms_with_binary_checks() + + def __getstate__(self): + """ + preperation for pickling: + remove the non trivial pickling parts + """ + return { + "_bin": self._bin, + "_check_binary_paths": self._check_binary_paths, + "_force_bin_present": self._force_bin_present, + } + + def __setstate__(self, state): + self.__dict__ = state self._found_binary_dir = {} self._found_binary = {} self._found_binary_paths = {} self._function_binary = {} # Check initial status of binaries - self._bin = self._check_binary_dir(in_bin_dir=in_bin_dir) self._check_all_binaries() self.__wrap_programms_with_binary_checks() @@ -39,7 +89,7 @@ def __init__(self, in_bin_dir: str, dummy: bool = False) -> Union[str, None]: """ @property - def bin(self) -> Union[str, None]: + def bin(self) -> str: """ This attribute is the binary directory. """ @@ -50,9 +100,14 @@ def bin(self) -> Union[str, None]: @bin.setter def bin(self, in_bin_dir: str): - self._bin = self._check_binary_dir(in_bin_dir=in_bin_dir) - if not (self._bin == "" and self._bin.endswith("/")): - self._bin += "/" + if in_bin_dir is None or in_bin_dir == "None" or in_bin_dir == "": + self._bin = "" + elif in_bin_dir.endswith("/"): + self._bin = in_bin_dir + else: + self._bin = in_bin_dir + "/" + + self._check_binary_dir(in_bin_dir=in_bin_dir) self._check_all_binaries() self.__wrap_programms_with_binary_checks() @@ -80,7 +135,7 @@ def _check_binary(self, test_program: str) -> bool: If the binary dir was not found and _dont_check_bin was False (default: False) """ - if test_program in self._found_binary and self._found_binary[test_program]: + if (test_program in self._found_binary and self._found_binary[test_program]) or not self._check_binary_paths: return True elif self.bin is not None and bash.command_exists(self._bin + test_program): @@ -95,7 +150,7 @@ def _check_binary(self, test_program: str) -> bool: else: self._found_binary[test_program] = False - if self._dont_check_bin: + if self._force_bin_present: raise IOError( "No binary could be found! Please make sure, the program was compiled and the path were passed to this obj." + " provided binary path: " @@ -123,27 +178,31 @@ def _check_binary_dir(self, in_bin_dir: str) -> bool: IOError If the binary dir was not found and _dont_check_bin was False (default: False) """ - if in_bin_dir in self._found_binary_dir and self._found_binary_dir[in_bin_dir]: # did we already check this - return "" if (in_bin_dir is None) else in_bin_dir + if ( + in_bin_dir in self._found_binary_dir and self._found_binary_dir[in_bin_dir] + ) or not self._check_binary_paths: # did we already check this + return True elif isinstance(in_bin_dir, str) and in_bin_dir != "" and bash.directory_exists(in_bin_dir): self._found_binary_dir[in_bin_dir] = True - return in_bin_dir + if not in_bin_dir.endswith("/"): + in_bin_dir += "/" + return True - elif not self._dont_check_bin and (in_bin_dir is None or in_bin_dir == ""): + elif not self._force_bin_present and (in_bin_dir is None or in_bin_dir == "" or in_bin_dir == "None"): self._found_binary_dir[in_bin_dir] = True - return "" - + return True else: self._found_binary_dir[in_bin_dir] = False - if self._dont_check_bin: + if self._force_bin_present: raise IOError( "No binary directory could be found! Please make sure the directory exists! " + " and either pass the path to the binary directory or set the PATH variable. The given folder path was: " + str(in_bin_dir) ) + return False - def _check_all_binaries(self, force_present: bool = False) -> bool: + def _check_all_binaries(self, _force_bin_present: bool = False) -> bool: """ This function checks all present programs of this class, if the binary can be found and executed. It does not trigger an Exception, if a binary cannot be found, except force_present is True. @@ -162,13 +221,13 @@ def _check_all_binaries(self, force_present: bool = False) -> bool: """ funcs = {key: getattr(self, key) for key in dir(self) if (not key.startswith("_") and key != "bin")} - tmp_dont_heck_bin = self._dont_check_bin - self._dont_check_bin = force_present + tmp_dont_check_bin = self._force_bin_present + self._force_bin_present = self._force_bin_present for key, f in funcs.items(): binary = inspect.signature(f).parameters["_binary_name"].default - self._check_binary(binary) + self._check_binary(test_program=binary) self._function_binary[binary] = key - self._dont_check_bin = tmp_dont_heck_bin + self._force_bin_present = tmp_dont_check_bin return all(self._found_binary.values()) @@ -176,7 +235,7 @@ def _check_all_binaries(self, force_present: bool = False) -> bool: Utils for the binary wrapping to check binary on the fly. """ - def __check_binaries_decorator(self, func: callable) -> callable: + def _check_binaries_decorator(self, func: Callable) -> Callable: """ This function wrapper adds a binary check before, the function is executed. @@ -198,6 +257,8 @@ def __check_binaries_decorator(self, func: callable) -> callable: @functools.wraps(func) def control_binary(*args, **kwargs) -> any: + # print("binaryChecker", func.__name__, args, kwargs) + func_signature = inspect.signature(func) if "_binary_name" in kwargs: self._check_binary(self._bin + kwargs["_binary_name"]) @@ -207,6 +268,8 @@ def control_binary(*args, **kwargs) -> any: raise Exception( "Could not find Binary name in function signature: " + str(func) + "\n found: " + str(kwargs) ) + + args = list(filter(lambda x: x != self, args)) # avoid double selfing return func(self, *args, **kwargs) return control_binary @@ -220,10 +283,14 @@ def __wrap_programms_with_binary_checks(self, remove=False): remove : bool, optional remove all wrappers?, by default False """ - v = {} - for binary, func in self._function_binary.items(): - if remove or self._found_binary[binary]: - v[func] = getattr(self.__class__, func) - else: - v[func] = self.__check_binaries_decorator(getattr(self, func)) - self.__dict__.update(v) + if self._check_binary_paths: + pass + + else: + v = {} + for binary, func in self._function_binary.items(): + if remove: # or self._found_binary[binary]: + v[func] = getattr(self.__class__, func) + else: + v[func] = self._check_binaries_decorator(getattr(self, func)) + self.__dict__.update(v)