Merge branch 'main' into executor

.ci_support/environment.yml

@@ -5,8 +5,8 @@ dependencies:
   - coveralls
   - coverage
   - codacy-coverage
-  - pandas =2.0.1
+  - pandas =2.0.3
   - pyyaml =6.0
   - jinja2 =3.1.2
-  - paramiko =3.1.0
+  - paramiko =3.2.0
   - tqdm =4.65.0

.github/workflows/UpdateDependabotPR.yml

@@ -15,11 +15,13 @@ jobs:
           fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
           token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
       - name: UpdateEnvironmentFile
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
         shell: bash -l {0}
         run: |
-          package=$(echo "${{ github.event.pull_request.title }}" | awk '{print $2}')
-          from=$(echo "${{ github.event.pull_request.title }}" | awk '{print $4}')
-          to=$(echo "${{ github.event.pull_request.title }}" | awk '{print $6}')
+          package=$(echo "$PR_TITLE" | awk '{print $2}')
+          from=$(echo "$PR_TITLE" | awk '{print $4}')
+          to=$(echo "$PR_TITLE" | awk '{print $6}')
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment.yml
       - name: UpdateDependabotPR commit
         run: |

pysqa/__main__.py

@@ -1,6 +1,6 @@
 import sys
-from pysqa.cmd import command_line
 
+from pysqa.cmd import command_line
 
 if __name__ == "__main__":
     command_line(sys.argv[1:])

pysqa/cmd.py

@@ -1,7 +1,8 @@
-import sys
-import os
-import json
 import getopt
+import json
+import os
+import sys
+
 from pysqa.queueadapter import QueueAdapter
 from pysqa.utils.execute import execute_command
 

pysqa/ext/modular.py

@@ -2,6 +2,7 @@
 # Copyright (c) Jan Janssen
 
 import pandas
+
 from pysqa.utils.basic import BasisQueueAdapter
 from pysqa.utils.execute import execute_command
 

pysqa/ext/remote.py

@@ -3,10 +3,12 @@
 
 import json
 import os
+import warnings
+
 import pandas
 import paramiko
-import warnings
 from tqdm import tqdm
+
 from pysqa.utils.basic import BasisQueueAdapter
 from pysqa.utils.execute import execute_command
 

pysqa/queueadapter.py

@@ -2,10 +2,11 @@
 # Copyright (c) Jan Janssen
 
 import os
+
+from pysqa.ext.modular import ModularQueueAdapter
 from pysqa.utils.basic import BasisQueueAdapter
 from pysqa.utils.config import read_config
 from pysqa.utils.execute import execute_command
-from pysqa.ext.modular import ModularQueueAdapter
 
 __author__ = "Jan Janssen"
 __copyright__ = "Copyright 2019, Jan Janssen"

pysqa/utils/basic.py

@@ -3,12 +3,15 @@
 
 import getpass
 import importlib
-from jinja2 import Template
 import os
 import re
+
 import pandas
-from pysqa.utils.queues import Queues
+from jinja2 import Template
+from jinja2.exceptions import TemplateSyntaxError
+
 from pysqa.utils.execute import execute_command
+from pysqa.utils.queues import Queues
 
 
 class BasisQueueAdapter(object):
@@ -483,7 +486,16 @@ def _load_templates(queue_lst_dict, directory="."):
         for queue_dict in queue_lst_dict.values():
             if "script" in queue_dict.keys():
                 with open(os.path.join(directory, queue_dict["script"]), "r") as f:
-                    queue_dict["template"] = Template(f.read())
+                    try:
+                        queue_dict["template"] = Template(f.read())
+                    except TemplateSyntaxError as error:
+                        raise TemplateSyntaxError(
+                            message="File: "
+                            + queue_dict["script"]
+                            + " - "
+                            + error.message,
+                            lineno=error.lineno,
+                        )
 
     @staticmethod
     def _value_error_if_none(value):

pysqa/wrapper/flux.py

@@ -1,6 +1,7 @@
 # coding: utf-8
-from flux.job import JobID
 import pandas
+from flux.job import JobID
+
 from pysqa.wrapper.generic import SchedulerCommands
 
 
@@ -19,7 +20,9 @@ def get_queue_status_command(self):
 
     @staticmethod
     def get_job_id_from_output(queue_submit_output):
-        return JobID(queue_submit_output.splitlines()[-1].rstrip().lstrip().split()[-1])
+        return int(
+            JobID(queue_submit_output.splitlines()[-1].rstrip().lstrip().split()[-1])
+        )
 
     @staticmethod
     def convert_queue_status(queue_status_output):
@@ -40,4 +43,5 @@ def convert_queue_status(queue_status_output):
         )
         df.loc[df.status == "R", "status"] = "running"
         df.loc[df.status == "S", "status"] = "pending"
+        df.loc[df.status == "CD", "status"] = "finished"
         return df

pysqa/wrapper/gent.py

@@ -3,8 +3,8 @@
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
 import pandas
-from pysqa.wrapper.slurm import SlurmCommands
 
+from pysqa.wrapper.slurm import SlurmCommands
 
 __author__ = "Jan Janssen"
 __copyright__ = (

pysqa/wrapper/sge.py

@@ -2,8 +2,9 @@
 # Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
-import pandas
 import defusedxml.ElementTree as ETree
+import pandas
+
 from pysqa.wrapper.generic import SchedulerCommands
 
 __author__ = "Jan Janssen"

pysqa/wrapper/slurm.py

@@ -3,8 +3,8 @@
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
 import pandas
-from pysqa.wrapper.generic import SchedulerCommands
 
+from pysqa.wrapper.generic import SchedulerCommands
 
 __author__ = "Jan Janssen"
 __copyright__ = (

pysqa/wrapper/torque.py

@@ -3,7 +3,9 @@
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
 import re
+
 import pandas as pd
+
 from pysqa.wrapper.generic import SchedulerCommands
 
 __author__ = "Jan Janssen"

setup.py

@@ -29,11 +29,11 @@
 
     keywords='pysqa',
     packages=find_packages(exclude=["*tests*"]),
-    install_requires=['jinja2==3.1.2', 'pandas==2.0.1', 'pyyaml==6.0'],
+    install_requires=['jinja2==3.1.2', 'pandas==2.0.3', 'pyyaml==6.0'],
     extras_require={
         "sge": ['defusedxml==0.7.1'],
-        "remote": ['paramiko==3.1.0', 'tqdm==4.65.0'],
-        "executor": ['pympipool==0.5.4', 'cloudpickle==2.2.1']
+        "remote": ['paramiko==3.2.0', 'tqdm==4.65.0'],
+        "executor": ['pympipool==0.5.4', 'cloudpickle==2.2.1'],
     },
     cmdclass=versioneer.get_cmdclass(),
     entry_points={

tests/config/bad_template/bad.sh

@@ -0,0 +1 @@
+{%- endif %}

tests/config/bad_template/queue.yaml

@@ -0,0 +1,4 @@
+queue_type: SLURM
+queue_primary: bad
+queues:
+  bad: {cores_max: 100, cores_min: 10, run_time_max: 259200, script: bad.sh}

tests/test_basic.py

@@ -3,6 +3,7 @@
 
 import os
 import unittest
+from jinja2.exceptions import TemplateSyntaxError
 from pysqa import QueueAdapter
 from pysqa.utils.basic import BasisQueueAdapter
 
@@ -21,9 +22,12 @@ def setUpClass(cls):
         cls.path = os.path.dirname(os.path.abspath(__file__))
 
     def test_missing_config(self):
-        self.assertRaises(
-            ValueError, QueueAdapter, directory=os.path.join(self.path, "config/error")
-        )
+        with self.assertRaises(ValueError):
+            QueueAdapter(directory=os.path.join(self.path, "config/error"))
+
+    def test_bad_queue_template(self):
+        with self.assertRaises(TemplateSyntaxError):
+            QueueAdapter(directory=os.path.join(self.path, "config/bad_template"))
 
 
 class TestBasisQueueAdapter(unittest.TestCase):