Fixed numerous small bugs, improved the Docker setup for testing and …

…fixed tests

Dockerfile

@@ -8,9 +8,9 @@ ENV PYTHONUNBUFFERED 1
 ENV CALCUS_SCR_HOME "/calcus/scr"
 ENV CALCUS_RESULTS_HOME "/calcus/results"
 ENV CALCUS_KEY_HOME "/calcus/keys"
-ENV CALCUS_TEST_SCR_HOME "/calcus/frontend/tests/scr"
-ENV CALCUS_TEST_RESULTS_HOME "/calcus/frontend/tests/results"
-ENV CALCUS_TEST_KEY_HOME "/calcus/frontend/tests/keys"
+ENV CALCUS_TEST_SCR_HOME "/calcus/scratch/scr"
+ENV CALCUS_TEST_RESULTS_HOME "/calcus/scratch/results"
+ENV CALCUS_TEST_KEY_HOME "/calcus/scratch/keys"
 
 ENV EBROOTORCA "/binaries/orca"
 ENV GAUSS_EXEDIR "/binaries/g16"
@@ -43,4 +43,8 @@ RUN adduser --disabled-password --gecos '' calcus
 FROM calcus_user as calcus_dev
 
 ADD ./test-requirements.txt /calcus/test-requirements.txt
-RUN pip install -r /calcus/test-requirements.txt
+RUN  pip install -r /calcus/test-requirements.txt
+RUN mkdir -p /calcus/scratch/keys
+RUN mkdir -p /calcus/scratch/scr
+RUN mkdir -p /calcus/scratch/results
+RUN chown -R calcus:calcus /calcus/scratch

calcus/settings.py

@@ -144,6 +144,9 @@
     }
 }
 
+if os.environ.get("IS_TEST_CLUSTER_DAEMON", "") != "":
+    DATABASES["default"]["NAME"] = "test_calcus"
+
 AUTH_PASSWORD_VALIDATORS = [
     {
         "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
@@ -183,7 +186,7 @@
 
 if IS_TEST:
     EMAIL_BACKEND = "django.core.mail.backends.filebased.EmailBackend"
-    EMAIL_FILE_PATH = os.path.join(BASE_DIR, "frontend", "tests", "sent_emails")
+    EMAIL_FILE_PATH = os.path.join(BASE_DIR, "scratch", "sent_emails")
 
 MEDIA_URL = "/media/"
 

frontend/calculation_unittest.py

@@ -10,15 +10,10 @@
 from .models import *
 from .gen_calc import gen_calc
 from .tasks import run_calc
+from .calcusliveserver import SCR_DIR, RESULTS_DIR
 
-
-dir_path = os.path.dirname(os.path.realpath(__file__))
 base_cwd = os.getcwd()
 
-tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
-
 
 class CalculationUnitTest(TestCase):
     @classmethod

frontend/calcusliveserver.py

@@ -49,12 +49,12 @@
 from .models import *
 from .environment_variables import *
 
+tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
-tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
-KEYS_DIR = os.path.join(tests_dir, "keys")
+SCR_DIR = "/calcus/scratch/scr"
+RESULTS_DIR = "/calcus/scratch/results"
+KEYS_DIR = "/calcus/scratch/keys"
 
 HEADLESS = os.getenv("CALCUS_HEADLESS")
 
@@ -269,18 +269,21 @@ def calc_input_params(self, params):
             self.driver.find_element_by_xpath(
                 f"//*[@id='calc_type']/option[text()='{params['type']}']"
             ).click()
+            self.wait_for_ajax()
 
         if "project" in params.keys():
             self.driver.find_element_by_xpath(
                 f"//*[@id='calc_project']/option[text()='{params['project']}']"
             ).click()
+            self.wait_for_ajax()
 
         if "solvation_model" in params.keys():
             self.driver.find_element_by_xpath(
                 "//*[@id='calc_solvation_model']/option[text()='{}']".format(
                     params["solvation_model"]
                 )
             ).click()
+            self.wait_for_ajax()
 
         if "solvation_radii" in params.keys():
             self.driver.find_element_by_xpath(
@@ -470,6 +473,14 @@ def handle_constraint(constraint, ind):
     def calc_launch(self):
         submit = self.driver.find_element_by_id("submit_button")
         submit.click()
+        self.wait_for_ajax()
+        try:
+            msg = self.driver.find_element_by_id("form_error_msg")
+        except selenium.common.exceptions.NoSuchElementException:
+            pass
+        else:
+            if msg.text != "":
+                raise Exception(f"Got error while submitting calculation: {msg.text}")
 
     def get_confirmed_specifications(self):
         assert self.is_on_page_ensemble()

frontend/cluster_daemon.py

@@ -123,9 +123,13 @@ def access_test(self, id, password):
     def check_binaries(self, conn):
         lock = self.locks[conn[0].id]
         ls_home = tasks.direct_command("ls ~", conn, lock)
-
-        if "calcus" not in ls_home:
-            return ConnectionCodes.NO_CALCUS_FOLDER
+        if isinstance(ls_home, ErrorCodes):
+            logger.warning(
+                f"Could not check the presence of a remote calcus folder: {ls_home}"
+            )
+        else:
+            if "calcus" not in ls_home:
+                return ConnectionCodes.NO_CALCUS_FOLDER
 
         return ""
 
@@ -180,7 +184,6 @@ def delete_access(self, access_id):
             logger.warning(f"No lock for connection with id {access_id} in memory")
 
     def job(self, calc):
-
         access_id = calc.order.resource.id
         tasks.connections[threading.get_ident()] = self.connections[access_id]
         tasks.locks[threading.get_ident()] = self.locks[access_id]
@@ -200,10 +203,6 @@ def connect(self, access_id, password):
         except ClusterAccess.DoesNotExist:
             access = None
 
-        logger.warning(
-            "Accesses: "
-            + str(list([i.cluster_address for i in ClusterAccess.objects.all()]))
-        )
         if not isinstance(c, int):
             if not access:
                 logger.warning(f"ClusterAccess with id {access_id} does not exist")

frontend/tasks.py

@@ -69,7 +69,6 @@
 )
 logger = logging.getLogger(__name__)
 
-dir_path = os.path.dirname(os.path.realpath(__file__))
 tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
 
 
@@ -101,9 +100,9 @@ def retry():
         response = conn[1].run("source ~/.bashrc; " + command, hide="both")
     except invoke.exceptions.UnexpectedExit as e:
         lock.release()
-        if (
-            e.result.exited == 1
-            and e.result.stderr.find("Invalid job id specified") != -1
+        if e.result.exited == 1 and (
+            e.result.stderr.find("Invalid job id specified") != -1
+            or e.result.stdout.find("Invalid job id specified") != -1
         ):
             return []
 
@@ -214,14 +213,15 @@ def wait_until_done(calc, conn, lock, ind=0):
 
     while True:
         output = direct_command(f"squeue -j {job_id}", conn, lock)
-        if not isinstance(output, ErrorCodes) and len(output) > 0:
-            _output = [i for i in output if i.strip() != ""]
+        _output = [i for i in output if i.strip() != ""]
+        if not isinstance(output, ErrorCodes) and len(_output) > 1:
             logger.info(f"Waiting ({job_id})")
             try:
                 status = _output[1].split()[4]
             except IndexError:
                 logger.warning("Got unexpected str: " + str(_output))
                 return
+
             if status == "R" and calc.status == 0:
                 calc.date_started = timezone.now()
                 calc.status = 1
@@ -449,14 +449,12 @@ def setup_cached_calc(calc):
     if not index:
         return False
 
-    if os.path.isdir(os.path.join(tests_dir, "scr", str(calc.id))):
-        rmtree(os.path.join(tests_dir, "scr", str(calc.id)))
-
-    # shutil.copytree(os.path.join(tests_dir, "cache", index), os.path.join(tests_dir, "scr", str(calc.id)))
+    if os.path.isdir(f"/calcus/scratch/scr/{calc.id}"):
+        rmtree(f"/calcus/scratch/scr/{calc.id}")
 
     os.symlink(
         os.path.join(tests_dir, "cache", index),
-        os.path.join(tests_dir, "scr", str(calc.id)),
+        f"/calcus/scratch/scr/{calc.id}",
     )
     logger.info(f"Using cache ({index})")
     return True
@@ -3362,14 +3360,8 @@ def get_calc(calc_id):
     in_file = os.path.join(workdir, "in.xyz")
 
     if calc.status == 0:
-        try:
-            os.mkdir(res_dir)
-        except OSError:
-            logger.info(f"Directory already exists: {res_dir}")
-        try:
-            os.mkdir(workdir)
-        except OSError:
-            logger.info(f"Directory already exists: {res_dir}")
+        os.makedirs(res_dir, exist_ok=True)
+        os.makedirs(workdir, exist_ok=True)
 
         with open(in_file, "w") as out:
             out.write(clean_xyz(calc.structure.xyz_structure))
@@ -3448,7 +3440,7 @@ def get_calc(calc_id):
     ):
         test_name = os.environ["TEST_NAME"]
         shutil.copytree(
-            os.path.join(tests_dir, "scr", str(calc.id)),
+            f"/calcus/scratch/scr/{calc.id}",
             os.path.join(tests_dir, "cache", test_name),
             dirs_exist_ok=True,
         )

frontend/templates/frontend/launch.html

@@ -153,7 +153,7 @@
 			{% endif %}
 
 			{% if structures %}
-			data.push({"name": "starting_structs", "value": {{ structures }}});
+			data.push({"name": "starting_structs", "value": "{{ structures }}"});
 			{% endif %}
 
 			{% if calc %}
@@ -168,9 +168,24 @@
 			}
 			{% endif %}
 
-			num_files = $("#file_structure")[0].files.length;
+			upload_el = $("#file_structure");
+			if(upload_el.length != 0) {
+				num_files = upload_el[0].files.length;
+			}
+			else {
+				num_files = 0;
+			}
 			data.push({"name": "num_files", "value": num_files});
 
+			upload_aux_el = $("#aux_file_structure");
+			if(upload_aux_el.length != 0) {
+				num_aux_files = upload_aux_el[0].files.length;
+			}
+			else {
+				num_aux_files = 0;
+			}
+			data.push({"name": "num_aux_files", "value": num_aux_files});
+
 			$.ajax({
 				data: data, 
 				headers: {

frontend/test_calculations.py

@@ -28,7 +28,8 @@
 from django.contrib.auth.models import User, Group
 from django.core.management import call_command
 
-from .calculation_unittest import CalculationUnitTest, tests_dir
+from .calculation_unittest import CalculationUnitTest
+from .calcusliveserver import tests_dir
 from .models import *
 from .libxyz import *
 

frontend/test_cluster.py

@@ -53,21 +53,14 @@
 from .environment_variables import *
 
 from django.core.management import call_command
-from .calcusliveserver import CalcusLiveServer
+from .calcusliveserver import CalcusLiveServer, SCR_DIR, RESULTS_DIR, KEYS_DIR
 
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.backends import default_backend
 
 from unittest import mock
 
-dir_path = os.path.dirname(os.path.realpath(__file__))
-
-tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
-KEYS_DIR = os.path.join(tests_dir, "keys")
-
 
 class ClusterTests(CalcusLiveServer):
     @classmethod
@@ -86,8 +79,8 @@ def setUp(self):
         connection.flushdb()
         connection.close()
 
-        p = Process(target=self.run_daemon)
-        p.start()
+        # p = Process(target=self.run_daemon)
+        # p.start()
 
     def tearDown(self):
         time.sleep(0.5)  # Give time to the daemon to disconnect cleanly
@@ -250,7 +243,7 @@ def test_autoselect_resource_remote(self):
         self.setup_cluster()
         params = {
             "mol_name": "test",
-            "type": "Single-Point Energy",
+            "type": "Geometrical Optimisation",
             "project": "New Project",
             "new_project_name": "SeleniumProject",
             "in_file": "ethanol.sdf",
@@ -523,17 +516,18 @@ def test_cancel_calc(self):
         s = self.get_calculation_statuses()
         self.assertEqual(s[0], "Error - Job cancelled")
 
+    @mock.patch.dict(os.environ, {"CAN_USE_CACHED_CALCULATIONS": "false"})
     def test_relaunch_calc(self):
         self.setup_cluster()
         params = {
             "mol_name": "test",
             "type": "Single-Point Energy",
             "project": "New Project",
             "new_project_name": "SeleniumProject",
-            "in_file": "CH4.mol",
+            "in_file": "ethanol.xyz",
             "software": "Gaussian",
             "theory": "HF",
-            "basis_set": "Def2-SVP",
+            "basis_set": "Def2-TZVP",
         }
 
         self.lget("/launch/")

frontend/test_fingerprint.py

@@ -33,8 +33,6 @@
 from .tasks import write_mol, gen_fingerprint
 
 tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
 
 
 class JobTestCase(TestCase):

frontend/test_selenium.py

@@ -42,17 +42,11 @@
 from .models import *
 from .libxyz import *
 from django.core.management import call_command
-from .calcusliveserver import CalcusLiveServer
+from .calcusliveserver import CalcusLiveServer, SCR_DIR, RESULTS_DIR, tests_dir
 
 from selenium.webdriver.support.select import Select
 from unittest import mock
 
-dir_path = os.path.dirname(os.path.realpath(__file__))
-
-tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
-
 
 class InterfaceTests(CalcusLiveServer):
     @classmethod

frontend/test_views.py

@@ -30,12 +30,9 @@
 from django.test import TestCase, Client
 from django.http import HttpRequest
 from .gen_calc import gen_calc, gen_param
-
-dir_path = os.path.dirname(os.path.realpath(__file__))
+from .calcusliveserver import SCR_DIR, RESULTS_DIR
 
 tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
 
 basic_params = {
     "file_structure": [""],

frontend/test_xyz.py

@@ -26,8 +26,6 @@
 from .libxyz import *
 
 tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/")
-SCR_DIR = os.path.join(tests_dir, "scr")
-RESULTS_DIR = os.path.join(tests_dir, "results")
 
 
 class XyzTests(TestCase):