From 2acce402a845c42e40d889935a3e1a92c0738a9b Mon Sep 17 00:00:00 2001 From: Calvin Date: Sun, 18 Dec 2022 12:10:32 +0200 Subject: [PATCH 1/3] Updated rmg_runner.py and test_rmg_runner.py rmg_runner.py: File has been updated with assertion checks to ensure the user is correctly choosing memory if they specifiy test_rmg_runner.py: Test has descriptions and also checks for assertion errors when the memory is incorrectly specified. --- t3/runners/rmg_runner.py | 3 +++ tests/test_rmg_runner.py | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/t3/runners/rmg_runner.py b/t3/runners/rmg_runner.py index df4ff582..33de2485 100644 --- a/t3/runners/rmg_runner.py +++ b/t3/runners/rmg_runner.py @@ -49,6 +49,9 @@ def write_submit_script(project_directory: str, max_iterations (str, optional): Max RMG iterations, e.g., ``-m 100``. t3_project_name (str, optional): THe T3 project name, used for setting a job name on the server for the RMG run. """ + if memory is not None: + assert 8000<= memory <=32000, "Memory is in MB and must be between 8000 and 32000" + assert memory % 1000 == 0, "Memory is in MB and must be divisable by 1000" global MEM submit_scripts_content = submit_scripts['rmg'].format(name=f'{t3_project_name}_RMG' or 'T3_RMG', cpus=cpus or CPUS, diff --git a/tests/test_rmg_runner.py b/tests/test_rmg_runner.py index 7efeeb11..9c21b34b 100644 --- a/tests/test_rmg_runner.py +++ b/tests/test_rmg_runner.py @@ -8,12 +8,12 @@ import os from t3.common import DATA_BASE_PATH, EXAMPLES_BASE_PATH from t3.runners.rmg_runner import write_submit_script - +import pytest class TestWriteSubmitScript(object): - #Need to think of multiple cases... + def test_minimial_write_submit_script(self): """Test the write_submit_script() function with minimal input. @@ -52,6 +52,11 @@ def test_minimial_write_submit_script(self): os.remove(os.path.join(project_directory_path,"job.sh")) def test_minimal_project_name_included(self): + """ + This test has been constructed to ensure that when the user provides a t3 project name, it will be displayed in the submit name whilst also + not affecting the job file. + """ + project_directory_path = os.path.join(EXAMPLES_BASE_PATH, "minimal") t3_proj_name = "T3_test_name" actual = write_submit_script(project_directory_path, @@ -110,6 +115,9 @@ def test_minimal_project_name_included(self): def test_minimal_parameters_set(self): + """ + This test has been built to ensure that chosen parameters by the user are being correctly written into the job and submit file + """ project_directory_path = os.path.join(EXAMPLES_BASE_PATH, "minimal") ####To be edited by user if required#### @@ -170,4 +178,29 @@ def test_minimal_parameters_set(self): assert content_submit == expected_submit os.remove(os.path.join(project_directory_path,"job.sh")) - os.remove(os.path.join(project_directory_path,"submit.sub")) \ No newline at end of file + os.remove(os.path.join(project_directory_path,"submit.sub")) + + def test_minimal_incorrect_mem(self): + """ + This test has been constructed to raise an assertion error due to the user specifiying the incorrent memory. + Memory selection for assertion error can either be less than 8000 or higher than 32000 (Subject to change) OR/AND + the selection of Memory may not be divisable by 1000 (Subject to change) + """ + project_directory_path = os.path.join(EXAMPLES_BASE_PATH, "minimal") + + ####To be edited by user if required#### + t3_proj_name = "T3_test_name" + cpus = 8 + max_iter = "-m 100" + mem = 16230 #in MB - This is to be incorrect + ######################################### + + with pytest.raises(AssertionError) as assertion_error: + actual = write_submit_script(project_directory_path, + cpus=cpus, + memory=mem, #in MB + verbose="-v 20", + max_iterations=max_iter, + t3_project_name= t3_proj_name) + + assert 'Memory is in MB and must be between 8000 and 32000' or "Memory is in MB and must be divisable by 1000" in str(assertion_error.value) \ No newline at end of file From a80cc698d232bc87a24632a12f7d464b5d87485e Mon Sep 17 00:00:00 2001 From: Calvin Date: Mon, 19 Dec 2022 13:52:38 +0200 Subject: [PATCH 2/3] Updated test_rmg_runner.py test_rmg_runner.py: Updated the class TestWriteSubmitScript with a function that runs executes t3 as a 'local' job, and then after 'converging' the function will ensure that the job.sh and submit.sub files are written as expected. --- t3/runners/rmg_runner.py | 6 +-- tests/test_rmg_runner.py | 80 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/t3/runners/rmg_runner.py b/t3/runners/rmg_runner.py index 33de2485..4cf5a556 100644 --- a/t3/runners/rmg_runner.py +++ b/t3/runners/rmg_runner.py @@ -49,9 +49,9 @@ def write_submit_script(project_directory: str, max_iterations (str, optional): Max RMG iterations, e.g., ``-m 100``. t3_project_name (str, optional): THe T3 project name, used for setting a job name on the server for the RMG run. """ - if memory is not None: - assert 8000<= memory <=32000, "Memory is in MB and must be between 8000 and 32000" - assert memory % 1000 == 0, "Memory is in MB and must be divisable by 1000" + if memory is not None and memory <= 1000: + raise ValueError("Memory is in MB and it is strongly recommended to set the memory above 1000") + global MEM submit_scripts_content = submit_scripts['rmg'].format(name=f'{t3_project_name}_RMG' or 'T3_RMG', cpus=cpus or CPUS, diff --git a/tests/test_rmg_runner.py b/tests/test_rmg_runner.py index 9c21b34b..729f65af 100644 --- a/tests/test_rmg_runner.py +++ b/tests/test_rmg_runner.py @@ -9,7 +9,8 @@ from t3.common import DATA_BASE_PATH, EXAMPLES_BASE_PATH from t3.runners.rmg_runner import write_submit_script import pytest - +from tests.common import run_minimal +import shutil class TestWriteSubmitScript(object): @@ -192,15 +193,84 @@ def test_minimal_incorrect_mem(self): t3_proj_name = "T3_test_name" cpus = 8 max_iter = "-m 100" - mem = 16230 #in MB - This is to be incorrect + mem = 850 #in MB - This is to be incorrect ######################################### - with pytest.raises(AssertionError) as assertion_error: - actual = write_submit_script(project_directory_path, + with pytest.raises(ValueError) as value_error: + write_submit_script(project_directory_path, cpus=cpus, memory=mem, #in MB verbose="-v 20", max_iterations=max_iter, t3_project_name= t3_proj_name) - assert 'Memory is in MB and must be between 8000 and 32000' or "Memory is in MB and must be divisable by 1000" in str(assertion_error.value) \ No newline at end of file + assert "Memory is in MB and it is strongly recommended to set the memory above 1000" in str(value_error.value) + + os.remove(os.path.join(project_directory_path,"job.sh")) + os.remove(os.path.join(project_directory_path,"submit.sub")) + + def test_run_rmg_into_write_submit_script(self): + + + + t3 = run_minimal(iteration=1, set_paths=True) + t3.rmg['rmg_execution_type'] = 'local' + + rmg_base_path = os.path.join(t3.project_directory, 'iteration_1','RMG') + + t3.schema['t3']['options']['max_RMG_exceptions_allowed']= 0 + os.makedirs(rmg_base_path) + with open(os.path.join(rmg_base_path,'RMG.log'), 'w') as rmg_log: + rmg_log.write("""MODEL GENERATION COMPLETED + """) + rmg_log.close() + + + t3.run_rmg() + + + expected_bash = """#!/bin/bash -l + +touch initial_time + +source /srv01/technion/$USER/.bashrc + +conda activate rmg_env + +python-jl /Local/ce_dana/Code/RMG-Py/rmg.py -n 10 input.py + +touch final_time + +""" + expected_submit = """Universe = vanilla + ++JobName = "T3_minimal_example_RMG" + +log = job.log +output = out.txt +error = err.txt + +getenv = True + +should_transfer_files = no + +executable = job.sh + +request_cpus = 10 +request_memory = 25000MB + +queue + +""" + + assert os.path.isfile(os.path.join(rmg_base_path,"job.sh")) == True + assert os.path.isfile(os.path.join(rmg_base_path, "submit.sub")) == True + with open(os.path.join(rmg_base_path,"job.sh"),"r") as bash_file: + content_bash = bash_file.read() + assert content_bash == expected_bash + + with open(os.path.join(rmg_base_path,"submit.sub"),"r") as submit_file: + content_submit = submit_file.read() + assert content_submit == expected_submit + + shutil.rmtree(os.path.join(t3.project_directory)) \ No newline at end of file From c3d43b46685af5c67584593db10b8d8e72f8fe71 Mon Sep 17 00:00:00 2001 From: Calvin Date: Mon, 19 Dec 2022 19:26:50 +0200 Subject: [PATCH 3/3] Fixed a os.remove error os.remove error that there was no job.sh to remove. Therefore, removed the function. --- tests/test_rmg_runner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_rmg_runner.py b/tests/test_rmg_runner.py index 729f65af..2a9cfd39 100644 --- a/tests/test_rmg_runner.py +++ b/tests/test_rmg_runner.py @@ -206,8 +206,7 @@ def test_minimal_incorrect_mem(self): assert "Memory is in MB and it is strongly recommended to set the memory above 1000" in str(value_error.value) - os.remove(os.path.join(project_directory_path,"job.sh")) - os.remove(os.path.join(project_directory_path,"submit.sub")) + def test_run_rmg_into_write_submit_script(self):