From f811c32560d3bb4ae54bc355745d3ced6ecdddc0 Mon Sep 17 00:00:00 2001 From: gioscarda Date: Thu, 31 Oct 2019 11:01:49 +0100 Subject: [PATCH] [Fixes #30] ProcessBot and ServiceBot tests --- src/wpsremote/mockutils.py | 24 ++ .../xmpp_data/test/logger_test.properties | 28 +++ .../xmpp_data/test/test_remote.config | 4 +- .../xmpp_data/test/test_service.config | 11 +- test/continoustest.bat | 1 + test/test_bot.py | 211 ++++++++++++++++++ test/test_computation_job_inputs.py | 9 +- test/test_computation_job_outputs.py | 3 + test/test_config_parser.py | 21 +- test_suite.sh | 3 + 10 files changed, 297 insertions(+), 18 deletions(-) create mode 100644 src/wpsremote/xmpp_data/test/logger_test.properties create mode 100644 test/test_bot.py diff --git a/src/wpsremote/mockutils.py b/src/wpsremote/mockutils.py index 834dce8..0572ffc 100644 --- a/src/wpsremote/mockutils.py +++ b/src/wpsremote/mockutils.py @@ -4,6 +4,9 @@ # This code is licensed under the GPL 2.0 license, available at the root # application directory. +from wpsagent import WPSAgentService, WPSAgentProcess +from wpsremote.resource_monitor import ResourceMonitor + __author__ = "Alessio Fabiani" __copyright__ = "Copyright 2016 Open Source Geospatial Foundation - all rights reserved" __license__ = "GPL" @@ -24,3 +27,24 @@ def readline(self): else: self._lp += 1 return self._lines[self._lp-1] + + +class MockWPSAgentService(WPSAgentService): + + def run(self): + pass + + +class MockWPSAgentProcess(WPSAgentProcess): + + def run(self): + pass + + +class MockResourceMonitor(ResourceMonitor): + def run(self): + pass + + +def mock_clean_up_all(): + pass diff --git a/src/wpsremote/xmpp_data/test/logger_test.properties b/src/wpsremote/xmpp_data/test/logger_test.properties new file mode 100644 index 0000000..3c281d6 --- /dev/null +++ b/src/wpsremote/xmpp_data/test/logger_test.properties @@ -0,0 +1,28 @@ +# (c) 2016 Open Source Geospatial Foundation - all rights reserved +# (c) 2014 - 2015 Centre for Maritime Research and Experimentation (CMRE) +# (c) 2013 - 2014 German Aerospace Center (DLR) +# This code is licensed under the GPL 2.0 license, available at the root +# application directory. + +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - [%(levelname)s] %(message)s +datefmt= \ No newline at end of file diff --git a/src/wpsremote/xmpp_data/test/test_remote.config b/src/wpsremote/xmpp_data/test/test_remote.config index 7ebd4fa..e1bcf6c 100644 --- a/src/wpsremote/xmpp_data/test/test_remote.config +++ b/src/wpsremote/xmpp_data/test/test_remote.config @@ -21,9 +21,9 @@ password = R3m0T3wP5 mucService = conference.%(domain)s mucServicePassword = geoserver -resource_file_dir = ./xmpp_data/resource_dir +resource_file_dir = ./src/wpsremote/xmpp_data/test/resource_dir -# . Configure this option (along with 'backup_on_wps_execution_shared_dir' +# . Configure this option (along with 'backup_on_wps_execution_shared_dir' # . on single outputs of 'service.config') in order to make a copy # . of the results into a shared folder before sending messages to XMPP # . WARNING: this option takes precedence on "UPLOADER" option diff --git a/src/wpsremote/xmpp_data/test/test_service.config b/src/wpsremote/xmpp_data/test/test_service.config index 160f2f5..34a9b5c 100644 --- a/src/wpsremote/xmpp_data/test/test_service.config +++ b/src/wpsremote/xmpp_data/test/test_service.config @@ -7,11 +7,12 @@ namespace = default description = foo service executable_path = \code\etl executable_cmd = python %(executable_path)s\etl.py -output_dir = \tmp\oaaOnDemand +output_dir = src\wpsremote\xmpp_data\test\tmp unique_execution_id = 123 workdir = %(output_dir)s\%(unique_execution_id)s active = True -max_running_time_seconds = 300 +max_running_time_seconds = 10 +load_average_scan_minutes = 1 servicePassword = admin [Input1] @@ -22,7 +23,7 @@ description = Start date of optmizier list = [item_0, item_1, item_2] int_list = [0, 1, 2, 3] float_list = [0.12, 1.6, 2.55, 3.4] -path_list = [%(workdir)s, \tmp\oaaOnDemand] +path_list = [%(workdir)s, src\wpsremote\xmpp_data\test\tmp] [Action1] class = cmdline @@ -70,7 +71,7 @@ class = createJSONfile input_ref = asset target_filepath = %(workdir)s\asset_${json_path_expr}.json json_path_expr = ['Asset']['id'] -json_schema = .\configs\OAAonDemand\asset_schema.json +json_schema = ./src/wpsremote/xmpp_data/test/asset_schema.json [Const1] @@ -130,7 +131,7 @@ value = %(unique_execution_id)s [Action10] class = updateINIfile input_ref = uniqueId -source_filepath = .\configs\OAAonDemand\geoserverCommands_template.properties +source_filepath = ./src/wpsremote/xmpp_data/test/geoserverCommands_template.properties target_filepath= %(workdir)s\geoserverCommands.properties section = DEFAULT alias = unique_id diff --git a/test/continoustest.bat b/test/continoustest.bat index 1161d73..1be3e41 100644 --- a/test/continoustest.bat +++ b/test/continoustest.bat @@ -30,6 +30,7 @@ python test_process_input_parameters.py python test_computation_job_outputs.py python test_config_parser.py python test_bus_independent_messages.py +python test_bot.py python resource_cleaner.py IF "%1"=="" GOTO exit diff --git a/test/test_bot.py b/test/test_bot.py new file mode 100644 index 0000000..16967a4 --- /dev/null +++ b/test/test_bot.py @@ -0,0 +1,211 @@ +# (c) 2019 Open Source Geospatial Foundation - all rights reserved +# (c) 2014 - 2015 Centre for Maritime Research and Experimentation (CMRE) +# (c) 2013 - 2014 German Aerospace Center (DLR) +# This code is licensed under the GPL 2.0 license, available at the root +# application directory. + +import unittest +import os +from argparse import Namespace +import pickle +import mock +import datetime +import psutil +from wpsremote.processbot import ProcessBot +from wpsremote.servicebot import ServiceBot +from wpsremote.resource_cleaner import Resource +import wpsremote.resource_monitor as resource_monitor +import wpsremote.configInstance as configInstance +from wpsremote.xmppBus import XMPPBus +from wpsremote.computation_job_inputs import ComputationJobInputs +from wpsremote.computational_job_input_actions import ComputationalJobInputActions +from wpsremote.output_parameters import OutputParameters +from wpsremote.path import path +from wpsremote.mockutils import ( + MockWPSAgentService, MockWPSAgentProcess, mock_clean_up_all, MockResourceMonitor +) + +__author__ = "Alessio Fabiani" +__copyright__ = "Copyright 2019 Open Source Geospatial Foundation - all rights reserved" +__license__ = "GPL" + +# due to xmppMessages chdir +os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + + +class TestBot(unittest.TestCase): + + def setUp(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + self.remote_config = configInstance.create( + "./src/wpsremote/xmpp_data/test/test_remote.config" + ) + self.xmpp_bus = XMPPBus(self.remote_config, "service_name", "service_name_namespace") + self.xep_0045_plugin = self.xmpp_bus.xmpp.plugin["xep_0045"] + self.from_obj_mock = mock.Mock() + self.from_obj_mock.__class__ = mock.Mock + rooms = { + self.xmpp_bus._get_MUC_JId(): { + "resource": { + "role": "not_visitor" + } + } + } + self.xep_0045_plugin.rooms = rooms + self.from_obj_mock.resource = "resource" + test_msg = pickle.dumps({ + "test_message": "test message" + }) + msg = { + "type": "normal", + "body": "topic=request&id=123&baseURL=test_base_url&message={}".format(test_msg), + "from": self.from_obj_mock + } + # ExecuteMessage + self.execute_message = self.xmpp_bus.CreateIndipendentMessage(msg) + # cleaner + Resource.clean_up_all = staticmethod(mock_clean_up_all) + self.cleaner = Resource() + # monitor + resource_monitor.ResourceMonitor = MockResourceMonitor + # wps agent args + self.params_file = "./src/wpsremote/xmpp_data/test/test_params.p" + pickle.dump( + self.execute_message, + open(self.params_file, "wb") + ) + self.args = Namespace( + executionid=None, + logconf='./src/wpsremote/xmpp_data/test/logger_test.properties', + params=self.params_file, + remoteconfig='./src/wpsremote/xmpp_data/test/test_remote.config', + runtype='service', + serviceconfig='./src/wpsremote/xmpp_data/test/test_service.config' + ) + + def test_process_bot(self): + wps_agent = MockWPSAgentProcess(self.args) + # test bot + process_bot = wps_agent.create_bot() + self.assertIsInstance(process_bot, ProcessBot) + self.assertTrue(process_bot._active) + self.assertFalse(process_bot._finished) + self.assertIsInstance(process_bot._input_parameters_defs, ComputationJobInputs) + self.assertIsInstance(process_bot._input_params_actions, ComputationalJobInputActions) + self.assertEqual(process_bot._input_values, {"test_message": "test message"}) + self.assertEqual(process_bot._max_running_time, datetime.timedelta(0, 10)) + self.assertEqual(process_bot.max_execution_time(), datetime.timedelta(0, 10)) + self.assertIsInstance(process_bot._output_parameters_defs, OutputParameters) + self.assertEqual(process_bot._remote_wps_baseurl, "test_base_url") + self.assertEqual( + process_bot._resource_file_dir, + path("./src/wpsremote/xmpp_data/test/resource_dir") + ) + self.assertEqual( + process_bot.get_resource_file_dir(), + path("./src/wpsremote/xmpp_data/test/resource_dir") + ) + self.assertEqual(len(process_bot._stdout_action), 6) + for stdout_action in process_bot._stdout_action: + self.assertIn(stdout_action, ['ignore', 'progress', 'log', 'log', 'log', 'abort']) + self.assertEqual(len(process_bot._stdout_parser), 6) + for stdout_parser in process_bot._stdout_parser: + self.assertIn(stdout_parser, [ + '.*\\[DEBUG\\](.*)', + '.*\\[INFO\\] ProgressInfo\\:([-+]?[0-9]*\\.?[0-9]*)\\%', + '.*\\[(INFO)\\](.*)', + '.*\\[(WARN)\\](.*)', + '.*\\[(ERROR)\\](.*)', + '.*\\[(CRITICAL)\\](.*)' + ]) + self.assertEqual(process_bot._uniqueExeId, "123") + self.assertEqual(process_bot._uploader, None) + self.assertEqual(process_bot._wps_execution_shared_dir, None) + self.assertEqual(process_bot.get_wps_execution_shared_dir(), None) + self.assertEqual(process_bot.description, "foo service") + self.assertEqual(process_bot.namespace, "default") + self.assertEqual(process_bot.service, "Service") + self.assertEqual( + process_bot.workdir(), + process_bot._output_dir + "/" + process_bot._uniqueExeId + ) + # handle_finish + finish_msg = { + "type": "normal", + "body": "topic=finish", + "from": self.from_obj_mock + } + finished_message = self.xmpp_bus.CreateIndipendentMessage(finish_msg) + with self.assertRaises(SystemExit) as cm_finish: + process_bot.handle_finish(finished_message) + self.assertEqual(cm_finish.exception.code, 0) + # handle_abort + abort_msg = { + "type": "normal", + "body": "topic=abort", + "from": self.from_obj_mock + } + abort_message = self.xmpp_bus.CreateIndipendentMessage(abort_msg) + with self.assertRaises(SystemExit) as cm_abort: + process_bot.handle_abort(abort_message) + self.assertEqual(cm_abort.exception.code, -1) + with self.assertRaises(SystemExit): + process_bot.exit(0) + # kill process + self.cleaner.kill_processbot() + self.assertFalse(psutil.pid_exists(self.cleaner._processbot_pid)) + + def test_service_bot(self): + wps_agent = MockWPSAgentService(self.args) + # test bot + service_bot = wps_agent.create_bot() + self.assertIsInstance(service_bot, ServiceBot) + self.assertTrue(service_bot._active) + self.assertIsInstance(service_bot._input_parameters_defs, ComputationJobInputs) + self.assertEqual(service_bot._max_running_time, datetime.timedelta(0, 10)) + self.assertEqual( + service_bot._output_dir, + path('src\\wpsremote\\xmpp_data\\test\\tmp') + ) + self.assertIsInstance(service_bot._output_parameters_defs, OutputParameters) + self.assertFalse(service_bot._process_blacklist) + self.assertTrue(service_bot._redirect_process_stdout_to_logger) + self.assertEqual( + service_bot._remote_config_filepath, + './src/wpsremote/xmpp_data/test/test_remote.config' + ) + self.assertIsNone(service_bot._remote_wps_endpoint) + self.assertEqual( + service_bot._resource_file_dir, + path('./src/wpsremote/xmpp_data/test/resource_dir') + ) + self.assertEqual( + service_bot.get_resource_file_dir(), + path("./src/wpsremote/xmpp_data/test/resource_dir") + ) + self.assertIsInstance(service_bot._resource_monitor, MockResourceMonitor) + self.assertEqual( + service_bot._service_config_file, + './src/wpsremote/xmpp_data/test/test_service.config' + ) + self.assertIsNone(service_bot._wps_execution_shared_dir) + self.assertEqual( + service_bot.max_execution_time(), + datetime.timedelta(0, 10) + ) + self.assertEqual( + service_bot.get_wps_execution_shared_dir(), + None + ) + self.assertIsNotNone(service_bot.bus) + self.assertEqual(service_bot.description, 'foo service') + self.assertEqual(service_bot.namespace, 'default') + self.assertEqual(service_bot.running_process, {}) + self.assertEqual(service_bot.service, 'Service') + # removing tmp files + params_file_path = path(self.params_file) + params_file_path.remove() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_computation_job_inputs.py b/test/test_computation_job_inputs.py index bfd6969..e6962de 100644 --- a/test/test_computation_job_inputs.py +++ b/test/test_computation_job_inputs.py @@ -7,7 +7,7 @@ import unittest import json import wpsremote.ConfigParser as ConfigParser - +import os from wpsremote import path from wpsremote import computation_job_param from wpsremote import computation_job_inputs @@ -124,6 +124,7 @@ def test_cmd_line_action_2_values(self): self.assertEquals(actions.get_cmd_line(), "--mypar1=1 --mypar2=abc") def test_create_json_file_action(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", "par descr 1") inputs = computation_job_inputs.ComputationJobInputs() inputs.add_input(p1) @@ -142,6 +143,7 @@ def test_create_json_file_action(self): self.assertTrue(a1.exists()) def test_create_2_json_file_action(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) p1 = computation_job_param.ComputationJobParam("mypar1", "application/json", "par 1", @@ -164,6 +166,7 @@ def test_create_2_json_file_action(self): self.assertTrue(a1.exists()) def test_update_json_file_action_with_int(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -187,6 +190,7 @@ def test_update_json_file_action_with_int(self): self.assertTrue(100, j['Config']['nEvaluations']) def test_update_json_file_action_with_string(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -432,6 +436,7 @@ def test_cmdpar_order_from_dict(self): self.assertEquals(actions.get_cmd_line(), '-w . --k=2.4') def test_update_json_list_in_file(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -452,6 +457,7 @@ def test_update_json_list_in_file(self): self.assertTrue(45.5, j['Config']['latLim'][1]) def test_update_json_list_in_file_two_times(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./json_out.json") if target_json_file.exists(): target_json_file.remove() @@ -478,6 +484,7 @@ def test_update_json_list_in_file_two_times(self): self.assertTrue(76.5, j['Config']['latLim'][1]) def test_copyfile(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) target_json_file = path.path("./copy_of_source_json.json") if target_json_file.exists(): target_json_file.remove() diff --git a/test/test_computation_job_outputs.py b/test/test_computation_job_outputs.py index af6c5f2..481be92 100644 --- a/test/test_computation_job_outputs.py +++ b/test/test_computation_job_outputs.py @@ -5,6 +5,7 @@ # application directory. import unittest +import os import wpsremote.ConfigParser as ConfigParser from wpsremote import mockutils from wpsremote.output_file_parameter import ( @@ -128,6 +129,7 @@ class TestComputationJobOutputs(unittest.TestCase): ] def test_string_output_type_from_ini(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) ini_text = '''[Output1] name = result1 type = string @@ -355,6 +357,7 @@ def test_application_owc_output_type_from_ini(self): self.assertEqual('', owcfp.get_value()) def test_string_output_type(self): + os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) ofp = OutputFileParameter( "result1", { diff --git a/test/test_config_parser.py b/test/test_config_parser.py index bf15c19..0bf24ed 100644 --- a/test/test_config_parser.py +++ b/test/test_config_parser.py @@ -20,9 +20,9 @@ class TestConfigParser(unittest.TestCase): DEFAULT_ITEMS = [ ('service', 'Service'), ('namespace', 'default'), ('description', 'foo service'), ('executable_path', '\\code\\etl'), ('executable_cmd', 'python %(executable_path)s\\etl.py'), - ('output_dir', '\\tmp\\oaaOnDemand'), ('unique_execution_id', '123'), + ('output_dir', 'src\\wpsremote\\xmpp_data\\test\\tmp'), ('unique_execution_id', '123'), ('workdir', '%(output_dir)s\\%(unique_execution_id)s'), ('active', 'True'), - ('max_running_time_seconds', '300'), ('servicepassword', 'admin') + ('max_running_time_seconds', '10'), ('servicepassword', 'admin'), ('load_average_scan_minutes', '1') ] CONFIG_SECTIONS = [ 'Input1', 'Action1', 'Input2', 'Action2', 'Input3', 'Action3', 'Input4', 'Action4', 'Const1', 'Action5', @@ -64,7 +64,7 @@ def test_options(self): options = [ 'class', 'input_ref', 'alias', 'template', 'service', 'namespace', 'description', 'executable_path', 'executable_cmd', 'output_dir', 'unique_execution_id', 'workdir', - 'active', 'max_running_time_seconds', 'servicepassword' + 'active', 'max_running_time_seconds', 'servicepassword', 'load_average_scan_minutes' ] cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") opts = cp.options("Action1") @@ -95,12 +95,13 @@ def test_items(self): items_list = [ ('service', 'Service'), ('namespace', 'default'), ('description', 'foo service'), ('executable_path', '\\code\\etl'), ('executable_cmd', 'python \\code\\etl\\etl.py'), - ('output_dir', '\\tmp\\oaaOnDemand'), ('unique_execution_id', '123'), - ('workdir', '\\tmp\\oaaOnDemand\\123'), ('active', 'True'), ('max_running_time_seconds', '300'), - ('servicepassword', 'admin'), ('class', 'updateJSONfile'), ('input_ref', 'timeHorizon'), + ('output_dir', 'src\\wpsremote\\xmpp_data\\test\\tmp'), ('unique_execution_id', '123'), + ('workdir', 'src\\wpsremote\\xmpp_data\\test\\tmp\\123'), ('active', 'True'), + ('max_running_time_seconds', '10'), ('servicepassword', 'admin'), ('class', 'updateJSONfile'), + ('input_ref', 'timeHorizon'), ('source_filepath', '.\\configs\\OAAonDemand\\CMREOAA_MainConfigFile_template.json'), - ('target_filepath', '\\tmp\\oaaOnDemand\\123\\config.json'), - ('json_path_expr', "['Config']['timeHorizon']") + ('target_filepath', 'src\\wpsremote\\xmpp_data\\test\\tmp\\123\\config.json'), + ('json_path_expr', "['Config']['timeHorizon']"), ('load_average_scan_minutes', '1') ] cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") for i in cp.items("Action3"): @@ -171,7 +172,7 @@ def test_get_list_float_impl(self): self.assertIn(i, float_list) def test_get_list_path_impl(self): - path_list = ['\\tmp\\oaaOnDemand\\123', '\\tmp\\oaaOnDemand'] + path_list = ['src\\wpsremote\\xmpp_data\\test\\tmp\\123', 'src\\wpsremote\\xmpp_data\\test\\tmp'] cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") items = configInstance.get_list_path_impl(cp, "Input1", "path_list") for i in items: @@ -187,7 +188,7 @@ def test_get_path(self): cp = configInstance.create("./src/wpsremote/xmpp_data/test/test_service.config") item = configInstance.get_path(cp, "DEFAULT", "output_dir") self.assertIsInstance(item, path.path) - self.assertEqual("\\tmp\\oaaOnDemand", item) + self.assertEqual("src\\wpsremote\\xmpp_data\\test\\tmp", item) if __name__ == '__main__': diff --git a/test_suite.sh b/test_suite.sh index 5a8213d..c154cf6 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -16,3 +16,6 @@ coverage run --source=src test/test_config_parser.py echo "Running... test_bus_independent_messages" coverage run --source=src test/test_bus_independent_messages.py +echo "Running... test_bot" +coverage run --source=src test/test_bot.py +