From c8e1666420a6ecba888f530aa2b6136c99130eb4 Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Mon, 2 Dec 2019 12:59:48 +0100 Subject: [PATCH 1/2] dird: fix RunScript parsing This commit fixes a bug introduced during the recent months. Without this patch, all RunScripts are configured to "Runs On Client = Yes". Because of this, Admin and Console RunScripts did not work and are silently ignored. With this change, RunScripts can by set to run locally or on the Client (Runs On Client = Yes/No). --- core/src/dird/dird_conf.cc | 13 +++++++++---- core/src/lib/runscript.cc | 10 ++++++---- core/src/lib/runscript.h | 7 ++++--- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/core/src/dird/dird_conf.cc b/core/src/dird/dird_conf.cc index f78206bc97b..4ddbe1b22f8 100644 --- a/core/src/dird/dird_conf.cc +++ b/core/src/dird/dird_conf.cc @@ -591,7 +591,7 @@ static ResourceItem runscript_items[] = { { "FailJobOnError", CFG_TYPE_RUNSCRIPT_BOOL, ITEM(res_runscript,fail_on_error), 0, 0, NULL, NULL, NULL }, { "AbortJobOnError", CFG_TYPE_RUNSCRIPT_BOOL, ITEM(res_runscript,fail_on_error), 0, 0, NULL, NULL, NULL }, { "RunsWhen", CFG_TYPE_RUNSCRIPT_WHEN, ITEM(res_runscript,when), 0, 0, NULL, NULL, NULL }, - { "RunsOnClient", CFG_TYPE_RUNSCRIPT_TARGET, ITEMC(res_runscript), 0, 0, NULL, NULL, NULL }, /* TODO */ + { "RunsOnClient", CFG_TYPE_RUNSCRIPT_TARGET, ITEMC(res_runscript), 0, 0, NULL, NULL, NULL }, {nullptr, 0, 0, nullptr, 0, 0, nullptr, nullptr, nullptr} }; @@ -3220,6 +3220,14 @@ static void StoreRunscript(LEX* lc, ResourceItem* item, int index, int pass) res_runscript = new RunScript(); + /* + * Run on client by default. + * Set this here, instead of in the class constructor, + * as the class is also used by other daemon, + * where the default differs. + */ + if (res_runscript->target.empty()) { res_runscript->SetTarget("%c"); } + while ((token = LexGetToken(lc, BCT_SKIP_EOL)) != BCT_EOF) { if (token == BCT_EOB) { break; } @@ -3261,9 +3269,6 @@ static void StoreRunscript(LEX* lc, ResourceItem* item, int index, int pass) } if (pass == 2) { - // Run on client by default - if (res_runscript->target.empty()) { res_runscript->SetTarget("%c"); } - alist** runscripts = GetItemVariablePointer(*item); if (!*runscripts) { *runscripts = new alist(10, not_owned_by_alist); } diff --git a/core/src/lib/runscript.cc b/core/src/lib/runscript.cc index aecb3a6b594..3520a1993f1 100644 --- a/core/src/lib/runscript.cc +++ b/core/src/lib/runscript.cc @@ -2,6 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2006-2011 Free Software Foundation Europe e.V. + Copyright (C) 2019-2019 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -130,8 +131,11 @@ int RunScripts(JobControlRecord* jcr, } foreach_alist (script, runscripts) { - Dmsg2(200, "runscript: try to run %s:%s\n", NSTDPRNT(script->target), - NSTDPRNT(script->command)); + Dmsg5(200, + "runscript: try to run (Target=%s, OnSuccess=%i, OnFailure=%i, " + "CurrentJobStatus=%c, command=%s)\n", + NSTDPRNT(script->target), script->on_success, script->on_failure, + jcr->JobStatus, NSTDPRNT(script->command)); runit = false; if (!script->IsLocal()) { @@ -216,8 +220,6 @@ void RunScript::SetTarget(const std::string& client_name) { Dmsg1(500, "runscript: setting target = %s\n", NSTDPRNT(client_name)); - if (client_name.empty()) { return; } - target = client_name; } diff --git a/core/src/lib/runscript.h b/core/src/lib/runscript.h index a017b3908cf..bd139b5f886 100644 --- a/core/src/lib/runscript.h +++ b/core/src/lib/runscript.h @@ -37,7 +37,6 @@ class alist; /* Usage: * - * #define USE_RUNSCRIPT * #include "lib/runscript.h" * * RunScript *script = new RunScript; @@ -45,7 +44,7 @@ class alist; * script->on_failure = true; * script->when = SCRIPT_After; * - * script->run("LabelBefore"); // the label must contain "Before" or "After" + * script->Run("LabelBefore"); // the label must contain "Before" or "After" * special keyword FreeRunscript(script); */ @@ -81,7 +80,9 @@ class RunScript : public BareosResource { RunScript(const RunScript& other) = default; std::string command; /* Command string */ - std::string target; /* Host target */ + std::string target; /* Host target. Values: + Empty string: run locally. + "%c": (Client’s name). Run on client. */ int when = SCRIPT_Never; /* SCRIPT_Before|Script_After BEFORE/AFTER JOB*/ int cmd_type = 0; /* Command type -- Shell, Console */ char level = 0; /* Base|Full|Incr...|All (NYI) */ From 720b0a3d2fc785adc3fef493ca0f6c09dab13317 Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Sun, 1 Dec 2019 17:07:03 +0100 Subject: [PATCH 2/2] systemtests (python-bareos-test): adding test for job runscripts Add test class PythonBareosJsonRunScriptTest to check if RunScripts are executed in the correct context. --- .../job/admin-runscript-client-server.conf.in | 27 +++ .../job/admin-runscript-client.conf.in | 16 ++ .../job/admin-runscript-server.conf.in | 16 ++ ...areos-fd-runscript-client-defaults.conf.in | 16 ++ .../backup-bareos-fd-runscript-client.conf.in | 16 ++ .../backup-bareos-fd-runscript-server.conf.in | 16 ++ .../jobdefs/jobdefs-runscript1.conf.in | 17 ++ .../jobdefs/jobdefs-runscript2.conf.in | 17 ++ .../python-bareos-unittest.py | 185 +++++++++++++++++- systemtests/tests/python-bareos-test/write.sh | 9 + 10 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client-server.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-server.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client-defaults.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-server.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript1.conf.in create mode 100644 systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript2.conf.in create mode 100755 systemtests/tests/python-bareos-test/write.sh diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client-server.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client-server.conf.in new file mode 100644 index 00000000000..4931cd4cce0 --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client-server.conf.in @@ -0,0 +1,27 @@ +Job { + Name = "admin-runscript-client-server" + JobDefs = "DefaultJob" + Type = Admin + RunScript { + RunsWhen = Before + Runs On Failure = Yes + #Runs On Client = Yes + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } + RunScript { + RunsWhen = Before + Runs On Failure = Yes + Runs On Client = No + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client.conf.in new file mode 100644 index 00000000000..979c0ab573d --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-client.conf.in @@ -0,0 +1,16 @@ +Job { + Name = "admin-runscript-client" + JobDefs = "DefaultJob" + Type = Admin + RunScript { + RunsWhen = Before + Runs On Failure = Yes + #Runs On Client = Yes + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-server.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-server.conf.in new file mode 100644 index 00000000000..fc911866989 --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/admin-runscript-server.conf.in @@ -0,0 +1,16 @@ +Job { + Name = "admin-runscript-server" + JobDefs = "DefaultJob" + Type = Admin + RunScript { + RunsWhen = Before + Runs On Failure = Yes + Runs On Client = No + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client-defaults.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client-defaults.conf.in new file mode 100644 index 00000000000..b949da66520 --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client-defaults.conf.in @@ -0,0 +1,16 @@ +Job { + Name = "backup-bareos-fd-runscript-client-defaults" + JobDefs = "DefaultJob" + Type = Backup + RunScript { + RunsWhen = Before + #Runs On Failure = Yes + #Runs On Client = Yes + #FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client.conf.in new file mode 100644 index 00000000000..32a3376996e --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-client.conf.in @@ -0,0 +1,16 @@ +Job { + Name = "backup-bareos-fd-runscript-client" + JobDefs = "DefaultJob" + Type = Backup + RunScript { + RunsWhen = Before + Runs On Failure = Yes + Runs On Client = Yes + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-server.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-server.conf.in new file mode 100644 index 00000000000..dcc847f0212 --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd-runscript-server.conf.in @@ -0,0 +1,16 @@ +Job { + Name = "backup-bareos-fd-runscript-server" + JobDefs = "DefaultJob" + Type = Backup + RunScript { + RunsWhen = Before + Runs On Failure = Yes + Runs On Client = No + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } +} diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript1.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript1.conf.in new file mode 100644 index 00000000000..e5e9b97bffc --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript1.conf.in @@ -0,0 +1,17 @@ +JobDefs { + Name = "jobdefs-runscript1" + Type = Admin + Messages = Standard + RunScript { + RunsWhen = Before + Runs On Client = No + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'jobdefs=jobdefs-runscript1' 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } + Jobdefs = "DefaultJob" +} + diff --git a/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript2.conf.in b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript2.conf.in new file mode 100644 index 00000000000..b0ae3c2f685 --- /dev/null +++ b/systemtests/tests/python-bareos-test/etc/bareos/bareos-dir.d/jobdefs/jobdefs-runscript2.conf.in @@ -0,0 +1,17 @@ +JobDefs { + Name = "jobdefs-runscript2" + Type = Admin + Messages = Standard + RunScript { + RunsWhen = Before + Runs On Client = No + FailJobOnError = Yes + # %d Daemon’s name (Such as host-dir or host-fd) + # %n Job name + # %t Job type (Backup, …) + # %i Job Id + Command = "@PROJECT_BINARY_DIR@/tests/@TEST_NAME@/write.sh @working_dir@/jobid-%i-runscript.log 'jobdefs=jobdefs-runscript2' 'daemon=%d' 'jobname=%n' 'jobtype=%t' 'jobid=%i'" + } + JobDefs = "jobdefs-runscript1" +} + diff --git a/systemtests/tests/python-bareos-test/python-bareos-unittest.py b/systemtests/tests/python-bareos-test/python-bareos-unittest.py index a28786a9f7a..88025ba457b 100755 --- a/systemtests/tests/python-bareos-test/python-bareos-unittest.py +++ b/systemtests/tests/python-bareos-test/python-bareos-unittest.py @@ -564,7 +564,7 @@ def configure_add(self, director, resourcesname, resourcename, cmd): u'Failed to find resource {} in {}.'.format( resourcename, resourcesname)) - def run_job(self, director, jobname, level, wait=False): + def run_job(self, director, jobname, level=None, wait=False): logger = logging.getLogger() run_parameter = ['job={}'.format(jobname), 'yes'] if level: @@ -640,6 +640,31 @@ def _test_list_with_invalid_jobid(self, director, jobid): u'Command {} should not return results. Current result: {} visible'. format(listcmd, str(result))) + def search_joblog(self, director, jobId, patterns): + + if isinstance(patterns, list): + pattern_dict = { i: False for i in patterns } + else: + pattern_dict = { patterns: False } + + result = director.call('list joblog jobid={}'.format(jobId)) + joblog_list = result['joblog'] + + found = False + for pattern in pattern_dict: + for logentry in joblog_list: + if re.search(pattern, logentry['logtext']): + pattern_dict[pattern] = True + self.assertTrue(pattern_dict[pattern], 'Failed to find pattern "{}" in Job Log of Job {}.'.format(pattern, jobId)) + + + def run_job_and_search_joblog(self, director, jobname, level, patterns): + + jobId = self.run_job(director, jobname, level, wait=True) + self.search_joblog(director, jobId, patterns) + return jobId + + class PythonBareosJsonBackendTest(PythonBareosJsonBase): def test_json_backend(self): @@ -1073,6 +1098,164 @@ def test_json_list_jobid_with_job_acl(self): self._test_list_with_invalid_jobid(director, jobid2) +class PythonBareosJsonRunScriptTest(PythonBareosJsonBase): + + def test_backup_runscript_client_defaults(self): + ''' + Run a job which contains a runscript. + Check the JobLog if the runscript worked as expected. + ''' + logger = logging.getLogger() + + username = self.get_operator_username() + password = self.get_operator_password(username) + + jobname='backup-bareos-fd-runscript-client-defaults' + level = None + expected_log = ': ClientBeforeJob: jobname={}'.format(jobname) + + director_root = bareos.bsock.DirectorConsoleJson( + address=self.director_address, + port=self.director_port, + name=username, + password=password) + + # Example log entry: + #{ + # "time": "2019-12-02 00:07:34", + # "logtext": "bareos-dir JobId 76: BeforeJob: jobname=admin-runscript-server\n" + #}, + + jobId = self.run_job_and_search_joblog(director_root, jobname, level, expected_log) + + + def test_backup_runscript_client(self): + ''' + Run a job which contains a runscript. + Check the JobLog if the runscript worked as expected. + ''' + logger = logging.getLogger() + + username = self.get_operator_username() + password = self.get_operator_password(username) + + jobname='backup-bareos-fd-runscript-client' + level = None + expected_log = ': ClientBeforeJob: jobname={}'.format(jobname) + + director_root = bareos.bsock.DirectorConsoleJson( + address=self.director_address, + port=self.director_port, + name=username, + password=password) + + # Example log entry: + #{ + # "time": "2019-12-02 00:07:34", + # "logtext": "bareos-dir JobId 76: ClientBeforeJob: jobname=admin-runscript-server\n" + #}, + + jobId = self.run_job_and_search_joblog(director_root, jobname, level, expected_log) + + + def test_backup_runscript_server(self): + ''' + Run a job which contains a runscript. + Check the JobLog if the runscript worked as expected. + ''' + logger = logging.getLogger() + + username = self.get_operator_username() + password = self.get_operator_password(username) + + jobname='backup-bareos-fd-runscript-server' + level = None + expected_logs = [ + ': BeforeJob: jobname={}'.format(jobname), + ': BeforeJob: daemon=bareos-dir' + ] + + director_root = bareos.bsock.DirectorConsoleJson( + address=self.director_address, + port=self.director_port, + name=username, + password=password) + + # Example log entry: + #{ + # "time": "2019-12-02 00:07:34", + # "logtext": "bareos-dir JobId 76: BeforeJob: jobname=admin-runscript-server\n" + #}, + + jobId = self.run_job_and_search_joblog(director_root, jobname, level, expected_logs) + + + def test_admin_runscript_server(self): + ''' + Run a job which contains a runscript. + Check the JobLog if the runscript worked as expected. + ''' + logger = logging.getLogger() + + username = self.get_operator_username() + password = self.get_operator_password(username) + + jobname='admin-runscript-server' + level = None + expected_logs = [ + ': BeforeJob: jobname={}'.format(jobname), + ': BeforeJob: daemon=bareos-dir', + ': BeforeJob: jobtype=Admin', + ] + + director_root = bareos.bsock.DirectorConsoleJson( + address=self.director_address, + port=self.director_port, + name=username, + password=password) + + # Example log entry: + #{ + # "time": "2019-12-02 00:07:34", + # "logtext": "bareos-dir JobId 76: BeforeJob: jobname=admin-runscript-server\n" + #}, + + jobId = self.run_job_and_search_joblog(director_root, jobname, level, expected_logs) + + + def test_admin_runscript_client(self): + ''' + RunScripts configured with "RunsOnClient = yes" (default) + are not executed in Admin Jobs. + Instead, a warning is written to the joblog. + ''' + logger = logging.getLogger() + + username = self.get_operator_username() + password = self.get_operator_password(username) + + jobname='admin-runscript-client' + level = None + expected_logs = [ + ': Invalid runscript definition', + ] + + director_root = bareos.bsock.DirectorConsoleJson( + address=self.director_address, + port=self.director_port, + name=username, + password=password) + + # Example log entry: + #{ + #"time": "2019-12-12 13:23:16", + #"logtext": "bareos-dir JobId 7: Warning: Invalid runscript definition (command=...). Admin Jobs only support local runscripts.\n" + #}, + + jobId = self.run_job_and_search_joblog(director_root, jobname, level, expected_logs) + + + class PythonBareosFiledaemonTest(PythonBareosBase): @unittest.skipUnless(bareos.bsock.DirectorConsole.is_tls_psk_available(), diff --git a/systemtests/tests/python-bareos-test/write.sh b/systemtests/tests/python-bareos-test/write.sh new file mode 100755 index 00000000000..1d81aa90b4f --- /dev/null +++ b/systemtests/tests/python-bareos-test/write.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +file=$1 +shift + +printf "date=%s\n" "$(LANG=C date)" | tee --append $file +for i in "$@"; do + echo "$i" | tee --append $file +done