From 605cfb88a8f3e856096f904d09455d1b1da5c132 Mon Sep 17 00:00:00 2001 From: Maik Aussendorf Date: Tue, 17 Dec 2019 15:50:49 +0100 Subject: [PATCH 01/25] filed: import percona plugin from contrib --- core/src/plugins/filed/BareosFdPercona.py | 431 ++++++++++++++++++++ core/src/plugins/filed/CMakeLists.txt | 5 + core/src/plugins/filed/bareos-fd-percona.py | 27 ++ 3 files changed, 463 insertions(+) create mode 100644 core/src/plugins/filed/BareosFdPercona.py create mode 100644 core/src/plugins/filed/bareos-fd-percona.py diff --git a/core/src/plugins/filed/BareosFdPercona.py b/core/src/plugins/filed/BareosFdPercona.py new file mode 100644 index 00000000000..002f66a7d07 --- /dev/null +++ b/core/src/plugins/filed/BareosFdPercona.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2015-2016 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# +# Author: Maik Aussendorf +# +# Uses Percona's xtrabackup for backup and restore of MySQL / MariaDB databases + +from bareosfd import * +from bareos_fd_consts import * +import os +from subprocess import * +from BareosFdPluginBaseclass import * +import BareosFdWrapper +import datetime +import time +import tempfile +import shutil +import json + + +class BareosFdPercona (BareosFdPluginBaseclass): + ''' + Plugin for backing up all mysql innodb databases found in a specific mysql server + using the Percona xtrabackup tool. + ''' + + def __init__(self, context, plugindef): + # BareosFdPluginBaseclass.__init__(self, context, plugindef) + super(BareosFdPercona, self).__init__(context, plugindef) + # we first create and backup the stream and after that + # the lsn file as restore-object + self.files_to_backup = ['lsnfile', 'stream'] + self.tempdir = tempfile.mkdtemp() + #self.logdir = GetValue(context, bVariable['bVarWorkingDir']) + self.logdir = '/var/log/bareos/' + self.log = 'bareos-plugin-percona.log' + self.rop_data = {} + self.max_to_lsn = 0 + self.err_fd = None + + def parse_plugin_definition(self, context, plugindef): + ''' + We have default options that should work out of the box in the most use cases + that the mysql/mariadb is on the same host and can be accessed without user/password information, + e.g. with a valid my.cnf for user root. + ''' + BareosFdPluginBaseclass.parse_plugin_definition(self, context, plugindef) + + if 'dumpbinary' in self.options: + self.dumpbinary = self.options['dumpbinary'] + else: + self.dumpbinary = "xtrabackup" + + if 'restorecommand' not in self.options: + self.restorecommand = "xbstream -x -C " + else: + self.restorecommand = self.options['restorecommand'] + + # Default is not to write an extra logfile + if 'log' not in self.options: + self.log = False + elif self.options['log'] == 'false': + self.log = False + elif os.path.isabs(self.options['log']): + self.log = self.options['log'] + else: + self.log = os.path.join(self.logdir, self.options['log']) + + # By default, standard mysql-config files will be used, set + # this option to use extra files + self.connect_options = { 'read_default_group': 'client' } + if 'mycnf' in self.options: + self.connect_options['read_default_file'] = self.options['mycnf'] + self.mycnf = "--defaults-extra-file=%s " % self.options['mycnf'] + else: + self.mycnf = "" + + # If true, incremental jobs will only be performed, if LSN has increased + # since last call. + if 'strictIncremental' in self.options and self.options['strictIncremental'] == 'true': + self.strictIncremental = True + else: + self.strictIncremental = False + + self.dumpoptions = self.mycnf + + # if dumpoptions is set, we use that here, otherwise defaults + if 'dumpoptions' in self.options: + self.dumpoptions += self.options['dumpoptions'] + else: + self.dumpoptions += " --backup --stream=xbstream" + + self.dumpoptions += " --extra-lsndir=%s" % self.tempdir + + if 'extradumpoptions' in self.options: + self.dumpoptions += " " + self.options['extradumpoptions'] + + # We need to call mysql to get the current Log Sequece Number (LSN) + if 'mysqlcmd' in self.options: + self.mysqlcmd = self.options['mysqlcmd'] + else: + self.mysqlcmd = "mysql %s -r" % self.mycnf + + return bRCs['bRC_OK'] + + def check_plugin_options(self, context, mandatory_options=None): + accurate_enabled = GetValue(context, bVariable['bVarAccurate']) + if accurate_enabled is not None and accurate_enabled != 0: + JobMessage(context, bJobMessageType['M_FATAL'], + "start_backup_job: Accurate backup not allowed please disable in Job\n") + return bRCs['bRC_Error'] + else: + return bRCs['bRC_OK'] + + def create_file(self, context, restorepkt): + ''' + On restore we create a subdirectory for the first base backup and each incremental backup. + Because percona expects an empty directory, we create a tree starting with jobId/ of restore job + ''' + FNAME = restorepkt.ofname + DebugMessage(context, 100, "create file with %s called\n" % FNAME) + self.writeDir = "%s/%d/" % (os.path.dirname(FNAME), self.jobId) + # FNAME contains originating jobId after last . + origJobId = int(FNAME.rpartition('.')[-1]) + if origJobId in self.rop_data: + rop_from_lsn = int(self.rop_data[origJobId]['from_lsn']) + rop_to_lsn = int(self.rop_data[origJobId]['to_lsn']) + self.writeDir += "/%020d_" % rop_from_lsn + self.writeDir += "%020d_" % rop_to_lsn + self.writeDir += "%010d" % origJobId + else: + JobMessage(context, bJobMessageType['M_ERROR'], + "No lsn information found in restore object for file %s from job %d\n" % (FNAME, origJobId)) + + # Create restore directory, if not existant + if not os.path.exists(self.writeDir): + bareosfd.DebugMessage( + context, 200, + "Directory %s does not exist, creating it now\n" % + self.writeDir) + os.makedirs(self.writeDir) + # Percona requires empty directory + if os.listdir(self.writeDir): + JobMessage(context, bJobMessageType['M_FATAL'], + "Restore with xbstream needs empty directoy: %s\n" % self.writeDir) + return bRCs['bRC_Error'] + self.restorecommand += self.writeDir + DebugMessage(context, 100, "Restore using xbstream to extract files with \"%s\"\n" % self.restorecommand) + restorepkt.create_status = bCFs['CF_EXTRACT'] + return bRCs['bRC_OK'] + + def start_backup_job(self, context): + ''' + We will check, if database has changed since last backup + in the incremental case + ''' + check_option_bRC = self.check_plugin_options(context) + if check_option_bRC != bRCs['bRC_OK']: + return check_option_bRC + bareosfd.DebugMessage( + context, 100, + "start_backup_job, level: %s\n" % chr(self.level)) + if chr(self.level) == 'I': + # We check, if we have a LSN received by restore object from previous job + if self.max_to_lsn == 0: + JobMessage(context, bJobMessageType['M_FATAL'], "No LSN received to be used with incremental backup\n") + return bRCs['bRC_Error'] + # Try to load MySQLdb module + hasMySQLdbModule = False + try: + import MySQLdb + hasMySQLdbModule = True + bareosfd.DebugMessage(context, 100, "Imported module MySQLdb\n") + except ImportError: + bareosfd.DebugMessage(context, 100, "Import of module MySQLdb failed. Using command pipe instead\n") + # contributed by https://github.com/kjetilho + if hasMySQLdbModule: + try: + conn = MySQLdb.connect(**self.connect_options) + cursor = conn.cursor() + cursor.execute('SHOW ENGINE INNODB STATUS') + result = cursor.fetchall() + if len(result) == 0: + JobMessage(context, bJobMessageType['M_FATAL'], "Could not fetch SHOW ENGINE INNODB STATUS, unpriveleged user?") + return bRCs['bRC_Error'] + info = result[0][2] + conn.close() + for line in info.split("\n"): + if line.startswith('Log sequence number'): + last_lsn = int(line.split(' ')[3]) + except Exception, e: + JobMessage(context, bJobMessageType['M_FATAL'], "Could not get LSN, Error: %s" % e) + return bRCs['bRC_Error'] + # use old method as fallback, if module MySQLdb not available + else: + get_lsn_command = ("echo 'SHOW ENGINE INNODB STATUS' | %s | grep 'Log sequence number' | cut -d ' ' -f 4" + % self.mysqlcmd) + last_lsn_proc = Popen(get_lsn_command, shell=True, stdout=PIPE, stderr=PIPE) + last_lsn_proc.wait() + returnCode = last_lsn_proc.poll() + (mysqlStdOut, mysqlStdErr) = last_lsn_proc.communicate() + if returnCode != 0 or mysqlStdErr: + JobMessage(context, bJobMessageType['M_FATAL'], "Could not get LSN with command \"%s\", Error: %s" + % (get_lsn_command, mysqlStdErr)) + return bRCs['bRC_Error'] + else: + try: + last_lsn = int(mysqlStdOut) + except: + JobMessage(context, bJobMessageType['M_FATAL'], "Error reading LSN: \"%s\" not an integer" % mysqlStdOut) + return bRCs['bRC_Error'] + JobMessage(context, bJobMessageType['M_INFO'], "Backup until LSN: %d\n" %last_lsn) + if self.max_to_lsn > 0 and self.max_to_lsn >= last_lsn and self.strictIncremental: + bareosfd.DebugMessage( + context, 100, + "Last LSN of DB %d is not higher than LSN from previous job %d. Skipping this incremental backup\n" + % (last_lsn, self.max_to_lsn)) + self.files_to_backup = ['lsn_only'] + return bRCs['bRC_OK'] + return bRCs['bRC_OK'] + + def start_backup_file(self, context, savepkt): + ''' + This method is called, when Bareos is ready to start backup a file + ''' + if not self.files_to_backup: + self.file_to_backup = None + DebugMessage(context, 100, "start_backup_file: None\n") + else: + self.file_to_backup = self.files_to_backup.pop() + DebugMessage(context, 100, "start_backup_file: %s\n" % self.file_to_backup) + + statp = StatPacket() + savepkt.statp = statp + + if self.file_to_backup == 'stream': + # This is the database backup as xbstream + savepkt.fname = "/_percona/xbstream.%010d" % self.jobId + savepkt.type = bFileType['FT_REG'] + if self.max_to_lsn > 0: + self.dumpoptions += " --incremental-lsn=%d" % self.max_to_lsn + self.dumpcommand = ("%s %s" % (self.dumpbinary, self.dumpoptions)) + DebugMessage(context, 100, "Dumper: '" + self.dumpcommand + "'\n") + elif self.file_to_backup == 'lsnfile': + # The restore object containing the log sequence number (lsn) + # Read checkpoints and create restore object + checkpoints = {} + # improve: Error handling + with open("%s/xtrabackup_checkpoints" % self.tempdir) as lsnfile: + for line in lsnfile: + key, value = line.partition("=")[::2] + checkpoints[key.strip()] = value.strip() + savepkt.fname = "/_percona/xtrabackup_checkpoints" + savepkt.type = bFileType['FT_RESTORE_FIRST'] + savepkt.object_name = savepkt.fname + savepkt.object = bytearray(json.dumps(checkpoints)) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = int(time.time()) + shutil.rmtree(self.tempdir) + elif self.file_to_backup == 'lsn_only': + # We have nothing to backup incremental, so we just have to pass + # the restore object from previous job + savepkt.fname = "/_percona/xtrabackup_checkpoints" + savepkt.type = bFileType['FT_RESTORE_FIRST'] + savepkt.object_name = savepkt.fname + savepkt.object = bytearray(self.row_rop_raw) + savepkt.object_len = len(savepkt.object) + savepkt.object_index = int(time.time()) + else: + # should not happen + JobMessage(context, bJobMessageType['M_FATAL'], "Unknown error. Don't know how to handle %s\n" + % self.file_to_backup) + + JobMessage(context, bJobMessageType['M_INFO'], "Starting backup of " + savepkt.fname + "\n") + return bRCs['bRC_OK'] + + def plugin_io(self, context, IOP): + ''' + Called for io operations. We read from pipe into buffers or on restore + send to xbstream + ''' + DebugMessage(context, 200, "plugin_io called with " + str(IOP.func) + "\n") + + if IOP.func == bIOPS['IO_OPEN']: + DebugMessage(context, 100, "plugin_io called with IO_OPEN\n") + if self.log: + try: + self.err_fd = open(self.log, "a") + except IOError, msg: + DebugMessage(context, 100, "Could not open log file (%s): %s\n" + % (self.log, format(str(msg)))) + if IOP.flags & (os.O_CREAT | os.O_WRONLY): + if self.log: + self.err_fd.write("%s Restore Job %s opens stream with \"%s\"\n" %(datetime.datetime.now(),self.jobId,self.restorecommand)) + self.stream = Popen(self.restorecommand, shell=True, stdin=PIPE, stderr=self.err_fd) + else: + if self.log: + self.err_fd.write("%s Backup Job %s opens stream with \"%s\"\n" %(datetime.datetime.now(),self.jobId,self.dumpcommand)) + self.stream = Popen(self.dumpcommand, shell=True, stdout=PIPE, stderr=self.err_fd) + return bRCs['bRC_OK'] + + elif IOP.func == bIOPS['IO_READ']: + IOP.buf = bytearray(IOP.count) + IOP.status = self.stream.stdout.readinto(IOP.buf) + IOP.io_errno = 0 + return bRCs['bRC_OK'] + + elif IOP.func == bIOPS['IO_WRITE']: + try: + self.stream.stdin.write(IOP.buf) + IOP.status = IOP.count + IOP.io_errno = 0 + except IOError, msg: + IOP.io_errno = -1 + DebugMessage(context, 100, "Error writing data: " + format(str(msg)) + "\n") + return bRCs['bRC_OK'] + + elif IOP.func == bIOPS['IO_CLOSE']: + DebugMessage(context, 100, "plugin_io called with IO_CLOSE\n") + self.subprocess_returnCode = self.stream.poll() + if self.subprocess_returnCode is None: + # Subprocess is open, we wait until it finishes and get results + try: + self.stream.communicate() + self.subprocess_returnCode = self.stream.poll() + except: + JobMessage(context, bJobMessageType['M_ERROR'], + "Dump / restore command not finished properly\n") + bRCs['bRC_Error'] + return bRCs['bRC_OK'] + else: + DebugMessage(context, 100, "Subprocess has terminated with returncode: %d\n" + % self.subprocess_returnCode) + return bRCs['bRC_OK'] + + elif IOP.func == bIOPS['IO_SEEK']: + return bRCs['bRC_OK'] + + else: + DebugMessage(context, 100, "plugin_io called with unsupported IOP:" + str(IOP.func) + "\n") + return bRCs['bRC_OK'] + + def end_backup_file(self, context): + ''' + Check if dump was successful. + ''' + # Usually the xtrabackup process should have terminated here, but on some servers + # it has not always. + if self.file_to_backup == 'stream': + returnCode = self.subprocess_returnCode + if returnCode is None: + JobMessage(context, bJobMessageType['M_ERROR'], "Dump command not finished properly for unknown reason\n") + returnCode = -99 + else: + DebugMessage(context, 100, "end_backup_file() entry point in Python called. Returncode: %d\n" + % self.stream.returncode) + if returnCode != 0: + msg = [ "Dump command returned non-zero value: %d" % returnCode, + "command: \"%s\"" % self.dumpcommand ] + if self.log: + msg += ["log file: \"%s\"" % self.log] + JobMessage(context, bJobMessageType['M_FATAL'], ", ".join(msg) + "\n") + if returnCode != 0: + return bRCs['bRC_Error'] + + if self.log: + self.err_fd.write("%s Backup Job %s closes stream\n" %(datetime.datetime.now(),self.jobId)) + self.err_fd.close() + + if self.files_to_backup: + return bRCs['bRC_More'] + else: + return bRCs['bRC_OK'] + + def end_restore_file(self, context): + ''' + Check, if writing to restore command was succesfull. + ''' + returnCode = self.subprocess_returnCode + if returnCode is None: + JobMessage(context, bJobMessageType['M_ERROR'], "Restore command not finished properly for unknown reason\n") + returnCode = -99 + else: + DebugMessage(context, 100, + "end_restore_file() entry point in Python called. Returncode: %d\n" + % self.stream.returncode) + if returnCode != 0: + msg = [ "Restore command returned non-zero value: %d" % return_code ] + if self.log: + msg += [ "log file: \"%s\"" % self.log ] + JobMessage(context, bJobMessageType['M_ERROR'], ", ".join(msg) + "\n") + if self.log: + self.err_fd.write("%s Restore Job %s closes stream\n" %(datetime.datetime.now(),self.jobId)) + self.err_fd.close() + + if returnCode == 0: + return bRCs['bRC_OK'] + else: + return bRCs['bRC_Error'] + + def restore_object_data(self, context, ROP): + ''' + Called on restore and on diff/inc jobs. + ''' + # Improve: sanity / consistence check of restore object + self.row_rop_raw = ROP.object + self.rop_data[ROP.jobid] = json.loads(str(self.row_rop_raw)) + if 'to_lsn' in self.rop_data[ROP.jobid] and self.rop_data[ROP.jobid]['to_lsn'] > self.max_to_lsn: + self.max_to_lsn = int(self.rop_data[ROP.jobid]['to_lsn']) + JobMessage(context, bJobMessageType['M_INFO'], + "Got to_lsn %d from restore object of job %d\n" % (self.max_to_lsn, ROP.jobid)) + return bRCs['bRC_OK'] + + +# vim: ts=4 tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/core/src/plugins/filed/CMakeLists.txt b/core/src/plugins/filed/CMakeLists.txt index da83cfdef1f..7c6035c4f21 100644 --- a/core/src/plugins/filed/CMakeLists.txt +++ b/core/src/plugins/filed/CMakeLists.txt @@ -136,10 +136,15 @@ set(PYFILES BareosFdPluginLocalFileset.py BareosFdWrapper.py bareos_fd_consts.py + bareos-fd-ldap.py BareosFdPluginLDAP.py + bareos-fd-ovirt.py BareosFdPluginOvirt.py + + bareos-fd-percona.py + BareosFdPercona.py ) install( diff --git a/core/src/plugins/filed/bareos-fd-percona.py b/core/src/plugins/filed/bareos-fd-percona.py new file mode 100644 index 00000000000..738aa12976d --- /dev/null +++ b/core/src/plugins/filed/bareos-fd-percona.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Bareos-fd-local-fileset a simple example for a python Bareos FD Plugin using BareosFdPluginLocalFileset +# The plugin argument 'filename' is used to read all files listed in that file and add it to the fileset +# License: AGPLv3 + +# Provided by the Bareos FD Python plugin interface +from bareosfd import * +from bareos_fd_consts import * + +# This module contains the wrapper functions called by the Bareos-FD, the functions call the corresponding +# methods from your plugin class +from BareosFdWrapper import * + +# This module contains the used plugin class +from BareosFdPercona import * + +def load_bareos_plugin(context, plugindef): + ''' + This function is called by the Bareos-FD to load the plugin + We use it to instantiate the plugin class + ''' + # BareosFdWrapper.bareos_fd_plugin_object is the module attribute that holds the plugin class object + BareosFdWrapper.bareos_fd_plugin_object = BareosFdPercona (context, plugindef); + return bRCs['bRC_OK']; + +# the rest is done in the Plugin module From 763686d6a692a7ff04aa81a73d7f44d6c28bae68 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 17 Dec 2019 15:56:41 +0100 Subject: [PATCH 02/25] imported Percona Xtrabackup Plugin Readme --- .../source/TasksAndConcepts/Plugins.rst | 223 ++++++++++++++++-- 1 file changed, 201 insertions(+), 22 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index bdf83ec5ce2..9883260738f 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -3,7 +3,7 @@ Plugins ======= -:index:`\ `\ +:index:`\ `\ The functionality of Bareos can be extended by plugins. They do exists plugins for the different daemons (Director, Storage- and File-Daemon). @@ -31,7 +31,7 @@ File Daemon plugins are configured by the :strong:`Plugin`\ directive of a :ref bpipe Plugin ~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ The bpipe plugin is a generic pipe program, that simply transmits the data from a specified program to Bareos for backup, and from Bareos to a specified program for restore. The purpose of the plugin is to provide an interface to any system program for backup and restore. That allows you, for example, to do database backups without a local dump. By using different command lines to bpipe, you can backup any kind of data (ASCII or binary) depending on the program called. @@ -98,35 +98,35 @@ See chapter :ref:`MSSQL`. LDAP Plugin ~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ This plugin is intended to backup (and restore) the contents of a LDAP server. It uses normal LDAP operation for this. The package **bareos-filedaemon-ldap-python-plugin** (:sinceVersion:`15.2.0: LDAP Plugin`) contains an example configuration file, that must be adapted to your envirnoment. Cephfs Plugin ~~~~~~~~~~~~~ -:index:`\ `\ :index:`\ `\ +:index:`\ `\ :index:`\ `\ Opposite to the :ref:`Rados Backend ` that is used to store data on a CEPH Object Store, this plugin is intended to backup a CEPH Object Store via the Cephfs interface to other media. The package **bareos-filedaemon-ceph-plugin** (:sinceVersion:`15.2.0: Cephfs Plugin`) contains an example configuration file, that must be adapted to your envirnoment. Rados Plugin ~~~~~~~~~~~~ -:index:`\ `\ :index:`\ `\ +:index:`\ `\ :index:`\ `\ Opposite to the :ref:`Rados Backend ` that is used to store data on a CEPH Object Store, this plugin is intended to backup a CEPH Object Store via the Rados interface to other media. The package **bareos-filedaemon-ceph-plugin** (:sinceVersion:`15.2.0: CEPH Rados Plugin`) contains an example configuration file, that must be adapted to your envirnoment. GlusterFS Plugin ~~~~~~~~~~~~~~~~ -:index:`\ `\ :index:`\ `\ +:index:`\ `\ :index:`\ `\ Opposite to the :ref:`GFAPI Backend ` that is used to store data on a Gluster system, this plugin is intended to backup data from a Gluster system to other media. The package **bareos-filedaemon-glusterfs-plugin** (:sinceVersion:`15.2.0: GlusterFS Plugin`) contains an example configuration file, that must be adapted to your envirnoment. python-fd Plugin ~~~~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ The **python-fd** plugin behaves similar to the :ref:`director-python-plugin`. Base plugins and an example get installed via the package bareos-filedaemon-python-plugin. Configuration is done in the :ref:`DirectorResourceFileSet` on the director. @@ -149,7 +149,7 @@ Command plugins are used to replace or extend the FileSet definition in the File File = "/etc" Plugin = "python:module_path=/usr/lib/bareos/plugins:module_name=bareos-fd-mysql" } - } + } :index:`\ `\ This example uses the :ref:`MySQL plugin ` to backup MySQL dumps in addition to :file:`/etc`. @@ -182,7 +182,7 @@ This plugin bareos-fd-file-interact from https://github.com/bareos/bareos-contri VMware Plugin ~~~~~~~~~~~~~ -:index:`\ `\ :index:`\ `\ +:index:`\ `\ :index:`\ `\ The |vmware| Plugin can be used for agentless backups of virtual machines running on |vsphere|. It makes use of CBT (Changed Block Tracking) to do space efficient full and incremental backups, see below for mandatory requirements. @@ -219,7 +219,7 @@ Current limitations amongst others are: Until Bareos Version 15.2.2, the restore has only be possible to the same existing VM with existing virtual disks. Since :sinceVersion:`15.2.3: VMware Plugin: restore to VMDK files` - %**bareos-vadp-dumper** :sinceVersion:`15.2.2-15: bareos-vadp-dumper` and + %**bareos-vadp-dumper** :sinceVersion:`15.2.2-15: bareos-vadp-dumper` and %**bareos-vmware-plugin** :sinceVersion:`15.2.2-27: bareos-vmware-plugin` it is also possible to restore to local VMDK files, see below for more details. @@ -460,7 +460,7 @@ For restore, the VM must be powered off and no snapshot must exist. In :command: Restore to local VMDK File ^^^^^^^^^^^^^^^^^^^^^^^^^^ -:index:`\ `\ +:index:`\ `\ Since :sinceVersion:`15.2.3: VMware Plugin: restore to VMDK files` it is possible to restore to local VMDK files. That means, instead of directly restoring a disk that belongs to the VM, the restore creates VMDK disk image files on the filesystem of the system that runs the |fd|. As the VM that the backup was taken from is not affected by this, it can remain switched on while restoring to local VMDK. Such a restored VMDK file can then be uploaded to a |vsphere| datastore or accessed by tools like `guestfish `_ to extract single files. @@ -501,7 +501,7 @@ For restoring to local VMDK, the plugin option :strong:`localvmdk=yes` must be p ... You have selected the following JobIds: 625,626,631,632,635 - Building directory tree for JobId(s) 625,626,631,632,635 ... + Building directory tree for JobId(s) 625,626,631,632,635 ... 10 files inserted into the tree. You are now entering file selection mode where you add (mark) and @@ -518,7 +518,7 @@ For restoring to local VMDK, the plugin option :strong:`localvmdk=yes` must be p The job will require the following Volume(s) Storage(s) SD Device(s) =========================================================================== - + Full-0001 File FileStorage ... Incremental-0078 File FileStorage @@ -983,6 +983,185 @@ This will create disk image files that could be examined for example by using the **guestfish** tool (see http://libguestfs.org/guestfish.1.html). This tool can also be used to extract single files from the disk image. +.. _PerconaXtrabackupPlugin: + +Percona Xtrabackup Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~ + +# The Bareos MySQL / MariaDB Percona xtrabackup Plugin + +This plugin uses Perconas xtrackup tool, to make full and incremental backups of Mysql / MariaDB databases. + +The key features of xtrabackup are: + +- Incremental backups +- Backups that complete quickly and reliably +- Uninterrupted transaction processing during backups +- Savings on disk space and network bandwidth +- Higher uptime due to faster restore time + +Incremental backups only work for INNODB tables, when using MYISAM, only full backups can be created. + +## Prerequisites +You need to have the 'mysql' client program and the xtrabackup tool installed. +More about xtrabackup: https://www.percona.com/software/mysql-database/percona-xtrabackup + +You will also need the package *bareos-filedaemon-python-plugin* installed on your client. + +## Compatibility + +There are different versions of _xtrabackup_ available. Older versions required an extra binary called _innobackupex_, especially when dealing with myISAM tables. In newer versions, _innobackupex_ is just a sysmbolic link to _xtrabackup_. + +We've tested some versions of xtrabackup together with the plugin: + +| xtrabackup version | Status | Remarks | +| -----------------: |:------:| -------:| +|2.0.8| - | InnoDB only seems to work | +|2.3.5| + | | +|2.4.4| + | | + +We've used the official Percona rpms on Centos 6 for testing. + +## Installation ## + +1. Make sure you have met the prerequisites. +2. Install the files *BareosFdPercona.py* and *bareos-fd-percona.py* in your Bareos plugin directory (usually */usr/lib64/bareos/plugins*) + + +## Configuration ## + +### Activate your plugin directory in the fd resource conf on the client +``` +FileDaemon { + Name = client-fd + ... + Plugin Directory = /usr/lib64/bareos/plugins +} +``` + +### Include the Plugin in the fileset definition on the director +``` +FileSet { + Name = "client-data" + Include { + Options { + compression=GZIP + signature = MD5 + } + File = /etc + #... + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona" + } +} +``` + +#### Options #### + +You can append options to the plugin call as key=value pairs, separated by ':'. +Please read more about the Bareos Python Plugin Interface here: http://doc.bareos.org/master/html/bareos-manual-main-reference.html#Python-fdPlugin + + +##### defaultsfile #### + +This parameter allows to specify a defaultsfile that shall be used for mysql(client) and *xtrabackup* command line utilities. +Example: + +``` +Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/path/to/your/my.cnf" +``` + +##### dumpbinary ##### + +Command (with or without full path) to create the dumps. Default: *xtrabackup* + +##### dumpoptions ##### + +Options to be used with the dumpbinary. +Default: + --backup --stream=xbstream + +##### extradumpoptions ##### + +Additional options appended to dumpoptions. + +###### Choosing databases ###### + +By default all found databases are backed up. You can restrict this +using the dumpoptions or extradumpoptions parameter. If you modify +dumpoptions, be careful that you include all necessary options. See +*xtrabackup* documentation for details. + + +##### restorecommand +Command used for restoring, default: + xbstream -x -C + +##### strictIncremental ##### +Default: False + +By default, an incremental will create data, even if the Log Sequence Number (LSN) wasn't increased since last backup. This is to ensure, that eventual changes to +MYISAM tables get into the backup. MYISAM does not support incremental backups, you will always get a full bakcup of these tables. + +If set to true, no data will be written into backup, if the LSN wasn't changed. + +##### log ##### +Default: false + +By default, no extra logfile is written on the FD running the plugin. If you want to have some additional debug information, you might specify a +logfile here. If you set a filename with path, this will be used. If you specify just a filename without path, the default path for logs +*/var/log/bareos/* will be prepended. + +If you use a logfilename that matches */var/log/bareos/bareos\*.log*, it will be handled by logrotate. + +## Backup ## + +When running full backups, the plugin will call the _xtrabackup_ command with the according options. Format is _xbstream_. LSN information +will be written into a temporary directory, using the _--extra-lsndir_ option. The information (LSN) will be used to write a so called +restore object. This information is needed for following incremental jobs, so they are aware of the previous backups (and how far by +means of LSN) they went. + +## Restore ## + +With the usual Bareos restore mechanism a file-hierarchy will be created on the restore client under the default restore location: + +*/tmp/bareos-restores/_percona/* + +Each restore job gets an own subdirectory, because Percona expects an empty directory. In that subdirectory, +a new directory is created for every backup job that was part of the Full-Incremental sequence. + +The naming scheme is: +*fromLSN_toLSN_jobid* + +Example: +``` +/tmp/bareos-restores/_percona/351/ +├── 00000000000000000000_00000000000010129154_0000000334 +├── 00000000000010129154_00000000000010142295_0000000335 +├── 00000000000010142295_00000000000010201260_0000000338 +``` + +This example shows the restore tree for restore job with ID 351. First subdirectory has all files +from the first full backup job with ID 334. It starts at LSN 0 and goes until LSN 10129154. + +Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. +The third line is the 2nd incremental job with ID 338. + +To further prepare the restored files, use the *xtrabackup --prepare* command. Read https://www.percona.com/doc/percona-xtrabackup/2.4/xtrabackup_bin/incremental_backups.html +for more information. + + +## Troubleshooting ## + +If things don't work as expected, make sure: + +- Bareos FileDaemon (FD) works in general, so that you can make simple file backups and restores +- Bareos FD Python plugins work in genreral, try one of the shipped simple sample plugins +- Make sure *xtrabackup* works as user root, MySQL access needs to be configured properly + +If this all does not help, you can start the Bareos FD in Debug mode and look deeper into it. + +Support is available here: https://www.bareos.com + .. _sdPlugins: @@ -994,7 +1173,7 @@ Storage Daemon Plugins autoxflate-sd ~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ This plugin is part of the **bareos-storage** package. @@ -1052,7 +1231,7 @@ Additional :config:option:`sd/storage/AutoXflateOnReplication`\ can be configur scsicrypto-sd ~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ This plugin is part of the **bareos-storage-tape** package. @@ -1129,9 +1308,9 @@ Some security levels need to be increased for the storage daemon to be able to u The following additional security is needed for the following operating systems: Linux (SG_IO ioctl interface): - -The user running the storage daemon needs the following additional capabilities: :index:`\ `\ + +The user running the storage daemon needs the following additional capabilities: :index:`\ `\ - :strong:`CAP_SYS_RAWIO` (see capabilities(7)) @@ -1162,9 +1341,9 @@ Check the setting with If :command:`bareos-sd` does not have the appropriate capabilities, all other tape operations may still work correctly, but you will get "Unable to perform SG\_IO ioctl" errors. Solaris (USCSI ioctl interface): - -The user running the storage daemon needs the following additional privileges: :index:`\ `\ + +The user running the storage daemon needs the following additional privileges: :index:`\ `\ - :strong:`PRIV_SYS_DEVICES` (see privileges(5)) @@ -1248,14 +1427,14 @@ of AME which may for big libraries be easier, although the overhead of using AME scsitapealert-sd ~~~~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ This plugin is part of the **bareos-storage-tape** package. python-sd Plugin ~~~~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ The **python-sd** plugin behaves similar to the :ref:`director-python-plugin`. @@ -1269,7 +1448,7 @@ Director Plugins python-dir Plugin ~~~~~~~~~~~~~~~~~ -:index:`\ `\ +:index:`\ `\ The **python-dir** plugin is intended to extend the functionality of the Bareos Director by Python code. A working example is included. From d5cb61dd52080ad4224c481c2aa95bc6014a94b6 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 17 Dec 2019 17:01:38 +0100 Subject: [PATCH 03/25] adapt spec file for percona plugin --- core/platforms/packaging/bareos.spec | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/core/platforms/packaging/bareos.spec b/core/platforms/packaging/bareos.spec index 7582419b222..f770c7fadcf 100644 --- a/core/platforms/packaging/bareos.spec +++ b/core/platforms/packaging/bareos.spec @@ -577,6 +577,13 @@ Requires: python-pycurl Requires: python-lxml Requires: python-ovirt-engine-sdk4 +%package filedaemon-percona-python-plugin +Summary: LDAP Python plugin for Bareos File daemon +Group: Productivity/Archiving/Backup +Requires: bareos-filedaemon = %{version} +Requires: bareos-filedaemon-python-plugin = %{version} +#Requires: python-percona + %package storage-python-plugin Summary: Python plugin for Bareos Storage daemon Group: Productivity/Archiving/Backup @@ -602,6 +609,11 @@ This package contains the LDAP python plugin for the file daemon This package contains the Ovirt python plugin for the file daemon +%description filedaemon-percona-python-plugin +%{dscr} + +This package contains the Percona python plugin for the file daemon + %description storage-python-plugin %{dscr} @@ -1387,6 +1399,14 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-ovirt.conf.example %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-ovirt.conf.example +%files filedaemon-percona-python-plugin +%defattr(-, root, root) +%{plugin_dir}/bareos-fd-percona.py* +%{plugin_dir}/BareosFdPercona.py* +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona.conf.example + %files director-python-plugin %defattr(-, root, root) %{plugin_dir}/python-dir.so @@ -1644,11 +1664,17 @@ fi; \ %if 0%{?python_plugins} + %post filedaemon-ldap-python-plugin %post_backup_file /etc/%{name}/bareos-dir.d/plugin-python-ldap.conf - %posttrans filedaemon-ldap-python-plugin %posttrans_restore_file /etc/%{name}/bareos-dir.d/plugin-python-ldap.conf + +#post filedaemon-percona-python-plugin +#post_backup_file /etc/#{name}/bareos-dir.d/plugin-python-percona.conf +#posttrans filedaemon-percona-python-plugin +#posttrans_restore_file /etc/#{name}/bareos-dir.d/plugin-python-percona.conf + %endif From b98a851c0fa1fbdcce300ff55db0931c58831827 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 10:41:56 +0100 Subject: [PATCH 04/25] first part of python-fd-percona-plugin-test --- core/CMakeLists.txt | 1 + core/cmake/BareosFindPrograms.cmake | 1 + systemtests/CMakeLists.txt | 8 +++ .../bareos-dir.d/catalog/MyCatalog.conf.in | 8 +++ .../bareos-dir.d/client/bareos-fd.conf.in | 7 ++ .../bareos-dir.d/console/bareos-mon.conf.in | 7 ++ .../bareos-dir.d/director/bareos-dir.conf.in | 27 ++++++++ .../bareos-dir.d/fileset/Catalog.conf.in | 11 +++ .../bareos-dir.d/fileset/PerconaTest.conf.in | 10 +++ .../bareos-dir.d/fileset/SelfTest.conf.in | 11 +++ .../bareos-dir.d/job/BackupCatalog.conf.in | 20 ++++++ .../bareos-dir.d/job/RestoreFiles.conf.in | 11 +++ .../bareos-dir.d/job/backup-bareos-fd.conf.in | 6 ++ .../bareos-dir.d/jobdefs/DefaultJob.conf.in | 15 ++++ .../bareos-dir.d/messages/Daemon.conf.in | 7 ++ .../bareos-dir.d/messages/Standard.conf.in | 7 ++ .../bareos-dir.d/pool/Differential.conf | 10 +++ .../etc/bareos/bareos-dir.d/pool/Full.conf | 10 +++ .../bareos/bareos-dir.d/pool/Incremental.conf | 10 +++ .../etc/bareos/bareos-dir.d/pool/Scratch.conf | 4 ++ .../bareos/bareos-dir.d/profile/operator.conf | 18 +++++ .../bareos/bareos-dir.d/storage/File.conf.in | 8 +++ .../bareos/bareos-fd.d/client/myself.conf.in | 19 ++++++ .../bareos-fd.d/director/bareos-dir.conf.in | 5 ++ .../bareos-fd.d/director/bareos-mon.conf.in | 6 ++ .../bareos/bareos-fd.d/messages/Standard.conf | 5 ++ .../bareos-sd.d/device/FileStorage.conf.in | 11 +++ .../bareos-sd.d/director/bareos-dir.conf.in | 5 ++ .../bareos-sd.d/director/bareos-mon.conf.in | 6 ++ .../bareos/bareos-sd.d/messages/Standard.conf | 5 ++ .../bareos-sd.d/storage/bareos-sd.conf.in | 14 ++++ .../etc/bareos/bconsole.conf.in | 10 +++ .../client/FileDaemon-local.conf.in | 5 ++ .../director/Director-local.conf.in | 4 ++ .../tray-monitor.d/monitor/bareos-mon.conf.in | 7 ++ .../storage/StorageDaemon-local.conf.in | 5 ++ .../python-fd-percona-plugin-test/testrunner | 68 +++++++++++++++++++ 37 files changed, 392 insertions(+) create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in create mode 100644 systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in create mode 100755 systemtests/tests/python-fd-percona-plugin-test/testrunner diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f353cae85da..92edf0b70b3 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -922,6 +922,7 @@ message(" AWK: ${AWK}") message(" GAWK: ${GAWK}") message(" RPCGEN: ${RPCGEN}") message(" MTX: ${MTX}") +message(" XTRABACKUP: ${XTRABACKUP}") message(" DEVELOPER: ${developer}") message(" LocalBuildDefinitionsFile: ${BareosLocalBuildDefinitionsFile}") message(" HAVE_IS_TRIVIALLY_COPYABLE: ${HAVE_IS_TRIVIALLY_COPYABLE}") diff --git a/core/cmake/BareosFindPrograms.cmake b/core/cmake/BareosFindPrograms.cmake index ce17e1f75eb..ff46060be6f 100644 --- a/core/cmake/BareosFindPrograms.cmake +++ b/core/cmake/BareosFindPrograms.cmake @@ -38,3 +38,4 @@ find_program(GCORE gcore) find_program(GDB gdb) find_program(DBX dbx) find_program(MDB mdb) +find_program(XTRABACKUP xtrabackup) diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index fa31c70c3e7..ba2d7f48458 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -101,6 +101,8 @@ macro(handle_python_plugin_modules) filed/BareosFdPluginOvirt.py filed/bareos-fd-ovirt.py + filed/BareosFdPluginPercona.py + filed/bareos-fd-percona.py dird/bareos_dir_consts.py dird/BareosDirPluginBaseclass.py @@ -370,6 +372,12 @@ else() list(APPEND SYSTEM_TESTS_DISABLED "python-fd-ovirt-plugin-test") endif() +if(TARGET python-fd AND XTRABACKUP) + list(APPEND SYSTEM_TESTS "python-fd-percona-plugin-test") +else() + list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona-plugin-test") +endif() + if(TARGET python-dir) list(APPEND SYSTEM_TESTS "python-dir-plugin-test") else() diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in new file mode 100644 index 00000000000..479bc6fecbb --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in @@ -0,0 +1,8 @@ +Catalog { + Name = MyCatalog + #dbdriver = "@DEFAULT_DB_TYPE@" + dbdriver = "XXX_REPLACE_WITH_DATABASE_DRIVER_XXX" + dbname = "@db_name@" + dbuser = "@db_user@" + dbpassword = "@db_password@" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in new file mode 100644 index 00000000000..470ca702035 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in @@ -0,0 +1,7 @@ +Client { + Name = bareos-fd + Description = "Client resource of the Director itself." + Address = localhost + Password = "@fd_password@" # password for FileDaemon + FD PORT = @fd_port@ +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in new file mode 100644 index 00000000000..d276adcb87d --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in @@ -0,0 +1,7 @@ +Console { + Name = bareos-mon + Description = "Restricted console used by tray-monitor to get the status of the director." + Password = "@mon_dir_password@" + CommandACL = status, .status + JobACL = *all* +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..8346e62ddc7 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in @@ -0,0 +1,27 @@ +Director { # define myself + Name = bareos-dir + QueryFile = "@scriptdir@/query.sql" + Maximum Concurrent Jobs = 10 + Password = "@dir_password@" # Console password + Messages = Daemon + Auditing = yes + + # Enable the Heartbeat if you experience connection losses + # (eg. because of your router or firewall configuration). + # Additionally the Heartbeat can be enabled in bareos-sd and bareos-fd. + # + # Heartbeat Interval = 1 min + + # remove comment in next line to load dynamic backends from specified directory + Backend Directory = @backenddir@ + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all director plugins (*-dir.so) from the "Plugin Directory". + # + # Plugin Directory = "@dir_plugin_binary_path@" + # Plugin Names = "" + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + DirPort = @dir_port@ +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in new file mode 100644 index 00000000000..c7cdc433fe1 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in @@ -0,0 +1,11 @@ +FileSet { + Name = "Catalog" + Description = "Backup the catalog dump and Bareos configuration files." + Include { + Options { + signature = MD5 + } + File = "@working_dir@/@db_name@.sql" # database dump + File = "@confdir@" # configuration + } +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in new file mode 100644 index 00000000000..c9a39fc57b4 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in @@ -0,0 +1,10 @@ +FileSet { + Name = "perconaTest" + Description = "Test the Plugin functionality of the oVirt Plugin." + Include { + Options { + signature = MD5 + } + Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona:ca=@current_test_directory@/etc/bareos/percona-ca.cert:server=@percona_server@:username=@percona_user@:password=@percona_password@:vm_name=@percona_vm_name@:include_disk_aliases=@percona_include_disk_alias@" + } +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in new file mode 100644 index 00000000000..ba39719ea3f --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in @@ -0,0 +1,11 @@ +FileSet { + Name = "SelfTest" + Description = "fileset just to backup some files for selftest" + Include { + Options { + Signature = MD5 # calculate md5 checksum per file + } + #File = "@sbindir@" + File=<@tmpdir@/file-list + } +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in new file mode 100644 index 00000000000..1da2a7af657 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in @@ -0,0 +1,20 @@ +Job { + Name = "BackupCatalog" + Description = "Backup the catalog database (after the nightly save)" + JobDefs = "DefaultJob" + Level = Full + FileSet="Catalog" + + # This creates an ASCII copy of the catalog + # Arguments to make_catalog_backup.pl are: + # make_catalog_backup.pl + RunBeforeJob = "@scriptdir@/make_catalog_backup.pl MyCatalog" + + # This deletes the copy of the catalog + RunAfterJob = "@scriptdir@/delete_catalog_backup" + + # This sends the bootstrap via mail for disaster recovery. + # Should be sent to another system, please change recipient accordingly + Write Bootstrap = "|@bindir@/bsmtp -h @smtp_host@ -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" @job_email@" # (#01) + Priority = 11 # run after main backup +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in new file mode 100644 index 00000000000..89256864d9a --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in @@ -0,0 +1,11 @@ +Job { + Name = "RestoreFiles" + Description = "Standard Restore template. Only one such job is needed for all standard Jobs/Clients/Storage ..." + Type = Restore + Client = bareos-fd + FileSet = SelfTest + Storage = File + Pool = Incremental + Messages = Standard + Where = @tmp@/bareos-restores +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in new file mode 100644 index 00000000000..137d4d51b92 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in @@ -0,0 +1,6 @@ +Job { + Name = "backup-bareos-fd" + JobDefs = "DefaultJob" + Client = "bareos-fd" + FileSet = "perconaTest" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in new file mode 100644 index 00000000000..563126477c9 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in @@ -0,0 +1,15 @@ +JobDefs { + Name = "DefaultJob" + Type = Backup + Level = Incremental + Client = bareos-fd + FileSet = "SelfTest" + Storage = File + Messages = Standard + Pool = Incremental + Priority = 10 + Write Bootstrap = "@working_dir@/%c.bsr" + Full Backup Pool = Full # write Full Backups into "Full" Pool + Differential Backup Pool = Differential # write Diff Backups into "Differential" Pool + Incremental Backup Pool = Incremental # write Incr Backups into "Incremental" Pool +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in new file mode 100644 index 00000000000..cf6a8cfa1e2 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in @@ -0,0 +1,7 @@ +Messages { + Name = Daemon + Description = "Message delivery for daemon messages (no job)." + console = all, !skipped, !saved, !audit + append = "@logdir@/bareos.log" = all, !skipped, !audit + append = "@logdir@/bareos-audit.log" = audit +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in new file mode 100644 index 00000000000..b3556ba8c23 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in @@ -0,0 +1,7 @@ +Messages { + Name = Standard + Description = "Reasonable message delivery -- send most everything to email address and to the console." + console = all, !skipped, !saved, !audit + append = "@logdir@/bareos.log" = all, !skipped, !saved, !audit + catalog = all, !skipped, !saved, !audit +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf new file mode 100644 index 00000000000..25ce24821ab --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf @@ -0,0 +1,10 @@ +Pool { + Name = Differential + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 90 days # How long should the Differential Backups be kept? (#09) + Maximum Volume Bytes = 10G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Differential-" # Volumes will be labeled "Differential-" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf new file mode 100644 index 00000000000..867fc66b483 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf @@ -0,0 +1,10 @@ +Pool { + Name = Full + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365 days # How long should the Full Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Full-" # Volumes will be labeled "Full-" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf new file mode 100644 index 00000000000..f4dbbab6650 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf @@ -0,0 +1,10 @@ +Pool { + Name = Incremental + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 30 days # How long should the Incremental Backups be kept? (#12) + Maximum Volume Bytes = 1G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Incremental-" # Volumes will be labeled "Incremental-" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf new file mode 100644 index 00000000000..3a489b19871 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf @@ -0,0 +1,4 @@ +Pool { + Name = Scratch + Pool Type = Scratch +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf new file mode 100644 index 00000000000..6edd0166dca --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf @@ -0,0 +1,18 @@ +Profile { + Name = operator + Description = "Profile allowing normal Bareos operations." + + Command ACL = !.bvfs_clear_cache, !.exit, !.sql + Command ACL = !configure, !create, !delete, !purge, !prune, !sqlquery, !umount, !unmount + Command ACL = *all* + + Catalog ACL = *all* + Client ACL = *all* + FileSet ACL = *all* + Job ACL = *all* + Plugin Options ACL = *all* + Pool ACL = *all* + Schedule ACL = *all* + Storage ACL = *all* + Where ACL = *all* +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in new file mode 100644 index 00000000000..4058ddc7edc --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in @@ -0,0 +1,8 @@ +Storage { + Name = File + Address = @hostname@ # N.B. Use a fully qualified name here (do not use "localhost" here). + Password = "@sd_password@" + Device = FileStorage + Media Type = File + SD Port = @sd_port@ +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in new file mode 100644 index 00000000000..49039c29d18 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in @@ -0,0 +1,19 @@ +Client { + Name = @basename@-fd + Maximum Concurrent Jobs = 20 + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all filedaemon plugins (*-fd.so) from the "Plugin Directory". + # + Plugin Directory = "@fd_plugin_binary_path@" + Plugin Names = "python" + + # if compatible is set to yes, we are compatible with bacula + # if set to no, new bareos features are enabled which is the default + # compatible = yes + + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + FD Port = @fd_port@ +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..c8dc7085a45 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in @@ -0,0 +1,5 @@ +Director { + Name = bareos-dir + Password = "@fd_password@" + Description = "Allow the configured Director to access this file daemon." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in new file mode 100644 index 00000000000..630c3a9abd3 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in @@ -0,0 +1,6 @@ +Director { + Name = bareos-mon + Password = "@mon_fd_password@" + Monitor = yes + Description = "Restricted Director, used by tray-monitor to get the status of this file daemon." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf new file mode 100644 index 00000000000..97788e00573 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf @@ -0,0 +1,5 @@ +Messages { + Name = Standard + Director = bareos-dir = all, !skipped, !restored + Description = "Send relevant messages to the Director." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in new file mode 100644 index 00000000000..d7f2d1dcf19 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in @@ -0,0 +1,11 @@ +Device { + Name = FileStorage + Media Type = File + Archive Device = @archivedir@ + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + Description = "File device. A connecting Director must have the same Name and MediaType." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in new file mode 100644 index 00000000000..deef3360c2d --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in @@ -0,0 +1,5 @@ +Director { + Name = bareos-dir + Password = "@sd_password@" + Description = "Director, who is permitted to contact this storage daemon." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in new file mode 100644 index 00000000000..e3cfdee6315 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in @@ -0,0 +1,6 @@ +Director { + Name = bareos-mon + Password = "@mon_sd_password@" + Monitor = yes + Description = "Restricted Director, used by tray-monitor to get the status of this storage daemon." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf new file mode 100644 index 00000000000..468348e62fc --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf @@ -0,0 +1,5 @@ +Messages { + Name = Standard + Director = bareos-dir = all + Description = "Send all messages to the Director." +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in new file mode 100644 index 00000000000..3e1723fa60c --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in @@ -0,0 +1,14 @@ +Storage { + Name = bareos-sd + Maximum Concurrent Jobs = 20 + + # remove comment from "Plugin Directory" to load plugins from specified directory. + # if "Plugin Names" is defined, only the specified plugins will be loaded, + # otherwise all storage plugins (*-sd.so) from the "Plugin Directory". + # + # Plugin Directory = "@python_plugin_module_src_sd@" + # Plugin Names = "" + Working Directory = "@working_dir@" + Pid Directory = "@piddir@" + SD Port = @sd_port@ +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in new file mode 100644 index 00000000000..ecb6ad00dce --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in @@ -0,0 +1,10 @@ +# +# Bareos User Agent (or Console) Configuration File +# + +Director { + Name = @basename@-dir + DIRport = @dir_port@ + address = @hostname@ + Password = "@dir_password@" +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in new file mode 100644 index 00000000000..dd00dd9f103 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in @@ -0,0 +1,5 @@ +Client { + Name = @basename@-fd + Address = localhost + Password = "@mon_fd_password@" # password for FileDaemon +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in new file mode 100644 index 00000000000..55dae492250 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in @@ -0,0 +1,4 @@ +Director { + Name = bareos-dir + Address = localhost +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in new file mode 100644 index 00000000000..cddfa253945 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in @@ -0,0 +1,7 @@ +Monitor { + # Name to establish connections to Director Console, Storage Daemon and File Daemon. + Name = bareos-mon + # Password to access the Director + Password = "@mon_dir_password@" # password for the Directors + RefreshInterval = 30 seconds +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in new file mode 100644 index 00000000000..6538f4fd248 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in @@ -0,0 +1,5 @@ +Storage { + Name = bareos-sd + Address = localhost + Password = "@mon_sd_password@" # password for StorageDaemon +} diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona-plugin-test/testrunner new file mode 100755 index 00000000000..d09801e2c1b --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/testrunner @@ -0,0 +1,68 @@ +#!/bin/sh +# +# This systemtest tests the oVirt plugin functionality +# of the Bareos FD by using the supplied module +# BareosFdPluginOvirt.py +# +TestName="$(basename "$(pwd)")" +export TestName + +JobName=backup-bareos-fd +. ./environment +. ${scripts}/functions + +${scripts}/cleanup +${scripts}/setup + +curl -k -o ${BAREOS_CONFIG_DIR}/ovirt-ca.cert "https://$OVIRT_SERVER/ovirt-engine/services/pki-resource?resource=ca-certificate&format=X509-PEM-CA" + +# Directory to backup. +# This directory will be created by setup_data(). +# BackupDirectory="${tmp}/data" + +# Use a tgz to setup data to be backed up. +# Data will be placed at "${tmp}/data/". +# setup_data + +start_test + +cat <$tmp/bconcmds +@$out /dev/null +messages +@$out $tmp/log1.out +setdebug level=100 storage=File +label volume=TestVolume001 storage=File pool=Full +run job=$JobName level=Full yes +status director +status client +status storage=File +wait +messages +@# +@# now do a restore +@# +@$out $tmp/log2.out +wait +restore client=bareos-fd fileset=OvirtTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done +yes +wait +messages +quit +END_OF_DATA + +run_bareos +check_for_zombie_jobs storage=File +stop_bareos + +check_two_logs + +# Check restored disk image content +restored_disk_image=$(find $tmp/bareos-restores -type f | grep -v "\.ovf" | tail -1) +if [ $(strings $restored_disk_image | tail -1) == "BareosTest" ]; then + echo "Restored disk image content OK" +else + echo "Restored disk image ERROR: Does not contain expected string" + dstat=1 +fi + +end_test From f2e2f51ee1e275eb09aad5d43bc5349959ae4ddc Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 11:01:44 +0100 Subject: [PATCH 05/25] add python-modules subdirectory --- .../etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in | 2 +- .../etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in | 2 +- .../python-fd-percona-plugin-test/python-modules/README | 1 + systemtests/tests/python-fd-percona-plugin-test/testrunner | 6 ++---- 4 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 systemtests/tests/python-fd-percona-plugin-test/python-modules/README diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in index c9a39fc57b4..dccd4b86ed9 100644 --- a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in @@ -1,5 +1,5 @@ FileSet { - Name = "perconaTest" + Name = "PerconaTest" Description = "Test the Plugin functionality of the oVirt Plugin." Include { Options { diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in index 137d4d51b92..db84a37b10c 100644 --- a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in @@ -2,5 +2,5 @@ Job { Name = "backup-bareos-fd" JobDefs = "DefaultJob" Client = "bareos-fd" - FileSet = "perconaTest" + FileSet = "PerconaTest" } diff --git a/systemtests/tests/python-fd-percona-plugin-test/python-modules/README b/systemtests/tests/python-fd-percona-plugin-test/python-modules/README new file mode 100644 index 00000000000..5c0cd6debe4 --- /dev/null +++ b/systemtests/tests/python-fd-percona-plugin-test/python-modules/README @@ -0,0 +1 @@ +If this directory exists, the python modules will be linked here diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona-plugin-test/testrunner index d09801e2c1b..15bd742cc32 100755 --- a/systemtests/tests/python-fd-percona-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona-plugin-test/testrunner @@ -2,7 +2,7 @@ # # This systemtest tests the oVirt plugin functionality # of the Bareos FD by using the supplied module -# BareosFdPluginOvirt.py +# BareosFdPluginPercona.py # TestName="$(basename "$(pwd)")" export TestName @@ -14,8 +14,6 @@ JobName=backup-bareos-fd ${scripts}/cleanup ${scripts}/setup -curl -k -o ${BAREOS_CONFIG_DIR}/ovirt-ca.cert "https://$OVIRT_SERVER/ovirt-engine/services/pki-resource?resource=ca-certificate&format=X509-PEM-CA" - # Directory to backup. # This directory will be created by setup_data(). # BackupDirectory="${tmp}/data" @@ -43,7 +41,7 @@ messages @# @$out $tmp/log2.out wait -restore client=bareos-fd fileset=OvirtTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done +restore client=bareos-fd fileset=PerconaTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done yes wait messages From 34f862b4e4dffccfd6f3c5a1846e3674a3d48a49 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 11:12:06 +0100 Subject: [PATCH 06/25] rename BareosFdPercona.py to BareosFdPluginPercona.py --- ...sFdPercona.py => BareosFdPluginPercona.py} | 0 core/src/plugins/filed/bareos-fd-percona.py | 2 +- .../bareos-dir.d/fileset/PerconaTest.conf.in | 4 +- .../python-fd-percona-plugin-test/testrunner | 98 ++++++++++++------- 4 files changed, 65 insertions(+), 39 deletions(-) rename core/src/plugins/filed/{BareosFdPercona.py => BareosFdPluginPercona.py} (100%) diff --git a/core/src/plugins/filed/BareosFdPercona.py b/core/src/plugins/filed/BareosFdPluginPercona.py similarity index 100% rename from core/src/plugins/filed/BareosFdPercona.py rename to core/src/plugins/filed/BareosFdPluginPercona.py diff --git a/core/src/plugins/filed/bareos-fd-percona.py b/core/src/plugins/filed/bareos-fd-percona.py index 738aa12976d..40dc7bd271c 100644 --- a/core/src/plugins/filed/bareos-fd-percona.py +++ b/core/src/plugins/filed/bareos-fd-percona.py @@ -13,7 +13,7 @@ from BareosFdWrapper import * # This module contains the used plugin class -from BareosFdPercona import * +from BareosFdPluginPercona import * def load_bareos_plugin(context, plugindef): ''' diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in index dccd4b86ed9..71d41f37201 100644 --- a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in +++ b/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in @@ -1,10 +1,10 @@ FileSet { Name = "PerconaTest" - Description = "Test the Plugin functionality of the oVirt Plugin." + Description = "Test the Plugin functionality of the Percona Plugin." Include { Options { signature = MD5 } - Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona:ca=@current_test_directory@/etc/bareos/percona-ca.cert:server=@percona_server@:username=@percona_user@:password=@percona_password@:vm_name=@percona_vm_name@:include_disk_aliases=@percona_include_disk_alias@" + Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona" } } diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona-plugin-test/testrunner index 15bd742cc32..962a6316e79 100755 --- a/systemtests/tests/python-fd-percona-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona-plugin-test/testrunner @@ -1,6 +1,6 @@ #!/bin/sh # -# This systemtest tests the oVirt plugin functionality +# This systemtest tests the Percona plugin functionality # of the Bareos FD by using the supplied module # BareosFdPluginPercona.py # @@ -14,53 +14,79 @@ JobName=backup-bareos-fd ${scripts}/cleanup ${scripts}/setup -# Directory to backup. -# This directory will be created by setup_data(). -# BackupDirectory="${tmp}/data" +BCONSOLE="${current_test_directory}/bin/bconsole" -# Use a tgz to setup data to be backed up. -# Data will be placed at "${tmp}/data/". -# setup_data start_test +echo "------ running PERCONA tests" +echo "drop database ${db_name}_percona" | mysql +echo "create database ${db_name}_percona" | mysql +echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | mysql ${db_name}_percona +echo "insert into test (data) VALUES ('test eintrag 1') " | mysql ${db_name}_percona + + + +#cat <$tmp/bconcmds +#@$out /dev/null +#messages +#@$out $tmp/log1.out +#setdebug level=100 storage=File +#label volume=TestVolume001 storage=File pool=Full +#run job=$JobName level=Full yes +#status director +#status client +#status storage=File +#wait +#messages +#@# +#@# now do a restore +#@# +#@$out $tmp/log2.out +#wait +#restore client=bareos-fd fileset=PerconaTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done +#yes +#wait +#messages +#quit +#END_OF_DATA + + cat <$tmp/bconcmds -@$out /dev/null -messages @$out $tmp/log1.out -setdebug level=100 storage=File -label volume=TestVolume001 storage=File pool=Full -run job=$JobName level=Full yes -status director -status client -status storage=File -wait -messages -@# -@# now do a restore -@# -@$out $tmp/log2.out -wait -restore client=bareos-fd fileset=PerconaTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done -yes -wait -messages -quit +run job=$JobName +wait JobName=$JobName +status dir END_OF_DATA +run_bareos + +echo "insert into test (data) VALUES ('test eintrag 2') " | mysql ${db_name}_percona +cat <$tmp/bconcmds +@$out $tmp/log1.out +run job=$JobName +wait JobName=$JobName +status dir +END_OF_DATA run_bareos + +cat <$tmp/bconcmds +@$out $tmp/log2.out +RESTORECMD="restore client=bareos-fd fileset=PerconaTest yes restorejob=RestoreFile select all\rdone\r" +wait JobName=RestoreFile +END_OF_DATA + +# Check if xtrabackup has extracted some files at least +# TODO: verify that xtrabackup --prepare works and eventually do complete datbase restore +ls -lR $tmp/bareos-restores/_percona/ +if [ -z "$(ls -A $tmp/bareos-restores/_percona/)" ]; then + echo "No restore data found" + estat=1 +fi + check_for_zombie_jobs storage=File stop_bareos check_two_logs -# Check restored disk image content -restored_disk_image=$(find $tmp/bareos-restores -type f | grep -v "\.ovf" | tail -1) -if [ $(strings $restored_disk_image | tail -1) == "BareosTest" ]; then - echo "Restored disk image content OK" -else - echo "Restored disk image ERROR: Does not contain expected string" - dstat=1 -fi - end_test From 2edd5ba309d79e133f665a66aa71d9063f03e566 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 12:44:29 +0100 Subject: [PATCH 07/25] python-fd-percona-plugin-test/testrunner: use run_bareos --- .../python-fd-percona-plugin-test/testrunner | 88 +++++++++---------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona-plugin-test/testrunner index 962a6316e79..f0b83ade736 100755 --- a/systemtests/tests/python-fd-percona-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona-plugin-test/testrunner @@ -14,9 +14,6 @@ JobName=backup-bareos-fd ${scripts}/cleanup ${scripts}/setup -BCONSOLE="${current_test_directory}/bin/bconsole" - - start_test echo "------ running PERCONA tests" @@ -26,57 +23,52 @@ echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCH echo "insert into test (data) VALUES ('test eintrag 1') " | mysql ${db_name}_percona +BCONSOLE="${current_test_directory}/bin/bconsole" -#cat <$tmp/bconcmds -#@$out /dev/null -#messages -#@$out $tmp/log1.out -#setdebug level=100 storage=File -#label volume=TestVolume001 storage=File pool=Full -#run job=$JobName level=Full yes -#status director -#status client -#status storage=File -#wait -#messages -#@# -#@# now do a restore -#@# -#@$out $tmp/log2.out -#wait -#restore client=bareos-fd fileset=PerconaTest where=$tmp/bareos-restores pluginoptions=python:local=yes select all done -#yes -#wait -#messages -#quit -#END_OF_DATA - - -cat <$tmp/bconcmds -@$out $tmp/log1.out -run job=$JobName -wait JobName=$JobName -status dir -END_OF_DATA run_bareos +echo "@$out $tmp/log1.out" | $BCONSOLE + +COMMAND="run job=$JobName" +echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" +#sleep 5 +echo -e "wait JobName=$JobName" | $BCONSOLE +#sleep 2 +echo "status dir" | $BCONSOLE +# insert data and run incremental echo "insert into test (data) VALUES ('test eintrag 2') " | mysql ${db_name}_percona +touch /root/testfile + +COMMAND="$COMMAND level=Incremental" +echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" +#sleep 5 +echo -e "wait JobName=$JobName" | $BCONSOLE +#sleep 2 +echo "status dir" | $BCONSOLE + + +# run incremental again without any new data +echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" +#sleep 5 +echo -e "wait JobName=$JobName" | $BCONSOLE +#sleep 2 +echo "status dir" | $BCONSOLE -cat <$tmp/bconcmds -@$out $tmp/log1.out -run job=$JobName -wait JobName=$JobName -status dir -END_OF_DATA -run_bareos -cat <$tmp/bconcmds -@$out $tmp/log2.out +# run restore RESTORECMD="restore client=bareos-fd fileset=PerconaTest yes restorejob=RestoreFile select all\rdone\r" -wait JobName=RestoreFile -END_OF_DATA -# Check if xtrabackup has extracted some files at least +echo "@$out $tmp/log2.out" | $BCONSOLE + +JOBID=`echo -e "$RESTORECMD" | $BCONSOLE | grep "Job queued. JobId=" | sed s'/.*=//' ` + +echo -e "wait jobid=$JOBID" | $BCONSOLE | grep -q "JobStatus=OK" +if [[ $? != 0 ]]; then + echo "Restore Job $JOBID failed" + estat=1 +fi + +# Check, if xtrabackup has extracted some files at least # TODO: verify that xtrabackup --prepare works and eventually do complete datbase restore ls -lR $tmp/bareos-restores/_percona/ if [ -z "$(ls -A $tmp/bareos-restores/_percona/)" ]; then @@ -84,6 +76,10 @@ if [ -z "$(ls -A $tmp/bareos-restores/_percona/)" ]; then estat=1 fi + + + + check_for_zombie_jobs storage=File stop_bareos From b3c613d31817aeff93df36add9a8e99309063a28 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 13:58:49 +0100 Subject: [PATCH 08/25] systemtests: only link python modules that are required in each test --- systemtests/CMakeLists.txt | 81 ++++++++++++++----- .../python-modules/README | 1 - 2 files changed, 59 insertions(+), 23 deletions(-) delete mode 100644 systemtests/tests/python-fd-percona-plugin-test/python-modules/README diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index ba2d7f48458..c45b088f56f 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -83,37 +83,74 @@ macro(CheckForEnabledAndDisabledListEntry TEST_NAME_TO_CHECK) endif() endmacro() -macro(handle_python_plugin_modules) - if(EXISTS ${python_plugin_module_src_test_dir}) +macro(handle_python_plugin_modules test_name) set(PYMODULES_TO_LINK_TO_SRC + ) - filed/BareosFdWrapper.py - filed/BareosFdPluginBaseclass.py - filed/bareos_fd_consts.py - filed/bareos-fd-mock-test.py + message("test_name: ${test_name}") + string(REGEX MATCH ^python-fd.* starts_with_python-fd ${test_name}) + if(starts_with_python-fd) + list(APPEND PYMODULES_TO_LINK_TO_SRC + filed/BareosFdWrapper.py + filed/BareosFdPluginBaseclass.py + filed/bareos_fd_consts.py + filed/bareos-fd-mock-test.py + ) + endif() - filed/bareos-fd-local-fileset.py - filed/BareosFdPluginLocalFileset.py + string(REGEX MATCH ^python-sd.* starts_with_python-sd ${test_name}) + if(starts_with_python-sd) + list(APPEND PYMODULES_TO_LINK_TO_SRC + stored/bareos_sd_consts.py + stored/bareos-sd-class-plugin.py + stored/BareosSdPluginBaseclass.py + stored/BareosSdWrapper.py + ) + endif() - filed/BareosFdPluginLDAP.py - filed/bareos-fd-ldap.py + string(REGEX MATCH ^python-dir.* starts_with_python-dir ${test_name}) + if(starts_with_python-dir) + list(APPEND PYMODULES_TO_LINK_TO_SRC + dird/bareos_dir_consts.py + dird/BareosDirPluginBaseclass.py + dird/bareos-dir-class-plugin.py + dird/BareosDirWrapper.py + ) + endif() - filed/BareosFdPluginOvirt.py - filed/bareos-fd-ovirt.py + if(${test_name} STREQUAL python-dir-plugin-test) + list(APPEND PYMODULES_TO_LINK_TO_SRC + ) + endif() + if(${test_name} STREQUAL python-fd-percona-plugin-test) + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginPercona.py filed/bareos-fd-percona.py + ) + endif() - dird/bareos_dir_consts.py - dird/BareosDirPluginBaseclass.py - dird/bareos-dir-class-plugin.py - dird/BareosDirWrapper.py + if(${test_name} STREQUAL python-fd-ovirt-plugin-test) + list(APPEND PYMODULES_TO_LINK_TO_SRC + filed/BareosFdPluginOvirt.py + filed/bareos-fd-ovirt.py + ) + endif() - stored/bareos_sd_consts.py - stored/bareos-sd-class-plugin.py - stored/BareosSdPluginBaseclass.py - stored/BareosSdWrapper.py + if(${test_name} STREQUAL python-fd-plugin-local-fileset-test) + list(APPEND PYMODULES_TO_LINK_TO_SRC + filed/bareos-fd-local-fileset.py + filed/BareosFdPluginLocalFileset.py ) + endif() + +# still missing: +# filed/BareosFdPluginLDAP.py +# filed/bareos-fd-ldap.py + + if(NOT EXISTS ${python_plugin_module_src_test_dir}) + file(MAKE_DIRECTORY ${python_plugin_module_src_test_dir}) + endif() foreach(PYMODULE_SOURCEPATH ${PYMODULES_TO_LINK_TO_SRC}) get_filename_component(PYMODULE_NAME ${PYMODULE_SOURCEPATH} NAME) @@ -123,7 +160,7 @@ macro(handle_python_plugin_modules) ${python_plugin_module_src_test_dir}/${PYMODULE_NAME} ) endforeach() - endif() +# endif() endmacro() macro(prepare_test) @@ -442,7 +479,7 @@ foreach(TEST_NAME ${SYSTEM_TESTS}) prepare_test() configurefilestosystemtest("systemtests" "tests/${TEST_NAME}" "*" @ONLY "") - handle_python_plugin_modules() + handle_python_plugin_modules(${TEST_NAME}) configure_file("environment.in" "tests/${TEST_NAME}/environment" @ONLY) # create a bin/bareos and bin/bconsole script in every testdir for start/stop diff --git a/systemtests/tests/python-fd-percona-plugin-test/python-modules/README b/systemtests/tests/python-fd-percona-plugin-test/python-modules/README deleted file mode 100644 index 5c0cd6debe4..00000000000 --- a/systemtests/tests/python-fd-percona-plugin-test/python-modules/README +++ /dev/null @@ -1 +0,0 @@ -If this directory exists, the python modules will be linked here From 87e78c827692308ef3d7ab07e98f69a326930dc4 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 14:03:20 +0100 Subject: [PATCH 09/25] systemtests: apply cmake-format --- systemtests/CMakeLists.txt | 77 ++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 44 deletions(-) diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index c45b088f56f..7c89e47e5d8 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -84,83 +84,70 @@ macro(CheckForEnabledAndDisabledListEntry TEST_NAME_TO_CHECK) endmacro() macro(handle_python_plugin_modules test_name) - set(PYMODULES_TO_LINK_TO_SRC - ) + set(PYMODULES_TO_LINK_TO_SRC) message("test_name: ${test_name}") string(REGEX MATCH ^python-fd.* starts_with_python-fd ${test_name}) if(starts_with_python-fd) - list(APPEND PYMODULES_TO_LINK_TO_SRC - filed/BareosFdWrapper.py - filed/BareosFdPluginBaseclass.py - filed/bareos_fd_consts.py - filed/bareos-fd-mock-test.py + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdWrapper.py + filed/BareosFdPluginBaseclass.py filed/bareos_fd_consts.py + filed/bareos-fd-mock-test.py ) endif() string(REGEX MATCH ^python-sd.* starts_with_python-sd ${test_name}) if(starts_with_python-sd) - list(APPEND PYMODULES_TO_LINK_TO_SRC - stored/bareos_sd_consts.py - stored/bareos-sd-class-plugin.py - stored/BareosSdPluginBaseclass.py - stored/BareosSdWrapper.py + list(APPEND PYMODULES_TO_LINK_TO_SRC stored/bareos_sd_consts.py + stored/bareos-sd-class-plugin.py + stored/BareosSdPluginBaseclass.py stored/BareosSdWrapper.py ) endif() string(REGEX MATCH ^python-dir.* starts_with_python-dir ${test_name}) if(starts_with_python-dir) - list(APPEND PYMODULES_TO_LINK_TO_SRC - dird/bareos_dir_consts.py - dird/BareosDirPluginBaseclass.py - dird/bareos-dir-class-plugin.py - dird/BareosDirWrapper.py + list(APPEND PYMODULES_TO_LINK_TO_SRC dird/bareos_dir_consts.py + dird/BareosDirPluginBaseclass.py + dird/bareos-dir-class-plugin.py dird/BareosDirWrapper.py ) endif() if(${test_name} STREQUAL python-dir-plugin-test) - list(APPEND PYMODULES_TO_LINK_TO_SRC - ) + list(APPEND PYMODULES_TO_LINK_TO_SRC) endif() if(${test_name} STREQUAL python-fd-percona-plugin-test) - list(APPEND PYMODULES_TO_LINK_TO_SRC - filed/BareosFdPluginPercona.py - filed/bareos-fd-percona.py + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginPercona.py + filed/bareos-fd-percona.py ) endif() if(${test_name} STREQUAL python-fd-ovirt-plugin-test) - list(APPEND PYMODULES_TO_LINK_TO_SRC - filed/BareosFdPluginOvirt.py - filed/bareos-fd-ovirt.py + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginOvirt.py + filed/bareos-fd-ovirt.py ) endif() if(${test_name} STREQUAL python-fd-plugin-local-fileset-test) - list(APPEND PYMODULES_TO_LINK_TO_SRC - filed/bareos-fd-local-fileset.py - filed/BareosFdPluginLocalFileset.py + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/bareos-fd-local-fileset.py + filed/BareosFdPluginLocalFileset.py ) endif() -# still missing: -# filed/BareosFdPluginLDAP.py -# filed/bareos-fd-ldap.py + # still missing: filed/BareosFdPluginLDAP.py filed/bareos-fd-ldap.py - if(NOT EXISTS ${python_plugin_module_src_test_dir}) - file(MAKE_DIRECTORY ${python_plugin_module_src_test_dir}) - endif() + if(NOT EXISTS ${python_plugin_module_src_test_dir}) + file(MAKE_DIRECTORY ${python_plugin_module_src_test_dir}) + endif() - foreach(PYMODULE_SOURCEPATH ${PYMODULES_TO_LINK_TO_SRC}) - get_filename_component(PYMODULE_NAME ${PYMODULE_SOURCEPATH} NAME) - execute_process( - COMMAND ${CMAKE_COMMAND} -E create_symlink - ${PROJECT_SOURCE_DIR}/../core/src/plugins/${PYMODULE_SOURCEPATH} - ${python_plugin_module_src_test_dir}/${PYMODULE_NAME} - ) - endforeach() -# endif() + foreach(PYMODULE_SOURCEPATH ${PYMODULES_TO_LINK_TO_SRC}) + get_filename_component(PYMODULE_NAME ${PYMODULE_SOURCEPATH} NAME) + execute_process( + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${PROJECT_SOURCE_DIR}/../core/src/plugins/${PYMODULE_SOURCEPATH} + ${python_plugin_module_src_test_dir}/${PYMODULE_NAME} + ) + endforeach() + # endif() endmacro() macro(prepare_test) @@ -193,7 +180,9 @@ macro(prepare_test) # set(DEFAULT_DB_TYPE ) set(archivedir ${current_test_directory}/storage) set(confdir ${current_test_directory}/etc/bareos) - set(config_directory_dir_additional_test_config ${current_test_directory}/etc/bareos/bareos-dir.d/additional_test_config) + set(config_directory_dir_additional_test_config + ${current_test_directory}/etc/bareos/bareos-dir.d/additional_test_config + ) set(logdir ${current_test_directory}/log) set(tmpdir ${current_test_directory}/tmp) From fc9423584297f6f01a1eca831d9f079490ef9091 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 14:31:01 +0100 Subject: [PATCH 10/25] systemtests: cleanup CMakeLists and percona testrunner --- systemtests/CMakeLists.txt | 11 +++++------ .../tests/python-fd-percona-plugin-test/testrunner | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index 7c89e47e5d8..153999c3ee4 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -1,6 +1,6 @@ # BAREOS® - Backup Archiving REcovery Open Sourced # -# Copyright (C) 2019-2019 Bareos GmbH & Co. KG +# Copyright (C) 2019-2020 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public @@ -45,10 +45,10 @@ function(ConfigureFilesToSystemtest srcbasedir dirname globexpression ${configure_option} ) endforeach() - message( - STATUS - "Configured ${COUNT} files from ${srcbasedir}/${srcdirname} to ${dirname} with glob ${globexpression} ${configure_option}" - ) + # message( + # STATUS + # "Configured ${COUNT} files from ${srcbasedir}/${srcdirname} to ${dirname} with glob ${globexpression} ${configure_option}" + #) endfunction() # generic function to probe for a python module @@ -86,7 +86,6 @@ endmacro() macro(handle_python_plugin_modules test_name) set(PYMODULES_TO_LINK_TO_SRC) - message("test_name: ${test_name}") string(REGEX MATCH ^python-fd.* starts_with_python-fd ${test_name}) if(starts_with_python-fd) list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdWrapper.py diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona-plugin-test/testrunner index f0b83ade736..8841b2c3066 100755 --- a/systemtests/tests/python-fd-percona-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona-plugin-test/testrunner @@ -37,7 +37,6 @@ echo -e "wait JobName=$JobName" | $BCONSOLE echo "status dir" | $BCONSOLE # insert data and run incremental echo "insert into test (data) VALUES ('test eintrag 2') " | mysql ${db_name}_percona -touch /root/testfile COMMAND="$COMMAND level=Incremental" echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" From a399711aa057f79921dfaadae1fb3b5cc0c36dec Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 14:43:23 +0100 Subject: [PATCH 11/25] Percona plugin: apply black format to python files --- .../plugins/filed/BareosFdPluginPercona.py | 441 +++++++++++------- core/src/plugins/filed/bareos-fd-percona.py | 10 +- 2 files changed, 287 insertions(+), 164 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPercona.py b/core/src/plugins/filed/BareosFdPluginPercona.py index 002f66a7d07..e7a99e9083e 100644 --- a/core/src/plugins/filed/BareosFdPluginPercona.py +++ b/core/src/plugins/filed/BareosFdPluginPercona.py @@ -34,66 +34,69 @@ import json -class BareosFdPercona (BareosFdPluginBaseclass): - ''' +class BareosFdPercona(BareosFdPluginBaseclass): + """ Plugin for backing up all mysql innodb databases found in a specific mysql server using the Percona xtrabackup tool. - ''' + """ def __init__(self, context, plugindef): # BareosFdPluginBaseclass.__init__(self, context, plugindef) super(BareosFdPercona, self).__init__(context, plugindef) # we first create and backup the stream and after that # the lsn file as restore-object - self.files_to_backup = ['lsnfile', 'stream'] + self.files_to_backup = ["lsnfile", "stream"] self.tempdir = tempfile.mkdtemp() - #self.logdir = GetValue(context, bVariable['bVarWorkingDir']) - self.logdir = '/var/log/bareos/' - self.log = 'bareos-plugin-percona.log' + # self.logdir = GetValue(context, bVariable['bVarWorkingDir']) + self.logdir = "/var/log/bareos/" + self.log = "bareos-plugin-percona.log" self.rop_data = {} self.max_to_lsn = 0 self.err_fd = None def parse_plugin_definition(self, context, plugindef): - ''' + """ We have default options that should work out of the box in the most use cases that the mysql/mariadb is on the same host and can be accessed without user/password information, e.g. with a valid my.cnf for user root. - ''' + """ BareosFdPluginBaseclass.parse_plugin_definition(self, context, plugindef) - if 'dumpbinary' in self.options: - self.dumpbinary = self.options['dumpbinary'] + if "dumpbinary" in self.options: + self.dumpbinary = self.options["dumpbinary"] else: self.dumpbinary = "xtrabackup" - if 'restorecommand' not in self.options: + if "restorecommand" not in self.options: self.restorecommand = "xbstream -x -C " else: - self.restorecommand = self.options['restorecommand'] + self.restorecommand = self.options["restorecommand"] # Default is not to write an extra logfile - if 'log' not in self.options: + if "log" not in self.options: self.log = False - elif self.options['log'] == 'false': + elif self.options["log"] == "false": self.log = False - elif os.path.isabs(self.options['log']): - self.log = self.options['log'] + elif os.path.isabs(self.options["log"]): + self.log = self.options["log"] else: - self.log = os.path.join(self.logdir, self.options['log']) + self.log = os.path.join(self.logdir, self.options["log"]) # By default, standard mysql-config files will be used, set # this option to use extra files - self.connect_options = { 'read_default_group': 'client' } - if 'mycnf' in self.options: - self.connect_options['read_default_file'] = self.options['mycnf'] - self.mycnf = "--defaults-extra-file=%s " % self.options['mycnf'] + self.connect_options = {"read_default_group": "client"} + if "mycnf" in self.options: + self.connect_options["read_default_file"] = self.options["mycnf"] + self.mycnf = "--defaults-extra-file=%s " % self.options["mycnf"] else: self.mycnf = "" # If true, incremental jobs will only be performed, if LSN has increased # since last call. - if 'strictIncremental' in self.options and self.options['strictIncremental'] == 'true': + if ( + "strictIncremental" in self.options + and self.options["strictIncremental"] == "true" + ): self.strictIncremental = True else: self.strictIncremental = False @@ -101,144 +104,196 @@ def parse_plugin_definition(self, context, plugindef): self.dumpoptions = self.mycnf # if dumpoptions is set, we use that here, otherwise defaults - if 'dumpoptions' in self.options: - self.dumpoptions += self.options['dumpoptions'] + if "dumpoptions" in self.options: + self.dumpoptions += self.options["dumpoptions"] else: self.dumpoptions += " --backup --stream=xbstream" self.dumpoptions += " --extra-lsndir=%s" % self.tempdir - if 'extradumpoptions' in self.options: - self.dumpoptions += " " + self.options['extradumpoptions'] + if "extradumpoptions" in self.options: + self.dumpoptions += " " + self.options["extradumpoptions"] # We need to call mysql to get the current Log Sequece Number (LSN) - if 'mysqlcmd' in self.options: - self.mysqlcmd = self.options['mysqlcmd'] + if "mysqlcmd" in self.options: + self.mysqlcmd = self.options["mysqlcmd"] else: self.mysqlcmd = "mysql %s -r" % self.mycnf - return bRCs['bRC_OK'] + return bRCs["bRC_OK"] def check_plugin_options(self, context, mandatory_options=None): - accurate_enabled = GetValue(context, bVariable['bVarAccurate']) + accurate_enabled = GetValue(context, bVariable["bVarAccurate"]) if accurate_enabled is not None and accurate_enabled != 0: - JobMessage(context, bJobMessageType['M_FATAL'], - "start_backup_job: Accurate backup not allowed please disable in Job\n") - return bRCs['bRC_Error'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + "start_backup_job: Accurate backup not allowed please disable in Job\n", + ) + return bRCs["bRC_Error"] else: - return bRCs['bRC_OK'] + return bRCs["bRC_OK"] def create_file(self, context, restorepkt): - ''' + """ On restore we create a subdirectory for the first base backup and each incremental backup. Because percona expects an empty directory, we create a tree starting with jobId/ of restore job - ''' + """ FNAME = restorepkt.ofname DebugMessage(context, 100, "create file with %s called\n" % FNAME) self.writeDir = "%s/%d/" % (os.path.dirname(FNAME), self.jobId) # FNAME contains originating jobId after last . - origJobId = int(FNAME.rpartition('.')[-1]) + origJobId = int(FNAME.rpartition(".")[-1]) if origJobId in self.rop_data: - rop_from_lsn = int(self.rop_data[origJobId]['from_lsn']) - rop_to_lsn = int(self.rop_data[origJobId]['to_lsn']) + rop_from_lsn = int(self.rop_data[origJobId]["from_lsn"]) + rop_to_lsn = int(self.rop_data[origJobId]["to_lsn"]) self.writeDir += "/%020d_" % rop_from_lsn self.writeDir += "%020d_" % rop_to_lsn self.writeDir += "%010d" % origJobId else: - JobMessage(context, bJobMessageType['M_ERROR'], - "No lsn information found in restore object for file %s from job %d\n" % (FNAME, origJobId)) + JobMessage( + context, + bJobMessageType["M_ERROR"], + "No lsn information found in restore object for file %s from job %d\n" + % (FNAME, origJobId), + ) # Create restore directory, if not existant if not os.path.exists(self.writeDir): bareosfd.DebugMessage( - context, 200, - "Directory %s does not exist, creating it now\n" % - self.writeDir) + context, + 200, + "Directory %s does not exist, creating it now\n" % self.writeDir, + ) os.makedirs(self.writeDir) # Percona requires empty directory if os.listdir(self.writeDir): - JobMessage(context, bJobMessageType['M_FATAL'], - "Restore with xbstream needs empty directoy: %s\n" % self.writeDir) - return bRCs['bRC_Error'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + "Restore with xbstream needs empty directoy: %s\n" % self.writeDir, + ) + return bRCs["bRC_Error"] self.restorecommand += self.writeDir - DebugMessage(context, 100, "Restore using xbstream to extract files with \"%s\"\n" % self.restorecommand) - restorepkt.create_status = bCFs['CF_EXTRACT'] - return bRCs['bRC_OK'] + DebugMessage( + context, + 100, + 'Restore using xbstream to extract files with "%s"\n' % self.restorecommand, + ) + restorepkt.create_status = bCFs["CF_EXTRACT"] + return bRCs["bRC_OK"] def start_backup_job(self, context): - ''' + """ We will check, if database has changed since last backup in the incremental case - ''' + """ check_option_bRC = self.check_plugin_options(context) - if check_option_bRC != bRCs['bRC_OK']: + if check_option_bRC != bRCs["bRC_OK"]: return check_option_bRC bareosfd.DebugMessage( - context, 100, - "start_backup_job, level: %s\n" % chr(self.level)) - if chr(self.level) == 'I': + context, 100, "start_backup_job, level: %s\n" % chr(self.level) + ) + if chr(self.level) == "I": # We check, if we have a LSN received by restore object from previous job if self.max_to_lsn == 0: - JobMessage(context, bJobMessageType['M_FATAL'], "No LSN received to be used with incremental backup\n") - return bRCs['bRC_Error'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + "No LSN received to be used with incremental backup\n", + ) + return bRCs["bRC_Error"] # Try to load MySQLdb module hasMySQLdbModule = False try: import MySQLdb + hasMySQLdbModule = True bareosfd.DebugMessage(context, 100, "Imported module MySQLdb\n") except ImportError: - bareosfd.DebugMessage(context, 100, "Import of module MySQLdb failed. Using command pipe instead\n") + bareosfd.DebugMessage( + context, + 100, + "Import of module MySQLdb failed. Using command pipe instead\n", + ) # contributed by https://github.com/kjetilho if hasMySQLdbModule: try: conn = MySQLdb.connect(**self.connect_options) cursor = conn.cursor() - cursor.execute('SHOW ENGINE INNODB STATUS') + cursor.execute("SHOW ENGINE INNODB STATUS") result = cursor.fetchall() if len(result) == 0: - JobMessage(context, bJobMessageType['M_FATAL'], "Could not fetch SHOW ENGINE INNODB STATUS, unpriveleged user?") - return bRCs['bRC_Error'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + "Could not fetch SHOW ENGINE INNODB STATUS, unpriveleged user?", + ) + return bRCs["bRC_Error"] info = result[0][2] conn.close() for line in info.split("\n"): - if line.startswith('Log sequence number'): - last_lsn = int(line.split(' ')[3]) - except Exception, e: - JobMessage(context, bJobMessageType['M_FATAL'], "Could not get LSN, Error: %s" % e) - return bRCs['bRC_Error'] + if line.startswith("Log sequence number"): + last_lsn = int(line.split(" ")[3]) + except Exception as e: + JobMessage( + context, + bJobMessageType["M_FATAL"], + "Could not get LSN, Error: %s" % e, + ) + return bRCs["bRC_Error"] # use old method as fallback, if module MySQLdb not available else: - get_lsn_command = ("echo 'SHOW ENGINE INNODB STATUS' | %s | grep 'Log sequence number' | cut -d ' ' -f 4" - % self.mysqlcmd) - last_lsn_proc = Popen(get_lsn_command, shell=True, stdout=PIPE, stderr=PIPE) + get_lsn_command = ( + "echo 'SHOW ENGINE INNODB STATUS' | %s | grep 'Log sequence number' | cut -d ' ' -f 4" + % self.mysqlcmd + ) + last_lsn_proc = Popen( + get_lsn_command, shell=True, stdout=PIPE, stderr=PIPE + ) last_lsn_proc.wait() returnCode = last_lsn_proc.poll() (mysqlStdOut, mysqlStdErr) = last_lsn_proc.communicate() if returnCode != 0 or mysqlStdErr: - JobMessage(context, bJobMessageType['M_FATAL'], "Could not get LSN with command \"%s\", Error: %s" - % (get_lsn_command, mysqlStdErr)) - return bRCs['bRC_Error'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + 'Could not get LSN with command "%s", Error: %s' + % (get_lsn_command, mysqlStdErr), + ) + return bRCs["bRC_Error"] else: try: last_lsn = int(mysqlStdOut) except: - JobMessage(context, bJobMessageType['M_FATAL'], "Error reading LSN: \"%s\" not an integer" % mysqlStdOut) - return bRCs['bRC_Error'] - JobMessage(context, bJobMessageType['M_INFO'], "Backup until LSN: %d\n" %last_lsn) - if self.max_to_lsn > 0 and self.max_to_lsn >= last_lsn and self.strictIncremental: + JobMessage( + context, + bJobMessageType["M_FATAL"], + 'Error reading LSN: "%s" not an integer' % mysqlStdOut, + ) + return bRCs["bRC_Error"] + JobMessage( + context, bJobMessageType["M_INFO"], "Backup until LSN: %d\n" % last_lsn + ) + if ( + self.max_to_lsn > 0 + and self.max_to_lsn >= last_lsn + and self.strictIncremental + ): bareosfd.DebugMessage( - context, 100, + context, + 100, "Last LSN of DB %d is not higher than LSN from previous job %d. Skipping this incremental backup\n" - % (last_lsn, self.max_to_lsn)) - self.files_to_backup = ['lsn_only'] - return bRCs['bRC_OK'] - return bRCs['bRC_OK'] + % (last_lsn, self.max_to_lsn), + ) + self.files_to_backup = ["lsn_only"] + return bRCs["bRC_OK"] + return bRCs["bRC_OK"] def start_backup_file(self, context, savepkt): - ''' + """ This method is called, when Bareos is ready to start backup a file - ''' + """ if not self.files_to_backup: self.file_to_backup = None DebugMessage(context, 100, "start_backup_file: None\n") @@ -249,15 +304,15 @@ def start_backup_file(self, context, savepkt): statp = StatPacket() savepkt.statp = statp - if self.file_to_backup == 'stream': + if self.file_to_backup == "stream": # This is the database backup as xbstream savepkt.fname = "/_percona/xbstream.%010d" % self.jobId - savepkt.type = bFileType['FT_REG'] + savepkt.type = bFileType["FT_REG"] if self.max_to_lsn > 0: self.dumpoptions += " --incremental-lsn=%d" % self.max_to_lsn - self.dumpcommand = ("%s %s" % (self.dumpbinary, self.dumpoptions)) + self.dumpcommand = "%s %s" % (self.dumpbinary, self.dumpoptions) DebugMessage(context, 100, "Dumper: '" + self.dumpcommand + "'\n") - elif self.file_to_backup == 'lsnfile': + elif self.file_to_backup == "lsnfile": # The restore object containing the log sequence number (lsn) # Read checkpoints and create restore object checkpoints = {} @@ -267,71 +322,94 @@ def start_backup_file(self, context, savepkt): key, value = line.partition("=")[::2] checkpoints[key.strip()] = value.strip() savepkt.fname = "/_percona/xtrabackup_checkpoints" - savepkt.type = bFileType['FT_RESTORE_FIRST'] + savepkt.type = bFileType["FT_RESTORE_FIRST"] savepkt.object_name = savepkt.fname savepkt.object = bytearray(json.dumps(checkpoints)) savepkt.object_len = len(savepkt.object) savepkt.object_index = int(time.time()) shutil.rmtree(self.tempdir) - elif self.file_to_backup == 'lsn_only': + elif self.file_to_backup == "lsn_only": # We have nothing to backup incremental, so we just have to pass # the restore object from previous job savepkt.fname = "/_percona/xtrabackup_checkpoints" - savepkt.type = bFileType['FT_RESTORE_FIRST'] + savepkt.type = bFileType["FT_RESTORE_FIRST"] savepkt.object_name = savepkt.fname savepkt.object = bytearray(self.row_rop_raw) savepkt.object_len = len(savepkt.object) savepkt.object_index = int(time.time()) else: # should not happen - JobMessage(context, bJobMessageType['M_FATAL'], "Unknown error. Don't know how to handle %s\n" - % self.file_to_backup) - - JobMessage(context, bJobMessageType['M_INFO'], "Starting backup of " + savepkt.fname + "\n") - return bRCs['bRC_OK'] + JobMessage( + context, + bJobMessageType["M_FATAL"], + "Unknown error. Don't know how to handle %s\n" % self.file_to_backup, + ) + + JobMessage( + context, + bJobMessageType["M_INFO"], + "Starting backup of " + savepkt.fname + "\n", + ) + return bRCs["bRC_OK"] def plugin_io(self, context, IOP): - ''' + """ Called for io operations. We read from pipe into buffers or on restore send to xbstream - ''' + """ DebugMessage(context, 200, "plugin_io called with " + str(IOP.func) + "\n") - if IOP.func == bIOPS['IO_OPEN']: + if IOP.func == bIOPS["IO_OPEN"]: DebugMessage(context, 100, "plugin_io called with IO_OPEN\n") if self.log: try: self.err_fd = open(self.log, "a") - except IOError, msg: - DebugMessage(context, 100, "Could not open log file (%s): %s\n" - % (self.log, format(str(msg)))) + except IOError as msg: + DebugMessage( + context, + 100, + "Could not open log file (%s): %s\n" + % (self.log, format(str(msg))), + ) if IOP.flags & (os.O_CREAT | os.O_WRONLY): if self.log: - self.err_fd.write("%s Restore Job %s opens stream with \"%s\"\n" %(datetime.datetime.now(),self.jobId,self.restorecommand)) - self.stream = Popen(self.restorecommand, shell=True, stdin=PIPE, stderr=self.err_fd) + self.err_fd.write( + '%s Restore Job %s opens stream with "%s"\n' + % (datetime.datetime.now(), self.jobId, self.restorecommand) + ) + self.stream = Popen( + self.restorecommand, shell=True, stdin=PIPE, stderr=self.err_fd + ) else: if self.log: - self.err_fd.write("%s Backup Job %s opens stream with \"%s\"\n" %(datetime.datetime.now(),self.jobId,self.dumpcommand)) - self.stream = Popen(self.dumpcommand, shell=True, stdout=PIPE, stderr=self.err_fd) - return bRCs['bRC_OK'] - - elif IOP.func == bIOPS['IO_READ']: + self.err_fd.write( + '%s Backup Job %s opens stream with "%s"\n' + % (datetime.datetime.now(), self.jobId, self.dumpcommand) + ) + self.stream = Popen( + self.dumpcommand, shell=True, stdout=PIPE, stderr=self.err_fd + ) + return bRCs["bRC_OK"] + + elif IOP.func == bIOPS["IO_READ"]: IOP.buf = bytearray(IOP.count) IOP.status = self.stream.stdout.readinto(IOP.buf) IOP.io_errno = 0 - return bRCs['bRC_OK'] + return bRCs["bRC_OK"] - elif IOP.func == bIOPS['IO_WRITE']: + elif IOP.func == bIOPS["IO_WRITE"]: try: self.stream.stdin.write(IOP.buf) IOP.status = IOP.count IOP.io_errno = 0 - except IOError, msg: + except IOError as msg: IOP.io_errno = -1 - DebugMessage(context, 100, "Error writing data: " + format(str(msg)) + "\n") - return bRCs['bRC_OK'] + DebugMessage( + context, 100, "Error writing data: " + format(str(msg)) + "\n" + ) + return bRCs["bRC_OK"] - elif IOP.func == bIOPS['IO_CLOSE']: + elif IOP.func == bIOPS["IO_CLOSE"]: DebugMessage(context, 100, "plugin_io called with IO_CLOSE\n") self.subprocess_returnCode = self.stream.poll() if self.subprocess_returnCode is None: @@ -340,92 +418,135 @@ def plugin_io(self, context, IOP): self.stream.communicate() self.subprocess_returnCode = self.stream.poll() except: - JobMessage(context, bJobMessageType['M_ERROR'], - "Dump / restore command not finished properly\n") - bRCs['bRC_Error'] - return bRCs['bRC_OK'] + JobMessage( + context, + bJobMessageType["M_ERROR"], + "Dump / restore command not finished properly\n", + ) + bRCs["bRC_Error"] + return bRCs["bRC_OK"] else: - DebugMessage(context, 100, "Subprocess has terminated with returncode: %d\n" - % self.subprocess_returnCode) - return bRCs['bRC_OK'] + DebugMessage( + context, + 100, + "Subprocess has terminated with returncode: %d\n" + % self.subprocess_returnCode, + ) + return bRCs["bRC_OK"] - elif IOP.func == bIOPS['IO_SEEK']: - return bRCs['bRC_OK'] + elif IOP.func == bIOPS["IO_SEEK"]: + return bRCs["bRC_OK"] else: - DebugMessage(context, 100, "plugin_io called with unsupported IOP:" + str(IOP.func) + "\n") - return bRCs['bRC_OK'] + DebugMessage( + context, + 100, + "plugin_io called with unsupported IOP:" + str(IOP.func) + "\n", + ) + return bRCs["bRC_OK"] def end_backup_file(self, context): - ''' + """ Check if dump was successful. - ''' + """ # Usually the xtrabackup process should have terminated here, but on some servers # it has not always. - if self.file_to_backup == 'stream': + if self.file_to_backup == "stream": returnCode = self.subprocess_returnCode if returnCode is None: - JobMessage(context, bJobMessageType['M_ERROR'], "Dump command not finished properly for unknown reason\n") + JobMessage( + context, + bJobMessageType["M_ERROR"], + "Dump command not finished properly for unknown reason\n", + ) returnCode = -99 else: - DebugMessage(context, 100, "end_backup_file() entry point in Python called. Returncode: %d\n" - % self.stream.returncode) + DebugMessage( + context, + 100, + "end_backup_file() entry point in Python called. Returncode: %d\n" + % self.stream.returncode, + ) if returnCode != 0: - msg = [ "Dump command returned non-zero value: %d" % returnCode, - "command: \"%s\"" % self.dumpcommand ] + msg = [ + "Dump command returned non-zero value: %d" % returnCode, + 'command: "%s"' % self.dumpcommand, + ] if self.log: - msg += ["log file: \"%s\"" % self.log] - JobMessage(context, bJobMessageType['M_FATAL'], ", ".join(msg) + "\n") + msg += ['log file: "%s"' % self.log] + JobMessage( + context, bJobMessageType["M_FATAL"], ", ".join(msg) + "\n" + ) if returnCode != 0: - return bRCs['bRC_Error'] + return bRCs["bRC_Error"] if self.log: - self.err_fd.write("%s Backup Job %s closes stream\n" %(datetime.datetime.now(),self.jobId)) + self.err_fd.write( + "%s Backup Job %s closes stream\n" + % (datetime.datetime.now(), self.jobId) + ) self.err_fd.close() if self.files_to_backup: - return bRCs['bRC_More'] + return bRCs["bRC_More"] else: - return bRCs['bRC_OK'] + return bRCs["bRC_OK"] def end_restore_file(self, context): - ''' + """ Check, if writing to restore command was succesfull. - ''' + """ returnCode = self.subprocess_returnCode if returnCode is None: - JobMessage(context, bJobMessageType['M_ERROR'], "Restore command not finished properly for unknown reason\n") + JobMessage( + context, + bJobMessageType["M_ERROR"], + "Restore command not finished properly for unknown reason\n", + ) returnCode = -99 else: - DebugMessage(context, 100, - "end_restore_file() entry point in Python called. Returncode: %d\n" - % self.stream.returncode) + DebugMessage( + context, + 100, + "end_restore_file() entry point in Python called. Returncode: %d\n" + % self.stream.returncode, + ) if returnCode != 0: - msg = [ "Restore command returned non-zero value: %d" % return_code ] + msg = ["Restore command returned non-zero value: %d" % return_code] if self.log: - msg += [ "log file: \"%s\"" % self.log ] - JobMessage(context, bJobMessageType['M_ERROR'], ", ".join(msg) + "\n") + msg += ['log file: "%s"' % self.log] + JobMessage(context, bJobMessageType["M_ERROR"], ", ".join(msg) + "\n") if self.log: - self.err_fd.write("%s Restore Job %s closes stream\n" %(datetime.datetime.now(),self.jobId)) + self.err_fd.write( + "%s Restore Job %s closes stream\n" + % (datetime.datetime.now(), self.jobId) + ) self.err_fd.close() if returnCode == 0: - return bRCs['bRC_OK'] + return bRCs["bRC_OK"] else: - return bRCs['bRC_Error'] + return bRCs["bRC_Error"] def restore_object_data(self, context, ROP): - ''' + """ Called on restore and on diff/inc jobs. - ''' + """ # Improve: sanity / consistence check of restore object self.row_rop_raw = ROP.object self.rop_data[ROP.jobid] = json.loads(str(self.row_rop_raw)) - if 'to_lsn' in self.rop_data[ROP.jobid] and self.rop_data[ROP.jobid]['to_lsn'] > self.max_to_lsn: - self.max_to_lsn = int(self.rop_data[ROP.jobid]['to_lsn']) - JobMessage(context, bJobMessageType['M_INFO'], - "Got to_lsn %d from restore object of job %d\n" % (self.max_to_lsn, ROP.jobid)) - return bRCs['bRC_OK'] + if ( + "to_lsn" in self.rop_data[ROP.jobid] + and self.rop_data[ROP.jobid]["to_lsn"] > self.max_to_lsn + ): + self.max_to_lsn = int(self.rop_data[ROP.jobid]["to_lsn"]) + JobMessage( + context, + bJobMessageType["M_INFO"], + "Got to_lsn %d from restore object of job %d\n" + % (self.max_to_lsn, ROP.jobid), + ) + return bRCs["bRC_OK"] # vim: ts=4 tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/core/src/plugins/filed/bareos-fd-percona.py b/core/src/plugins/filed/bareos-fd-percona.py index 40dc7bd271c..47da5e58e7e 100644 --- a/core/src/plugins/filed/bareos-fd-percona.py +++ b/core/src/plugins/filed/bareos-fd-percona.py @@ -15,13 +15,15 @@ # This module contains the used plugin class from BareosFdPluginPercona import * + def load_bareos_plugin(context, plugindef): - ''' + """ This function is called by the Bareos-FD to load the plugin We use it to instantiate the plugin class - ''' + """ # BareosFdWrapper.bareos_fd_plugin_object is the module attribute that holds the plugin class object - BareosFdWrapper.bareos_fd_plugin_object = BareosFdPercona (context, plugindef); - return bRCs['bRC_OK']; + BareosFdWrapper.bareos_fd_plugin_object = BareosFdPercona(context, plugindef) + return bRCs["bRC_OK"] + # the rest is done in the Plugin module From dc98dee892b23463ba1c104c9a4b859e7096f442 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 16:16:44 +0100 Subject: [PATCH 12/25] BareosFdPluginPercona.py: fix typos --- core/src/plugins/filed/BareosFdPluginPercona.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/plugins/filed/BareosFdPluginPercona.py b/core/src/plugins/filed/BareosFdPluginPercona.py index e7a99e9083e..7546e5ab2e5 100644 --- a/core/src/plugins/filed/BareosFdPluginPercona.py +++ b/core/src/plugins/filed/BareosFdPluginPercona.py @@ -158,7 +158,7 @@ def create_file(self, context, restorepkt): % (FNAME, origJobId), ) - # Create restore directory, if not existant + # Create restore directory, if not existent if not os.path.exists(self.writeDir): bareosfd.DebugMessage( context, @@ -171,7 +171,7 @@ def create_file(self, context, restorepkt): JobMessage( context, bJobMessageType["M_FATAL"], - "Restore with xbstream needs empty directoy: %s\n" % self.writeDir, + "Restore with xbstream needs empty directory: %s\n" % self.writeDir, ) return bRCs["bRC_Error"] self.restorecommand += self.writeDir @@ -227,7 +227,7 @@ def start_backup_job(self, context): JobMessage( context, bJobMessageType["M_FATAL"], - "Could not fetch SHOW ENGINE INNODB STATUS, unpriveleged user?", + "Could not fetch SHOW ENGINE INNODB STATUS, unprivileged user?", ) return bRCs["bRC_Error"] info = result[0][2] @@ -494,7 +494,7 @@ def end_backup_file(self, context): def end_restore_file(self, context): """ - Check, if writing to restore command was succesfull. + Check if writing to restore command was successful. """ returnCode = self.subprocess_returnCode if returnCode is None: From 0b055b270fec6bb0d7837f89f5cdb2f70db435db Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 17:03:49 +0100 Subject: [PATCH 13/25] bareos-fd-percona.py: add bareos header --- core/src/plugins/filed/bareos-fd-percona.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/src/plugins/filed/bareos-fd-percona.py b/core/src/plugins/filed/bareos-fd-percona.py index 47da5e58e7e..f7e6d0ee911 100644 --- a/core/src/plugins/filed/bareos-fd-percona.py +++ b/core/src/plugins/filed/bareos-fd-percona.py @@ -1,8 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Bareos-fd-local-fileset a simple example for a python Bareos FD Plugin using BareosFdPluginLocalFileset -# The plugin argument 'filename' is used to read all files listed in that file and add it to the fileset -# License: AGPLv3 +# Copyright (C) 2015-2020 Bareos GmbH & Co. KG +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of version three of the GNU Affero General Public +# License as published by the Free Software Foundation, which is +# listed in the file LICENSE. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# # Provided by the Bareos FD Python plugin interface from bareosfd import * From f6b8a2c1faa076f5ba9437d56cf2780249611404 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 6 Jan 2020 17:50:12 +0100 Subject: [PATCH 14/25] percon_xtrabackup: rename BareosFdPercona to BareosFdPluginPercona both in spec and cmakelists --- core/platforms/packaging/bareos.spec | 2 +- core/src/plugins/filed/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/platforms/packaging/bareos.spec b/core/platforms/packaging/bareos.spec index f770c7fadcf..96760a5af46 100644 --- a/core/platforms/packaging/bareos.spec +++ b/core/platforms/packaging/bareos.spec @@ -1402,7 +1402,7 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %files filedaemon-percona-python-plugin %defattr(-, root, root) %{plugin_dir}/bareos-fd-percona.py* -%{plugin_dir}/BareosFdPercona.py* +%{plugin_dir}/BareosFdPluginPercona.py* #%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona.conf.example #%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona.conf.example #%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona.conf.example diff --git a/core/src/plugins/filed/CMakeLists.txt b/core/src/plugins/filed/CMakeLists.txt index 7c6035c4f21..a8c7892bf7c 100644 --- a/core/src/plugins/filed/CMakeLists.txt +++ b/core/src/plugins/filed/CMakeLists.txt @@ -1,6 +1,6 @@ # BAREOS® - Backup Archiving REcovery Open Sourced # -# Copyright (C) 2017-2019 Bareos GmbH & Co. KG +# Copyright (C) 2017-2020 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public @@ -144,7 +144,7 @@ set(PYFILES BareosFdPluginOvirt.py bareos-fd-percona.py - BareosFdPercona.py + BareosFdPluginPercona.py ) install( From 31fc8109952951560134331abe8dfffcf0f47dbd Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 7 Jan 2020 08:52:30 +0100 Subject: [PATCH 15/25] percona_xtrabackup: renamed percona to perona_xtrabackup also put all docs into plugins.rst --- core/platforms/packaging/bareos.spec | 24 +-- ....py => BareosFdPluginPerconaXtraBackup.py} | 2 +- core/src/plugins/filed/CMakeLists.txt | 4 +- ...ona.py => bareos-fd-percona_xtrabackup.py} | 2 +- docs/manuals/source/Appendix/Howtos.rst | 99 --------- .../source/TasksAndConcepts/Plugins.rst | 196 +++++++----------- systemtests/CMakeLists.txt | 10 +- .../bareos-dir.d/catalog/MyCatalog.conf.in | 0 .../bareos-dir.d/client/bareos-fd.conf.in | 0 .../bareos-dir.d/console/bareos-mon.conf.in | 0 .../bareos-dir.d/director/bareos-dir.conf.in | 0 .../bareos-dir.d/fileset/Catalog.conf.in | 0 .../fileset/PerconaXtraBackupTest.conf.in} | 4 +- .../bareos-dir.d/fileset/SelfTest.conf.in | 0 .../bareos-dir.d/job/BackupCatalog.conf.in | 0 .../bareos-dir.d/job/RestoreFiles.conf.in | 0 .../bareos-dir.d/job/backup-bareos-fd.conf.in | 2 +- .../bareos-dir.d/jobdefs/DefaultJob.conf.in | 0 .../bareos-dir.d/messages/Daemon.conf.in | 0 .../bareos-dir.d/messages/Standard.conf.in | 0 .../bareos-dir.d/pool/Differential.conf | 0 .../etc/bareos/bareos-dir.d/pool/Full.conf | 0 .../bareos/bareos-dir.d/pool/Incremental.conf | 0 .../etc/bareos/bareos-dir.d/pool/Scratch.conf | 0 .../bareos/bareos-dir.d/profile/operator.conf | 0 .../bareos/bareos-dir.d/storage/File.conf.in | 0 .../bareos/bareos-fd.d/client/myself.conf.in | 0 .../bareos-fd.d/director/bareos-dir.conf.in | 0 .../bareos-fd.d/director/bareos-mon.conf.in | 0 .../bareos/bareos-fd.d/messages/Standard.conf | 0 .../bareos-sd.d/device/FileStorage.conf.in | 0 .../bareos-sd.d/director/bareos-dir.conf.in | 0 .../bareos-sd.d/director/bareos-mon.conf.in | 0 .../bareos/bareos-sd.d/messages/Standard.conf | 0 .../bareos-sd.d/storage/bareos-sd.conf.in | 0 .../etc/bareos/bconsole.conf.in | 0 .../client/FileDaemon-local.conf.in | 0 .../director/Director-local.conf.in | 0 .../tray-monitor.d/monitor/bareos-mon.conf.in | 0 .../storage/StorageDaemon-local.conf.in | 0 .../testrunner | 16 +- 41 files changed, 110 insertions(+), 249 deletions(-) rename core/src/plugins/filed/{BareosFdPluginPercona.py => BareosFdPluginPerconaXtraBackup.py} (99%) rename core/src/plugins/filed/{bareos-fd-percona.py => bareos-fd-percona_xtrabackup.py} (97%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in => python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in} (71%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in (70%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/messages/Daemon.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/messages/Standard.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Differential.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Full.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Incremental.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Scratch.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/profile/operator.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/storage/File.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/client/myself.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/messages/Standard.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/device/FileStorage.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/messages/Standard.conf (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/bconsole.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/director/Director-local.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in (100%) rename systemtests/tests/{python-fd-percona-plugin-test => python-fd-percona_xtrabackup-plugin-test}/testrunner (78%) diff --git a/core/platforms/packaging/bareos.spec b/core/platforms/packaging/bareos.spec index 96760a5af46..7b4447cff07 100644 --- a/core/platforms/packaging/bareos.spec +++ b/core/platforms/packaging/bareos.spec @@ -577,7 +577,7 @@ Requires: python-pycurl Requires: python-lxml Requires: python-ovirt-engine-sdk4 -%package filedaemon-percona-python-plugin +%package filedaemon-percona_xtrabackup-python-plugin Summary: LDAP Python plugin for Bareos File daemon Group: Productivity/Archiving/Backup Requires: bareos-filedaemon = %{version} @@ -609,7 +609,7 @@ This package contains the LDAP python plugin for the file daemon This package contains the Ovirt python plugin for the file daemon -%description filedaemon-percona-python-plugin +%description filedaemon-percona_xtrabackup-python-plugin %{dscr} This package contains the Percona python plugin for the file daemon @@ -1399,13 +1399,13 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-ovirt.conf.example %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-ovirt.conf.example -%files filedaemon-percona-python-plugin +%files filedaemon-percona_xtrabackup-python-plugin %defattr(-, root, root) -%{plugin_dir}/bareos-fd-percona.py* -%{plugin_dir}/BareosFdPluginPercona.py* -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona.conf.example -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona.conf.example -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona.conf.example +%{plugin_dir}/bareos-fd-percona_xtrabackup.py* +%{plugin_dir}/BareosFdPluginPerconaXtraBackup.py* +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona_xtrabackup.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona_xtrabackup.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona_xtrabackup.conf.example %files director-python-plugin %defattr(-, root, root) @@ -1670,10 +1670,10 @@ fi; \ %posttrans filedaemon-ldap-python-plugin %posttrans_restore_file /etc/%{name}/bareos-dir.d/plugin-python-ldap.conf -#post filedaemon-percona-python-plugin -#post_backup_file /etc/#{name}/bareos-dir.d/plugin-python-percona.conf -#posttrans filedaemon-percona-python-plugin -#posttrans_restore_file /etc/#{name}/bareos-dir.d/plugin-python-percona.conf +#post filedaemon-percona_xtrabackup-python-plugin +#post_backup_file /etc/#{name}/bareos-dir.d/plugin-python-percona_xtrabackup.conf +#posttrans filedaemon-percona_xtrabackup-python-plugin +#posttrans_restore_file /etc/#{name}/bareos-dir.d/plugin-python-percona_xtrabackup.conf %endif diff --git a/core/src/plugins/filed/BareosFdPluginPercona.py b/core/src/plugins/filed/BareosFdPluginPerconaXtraBackup.py similarity index 99% rename from core/src/plugins/filed/BareosFdPluginPercona.py rename to core/src/plugins/filed/BareosFdPluginPerconaXtraBackup.py index 7546e5ab2e5..d3279912c47 100644 --- a/core/src/plugins/filed/BareosFdPluginPercona.py +++ b/core/src/plugins/filed/BareosFdPluginPerconaXtraBackup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (C) 2015-2016 Bareos GmbH & Co. KG +# Copyright (C) 2015-2020 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public diff --git a/core/src/plugins/filed/CMakeLists.txt b/core/src/plugins/filed/CMakeLists.txt index a8c7892bf7c..e55c4cdbcee 100644 --- a/core/src/plugins/filed/CMakeLists.txt +++ b/core/src/plugins/filed/CMakeLists.txt @@ -143,8 +143,8 @@ set(PYFILES bareos-fd-ovirt.py BareosFdPluginOvirt.py - bareos-fd-percona.py - BareosFdPluginPercona.py + bareos-fd-percona_xtrabackup.py + BareosFdPluginPerconaXtraBackup.py ) install( diff --git a/core/src/plugins/filed/bareos-fd-percona.py b/core/src/plugins/filed/bareos-fd-percona_xtrabackup.py similarity index 97% rename from core/src/plugins/filed/bareos-fd-percona.py rename to core/src/plugins/filed/bareos-fd-percona_xtrabackup.py index f7e6d0ee911..6f5713b2e2f 100644 --- a/core/src/plugins/filed/bareos-fd-percona.py +++ b/core/src/plugins/filed/bareos-fd-percona_xtrabackup.py @@ -27,7 +27,7 @@ from BareosFdWrapper import * # This module contains the used plugin class -from BareosFdPluginPercona import * +from BareosFdPluginPerconaXtraBackup import * def load_bareos_plugin(context, plugindef): diff --git a/docs/manuals/source/Appendix/Howtos.rst b/docs/manuals/source/Appendix/Howtos.rst index cc4d681b1bb..62be197f2b1 100644 --- a/docs/manuals/source/Appendix/Howtos.rst +++ b/docs/manuals/source/Appendix/Howtos.rst @@ -754,105 +754,6 @@ Backup of a MySQL Database In this section, we describe different methods to do a full backup of a MySQL database. -.. _backup-mysql-xtrabackup: - -Backup of MySQL Databases using the Bareos MySQL Percona xtrabackup Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:index:`\ ` -:index:`\ ` -:index:`\ ` - -This plugin is available since :sinceVersion:`16.2.4: MySQL Incremental Backup Plugin for using Percona xtrabackup`, it uses the :command:`xtrabackup` tool from Percona to perform full and incremental hot-backups of MySQL / MariaDB tables of type InnoDB. It can also backup MyISAM tables but only as full backups. On restore it requires a preparation using the xtrabackup tools, before the tables can be restored. If you simply want to backup full dumps, then using -:ref:`backup-mysql-python` is the easier way. - -Prerequisites -''''''''''''' - -Install the xtrabackup tool from Percona. Documentation and packages are available here: https://www.percona.com/software/mysql-database/percona-xtrabackup. The plugin was successfully tested with xtrabackup versions 2.3.5 and 2.4.4. - -As it is a Python plugin, it will also require to have the package **bareos-filedaemon-python-plugin** installed on the |fd|, where you run it. - -For authentication the :file:`.mycnf` file of the user running the |fd|. Before proceeding, make sure that xtrabackup can connect to the database and create backups. - -Installation -'''''''''''' - -Make sure you have met the prerequisites. Install the files :file:`BareosFdPercona.py` and :file:`bareos-fd-percona.py` in your Bareos plugin directory (usually :file:`/usr/lib64/bareos/plugins`). These files are available in the Git repository https://github.com/bareos/bareos-contrib/tree/master/fd-plugins/bareos_percona. - -Configuration -''''''''''''' - -Activate your plugin directory in the |fd| configuration. See :ref:`fdPlugins` for more about plugins in general. - -.. code-block:: bareosconfig - :caption: bareos-fd.d/client/myself.conf - - Client { - ... - Plugin Directory = /usr/lib64/bareos/plugins - Plugin Names = "python" - } - -Now include the plugin as command-plugin in the Fileset resource: - -.. code-block:: bareosconfig - :caption: bareos-dir.d/fileset/mysql.conf - - FileSet { - Name = "mysql" - Include { - Options { - compression=GZIP - signature = MD5 - } - File = /etc - #... - Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/root/.my.cnf" - } - } - -If used this way, the plugin will call xtrabackup to create a backup of all databases in the xbstream format. This stream will be processed by Bareos. If job level is incremental, xtrabackup will perform an incremental backup since the last backup – for InnoDB tables. If you have MyISAM tables, you will get a full backup of those. - -You can append options to the plugin call as key=value pairs, separated by ’:’. The following options are available: - -- With :strong:`mycnf` you can make xtrabackup use a special mycnf-file with login credentials. - -- :strong:`dumpbinary` lets you modify the default command xtrabackup. - -- :strong:`dumpoptions` to modify the options for xtrabackup. Default setting is: :command:`--backup --datadir=/var/lib/mysql/ --stream=xbstream --extra-lsndir=/tmp/individual_tempdir` - -- :strong:`restorecommand` to modify the command for restore. Default setting is: :command:`xbstream -x -C` - -- :strong:`strictIncremental`: By default (false), an incremental backup will create data, even if the Log Sequence Number (LSN) wasn’t increased since last backup. This is to ensure, that eventual changes to MYISAM tables get into the backup. MYISAM does not support incremental backups, you will always get a full bakcup of these tables. If set to true, no data will be written into backup, if the LSN wasn’t changed. - -Restore -''''''' - -With the usual Bareos restore mechanism a file-hierarchy will be created on the restore client under the default restore location: - -:file:`/tmp/bareos-restores/_percona/` - -Each restore job gets an own subdirectory, because Percona expects an empty directory. In that subdirectory, a new directory is created for every backup job that was part of the Full-Incremental sequence. - -The naming scheme is: :file:`fromLSN_toLSN_jobid` - -Example: - -:: - - /tmp/bareos-restores/_percona/351/ - |-- 00000000000000000000_00000000000010129154_0000000334 - |-- 00000000000010129154_00000000000010142295_0000000335 - |-- 00000000000010142295_00000000000010201260_0000000338 - -This example shows the restore tree for restore job with ID 351. First subdirectory has all files from the first full backup job with ID 334. It starts at LSN 0 and goes until LSN 10129154. - -Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. The third line is the 2nd incremental job with ID 338. - -To further prepare the restored files, use the :command:`xtrabackup --prepare` command. Read https://www.percona.com/doc/percona-xtrabackup/2.4/xtrabackup_bin/incremental_backups.html for more information. - - .. _backup-mysql-python: Backup of MySQL Databases using the Python MySQL plugin diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index 9883260738f..1620989f31d 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -88,7 +88,7 @@ See chapter :ref:`backup-postgresql-plugin`. MySQL Plugin ~~~~~~~~~~~~ -See the chapters :ref:`backup-mysql-xtrabackup` and :ref:`backup-mysql-python`. +See the chapters :ref:`backup-mysql-XtraBackup` and :ref:`backup-mysql-python`. MSSQL Plugin ~~~~~~~~~~~~ @@ -983,16 +983,20 @@ This will create disk image files that could be examined for example by using the **guestfish** tool (see http://libguestfs.org/guestfish.1.html). This tool can also be used to extract single files from the disk image. + .. _PerconaXtrabackupPlugin: +.. _backup-mysql-XtraBackup: -Percona Xtrabackup Plugin -~~~~~~~~~~~~~~~~~~~~~~~~~ +Backup of MySQL Databases using the Bareos MySQL Percona XtraBackup Plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# The Bareos MySQL / MariaDB Percona xtrabackup Plugin +:index:`\ ` +:index:`\ ` +:index:`\ ` -This plugin uses Perconas xtrackup tool, to make full and incremental backups of Mysql / MariaDB databases. +This plugin uses Perconas XtraBackup tool, to make full and incremental backups of Mysql / MariaDB databases. -The key features of xtrabackup are: +The key features of XtraBackup are: - Incremental backups - Backups that complete quickly and reliably @@ -1002,19 +1006,22 @@ The key features of xtrabackup are: Incremental backups only work for INNODB tables, when using MYISAM, only full backups can be created. -## Prerequisites -You need to have the 'mysql' client program and the xtrabackup tool installed. -More about xtrabackup: https://www.percona.com/software/mysql-database/percona-xtrabackup -You will also need the package *bareos-filedaemon-python-plugin* installed on your client. +Prerequisites +''''''''''''' + +Install the XtraBackup tool from Percona. Documentation and packages are available here: https://www.percona.com/software/mysql-database/percona-XtraBackup. The plugin was successfully tested with XtraBackup versions 2.3.5 and 2.4.4. + +As it is a Python plugin, it will also require to have the package **bareos-filedaemon-python-plugin** installed on the |fd|, where you run it. -## Compatibility +For authentication the :file:`.mycnf` file of the user running the |fd|. Before proceeding, make sure that XtraBackup can connect to the database and create backups. -There are different versions of _xtrabackup_ available. Older versions required an extra binary called _innobackupex_, especially when dealing with myISAM tables. In newer versions, _innobackupex_ is just a sysmbolic link to _xtrabackup_. -We've tested some versions of xtrabackup together with the plugin: +There are different versions of _XtraBackup_ available. Older versions required an extra binary called _innobackupex_, especially when dealing with myISAM tables. In newer versions, _innobackupex_ is just a sysmbolic link to _XtraBackup_. -| xtrabackup version | Status | Remarks | +We've tested some versions of XtraBackup together with the plugin: + +| XtraBackup version | Status | Remarks | | -----------------: |:------:| -------:| |2.0.8| - | InnoDB only seems to work | |2.3.5| + | | @@ -1022,143 +1029,94 @@ We've tested some versions of xtrabackup together with the plugin: We've used the official Percona rpms on Centos 6 for testing. -## Installation ## - -1. Make sure you have met the prerequisites. -2. Install the files *BareosFdPercona.py* and *bareos-fd-percona.py* in your Bareos plugin directory (usually */usr/lib64/bareos/plugins*) - - -## Configuration ## - -### Activate your plugin directory in the fd resource conf on the client -``` -FileDaemon { - Name = client-fd - ... - Plugin Directory = /usr/lib64/bareos/plugins -} -``` - -### Include the Plugin in the fileset definition on the director -``` -FileSet { - Name = "client-data" - Include { - Options { - compression=GZIP - signature = MD5 - } - File = /etc - #... - Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona" - } -} -``` - -#### Options #### - -You can append options to the plugin call as key=value pairs, separated by ':'. -Please read more about the Bareos Python Plugin Interface here: http://doc.bareos.org/master/html/bareos-manual-main-reference.html#Python-fdPlugin - - -##### defaultsfile #### - -This parameter allows to specify a defaultsfile that shall be used for mysql(client) and *xtrabackup* command line utilities. -Example: - -``` -Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/path/to/your/my.cnf" -``` - -##### dumpbinary ##### - -Command (with or without full path) to create the dumps. Default: *xtrabackup* - -##### dumpoptions ##### +Installation +'''''''''''' -Options to be used with the dumpbinary. -Default: - --backup --stream=xbstream +Make sure you have met the prerequisites. Install the package **bareos-filedaemon-percona_XtraBackup-python-plugin**. -##### extradumpoptions ##### +Configuration +''''''''''''' -Additional options appended to dumpoptions. +Activate your plugin directory in the |fd| configuration. See :ref:`fdPlugins` for more about plugins in general. -###### Choosing databases ###### +.. code-block:: bareosconfig + :caption: bareos-fd.d/client/myself.conf -By default all found databases are backed up. You can restrict this -using the dumpoptions or extradumpoptions parameter. If you modify -dumpoptions, be careful that you include all necessary options. See -*xtrabackup* documentation for details. + Client { + ... + Plugin Directory = /usr/lib64/bareos/plugins + Plugin Names = "python" + } +Now include the plugin as command-plugin in the Fileset resource: -##### restorecommand -Command used for restoring, default: - xbstream -x -C +.. code-block:: bareosconfig + :caption: bareos-dir.d/fileset/mysql.conf -##### strictIncremental ##### -Default: False + FileSet { + Name = "mysql" + Include { + Options { + compression=GZIP + signature = MD5 + } + File = /etc + #... + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/root/.my.cnf" + } + } -By default, an incremental will create data, even if the Log Sequence Number (LSN) wasn't increased since last backup. This is to ensure, that eventual changes to -MYISAM tables get into the backup. MYISAM does not support incremental backups, you will always get a full bakcup of these tables. +If used this way, the plugin will call XtraBackup to create a backup of all databases in the xbstream format. This stream will be processed by Bareos. If job level is incremental, XtraBackup will perform an incremental backup since the last backup – for InnoDB tables. If you have MyISAM tables, you will get a full backup of those. -If set to true, no data will be written into backup, if the LSN wasn't changed. +You can append options to the plugin call as key=value pairs, separated by ’:’. The following options are available: -##### log ##### -Default: false +- With :strong:`mycnf` you can make XtraBackup use a special mycnf-file with login credentials. -By default, no extra logfile is written on the FD running the plugin. If you want to have some additional debug information, you might specify a -logfile here. If you set a filename with path, this will be used. If you specify just a filename without path, the default path for logs -*/var/log/bareos/* will be prepended. +- :strong:`dumpbinary` lets you modify the default command XtraBackup. -If you use a logfilename that matches */var/log/bareos/bareos\*.log*, it will be handled by logrotate. +- :strong:`dumpoptions` to modify the options for XtraBackup. Default setting is: :command:`--backup --datadir=/var/lib/mysql/ --stream=xbstream --extra-lsndir=/tmp/individual_tempdir` -## Backup ## +- :strong:`restorecommand` to modify the command for restore. Default setting is: :command:`xbstream -x -C` -When running full backups, the plugin will call the _xtrabackup_ command with the according options. Format is _xbstream_. LSN information -will be written into a temporary directory, using the _--extra-lsndir_ option. The information (LSN) will be used to write a so called -restore object. This information is needed for following incremental jobs, so they are aware of the previous backups (and how far by -means of LSN) they went. +- :strong:`strictIncremental`: By default (false), an incremental backup will create data, even if the Log Sequence Number (LSN) wasn’t increased since last backup. This is to ensure, that eventual changes to MYISAM tables get into the backup. MYISAM does not support incremental backups, you will always get a full bakcup of these tables. If set to true, no data will be written into backup, if the LSN wasn’t changed. -## Restore ## +Restore +''''''' With the usual Bareos restore mechanism a file-hierarchy will be created on the restore client under the default restore location: -*/tmp/bareos-restores/_percona/* +:file:`/tmp/bareos-restores/_percona/` -Each restore job gets an own subdirectory, because Percona expects an empty directory. In that subdirectory, -a new directory is created for every backup job that was part of the Full-Incremental sequence. +Each restore job gets an own subdirectory, because Percona expects an empty directory. In that subdirectory, a new directory is created for every backup job that was part of the Full-Incremental sequence. -The naming scheme is: -*fromLSN_toLSN_jobid* +The naming scheme is: :file:`fromLSN_toLSN_jobid` Example: -``` -/tmp/bareos-restores/_percona/351/ -├── 00000000000000000000_00000000000010129154_0000000334 -├── 00000000000010129154_00000000000010142295_0000000335 -├── 00000000000010142295_00000000000010201260_0000000338 -``` -This example shows the restore tree for restore job with ID 351. First subdirectory has all files -from the first full backup job with ID 334. It starts at LSN 0 and goes until LSN 10129154. +:: + + /tmp/bareos-restores/_percona/351/ + |-- 00000000000000000000_00000000000010129154_0000000334 + |-- 00000000000010129154_00000000000010142295_0000000335 + |-- 00000000000010142295_00000000000010201260_0000000338 + +This example shows the restore tree for restore job with ID 351. First subdirectory has all files from the first full backup job with ID 334. It starts at LSN 0 and goes until LSN 10129154. -Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. -The third line is the 2nd incremental job with ID 338. +Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. The third line is the 2nd incremental job with ID 338. -To further prepare the restored files, use the *xtrabackup --prepare* command. Read https://www.percona.com/doc/percona-xtrabackup/2.4/xtrabackup_bin/incremental_backups.html -for more information. +To further prepare the restored files, use the :command:`XtraBackup --prepare` command. Read https://www.percona.com/doc/percona-XtraBackup/2.4/XtraBackup_bin/incremental_backups.html for more information. ## Troubleshooting ## -If things don't work as expected, make sure: +If things don't work as expected, make sure that -- Bareos FileDaemon (FD) works in general, so that you can make simple file backups and restores -- Bareos FD Python plugins work in genreral, try one of the shipped simple sample plugins -- Make sure *xtrabackup* works as user root, MySQL access needs to be configured properly +- Bareos FileDaemon (FD) works in general, so that you can make simple file + backups and restores - Bareos FD Python plugins work in general, try one of + the shipped simple sample plugins +- Make sure *XtraBackup* works as user root, MySQL access needs to be + configured properly -If this all does not help, you can start the Bareos FD in Debug mode and look deeper into it. Support is available here: https://www.bareos.com diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index 153999c3ee4..f04069fbfed 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -114,9 +114,9 @@ macro(handle_python_plugin_modules test_name) list(APPEND PYMODULES_TO_LINK_TO_SRC) endif() - if(${test_name} STREQUAL python-fd-percona-plugin-test) - list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginPercona.py - filed/bareos-fd-percona.py + if(${test_name} STREQUAL python-fd-percona_xtrabackup-plugin-test) + list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginPerconaXtraBackup.py + filed/bareos-fd-percona_xtrabackup.py ) endif() @@ -398,9 +398,9 @@ else() endif() if(TARGET python-fd AND XTRABACKUP) - list(APPEND SYSTEM_TESTS "python-fd-percona-plugin-test") + list(APPEND SYSTEM_TESTS "python-fd-percona_xtrabackup-plugin-test") else() - list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona-plugin-test") + list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona_xtrabackup-plugin-test") endif() if(TARGET python-dir) diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in similarity index 71% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in index 71d41f37201..4db71b42590 100644 --- a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaTest.conf.in +++ b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in @@ -1,10 +1,10 @@ FileSet { - Name = "PerconaTest" + Name = "PerconaXtraBackupTest" Description = "Test the Plugin functionality of the Percona Plugin." Include { Options { signature = MD5 } - Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona" + Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona_xtrabackup" } } diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in similarity index 70% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in index db84a37b10c..f773d37865d 100644 --- a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in +++ b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in @@ -2,5 +2,5 @@ Job { Name = "backup-bareos-fd" JobDefs = "DefaultJob" Client = "bareos-fd" - FileSet = "PerconaTest" + FileSet = "PerconaXtraBackupTest" } diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bconsole.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/bconsole.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bconsole.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in diff --git a/systemtests/tests/python-fd-percona-plugin-test/testrunner b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/testrunner similarity index 78% rename from systemtests/tests/python-fd-percona-plugin-test/testrunner rename to systemtests/tests/python-fd-percona_xtrabackup-plugin-test/testrunner index 8841b2c3066..a1d2c9e8c85 100755 --- a/systemtests/tests/python-fd-percona-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/testrunner @@ -2,7 +2,7 @@ # # This systemtest tests the Percona plugin functionality # of the Bareos FD by using the supplied module -# BareosFdPluginPercona.py +# BareosFdPluginPerconaXtraBackup.py # TestName="$(basename "$(pwd)")" export TestName @@ -14,13 +14,15 @@ JobName=backup-bareos-fd ${scripts}/cleanup ${scripts}/setup +xtrabackup_test_db="${db_name}_xtrabackup" + start_test echo "------ running PERCONA tests" -echo "drop database ${db_name}_percona" | mysql -echo "create database ${db_name}_percona" | mysql -echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | mysql ${db_name}_percona -echo "insert into test (data) VALUES ('test eintrag 1') " | mysql ${db_name}_percona +echo "drop database ${xtrabackup_test_db}" | mysql +echo "create database ${xtrabackup_test_db}" | mysql +echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | mysql ${xtrabackup_test_db} +echo "insert into test (data) VALUES ('test entry 1') " | mysql ${xtrabackup_test_db} BCONSOLE="${current_test_directory}/bin/bconsole" @@ -36,7 +38,7 @@ echo -e "wait JobName=$JobName" | $BCONSOLE #sleep 2 echo "status dir" | $BCONSOLE # insert data and run incremental -echo "insert into test (data) VALUES ('test eintrag 2') " | mysql ${db_name}_percona +echo "insert into test (data) VALUES ('test entry 2') " | mysql ${xtrabackup_test_db} COMMAND="$COMMAND level=Incremental" echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" @@ -55,7 +57,7 @@ echo "status dir" | $BCONSOLE # run restore -RESTORECMD="restore client=bareos-fd fileset=PerconaTest yes restorejob=RestoreFile select all\rdone\r" +RESTORECMD="restore client=bareos-fd fileset=PerconaXtraBackupTest yes restorejob=RestoreFile select all\rdone\r" echo "@$out $tmp/log2.out" | $BCONSOLE From 6af3ceaa23a4642adf67545f451fab6238102806 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 7 Jan 2020 15:06:13 +0100 Subject: [PATCH 16/25] docs: shorten percona xtrabackup plugin documentation --- .../source/TasksAndConcepts/Plugins.rst | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index 1620989f31d..b4dc4f5fa3a 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1017,18 +1017,6 @@ As it is a Python plugin, it will also require to have the package **bareos-file For authentication the :file:`.mycnf` file of the user running the |fd|. Before proceeding, make sure that XtraBackup can connect to the database and create backups. -There are different versions of _XtraBackup_ available. Older versions required an extra binary called _innobackupex_, especially when dealing with myISAM tables. In newer versions, _innobackupex_ is just a sysmbolic link to _XtraBackup_. - -We've tested some versions of XtraBackup together with the plugin: - -| XtraBackup version | Status | Remarks | -| -----------------: |:------:| -------:| -|2.0.8| - | InnoDB only seems to work | -|2.3.5| + | | -|2.4.4| + | | - -We've used the official Percona rpms on Centos 6 for testing. - Installation '''''''''''' @@ -1062,7 +1050,7 @@ Now include the plugin as command-plugin in the Fileset resource: } File = /etc #... - Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/root/.my.cnf" + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona_xtrabackup:mycnf=/root/.my.cnf" } } @@ -1104,11 +1092,11 @@ This example shows the restore tree for restore job with ID 351. First subdirect Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. The third line is the 2nd incremental job with ID 338. -To further prepare the restored files, use the :command:`XtraBackup --prepare` command. Read https://www.percona.com/doc/percona-XtraBackup/2.4/XtraBackup_bin/incremental_backups.html for more information. - +To further prepare the restored files, use the :command:`XtraBackup --prepare` command. Read https://www.percona.com/doc/percona-xtrabackup/2.4/backup_scenarios/incremental_backup.html for more information. -## Troubleshooting ## +Troubleshooting +''''''''''''''' If things don't work as expected, make sure that - Bareos FileDaemon (FD) works in general, so that you can make simple file From 42408fc76b48f0d0168333e29a023b848c45996a Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 8 Jan 2020 10:30:58 +0100 Subject: [PATCH 17/25] Apply suggestions from code review Co-Authored-By: Frank Ueberschar --- docs/manuals/source/TasksAndConcepts/Plugins.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index b4dc4f5fa3a..a38afb03eed 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1014,13 +1014,13 @@ Install the XtraBackup tool from Percona. Documentation and packages are availab As it is a Python plugin, it will also require to have the package **bareos-filedaemon-python-plugin** installed on the |fd|, where you run it. -For authentication the :file:`.mycnf` file of the user running the |fd|. Before proceeding, make sure that XtraBackup can connect to the database and create backups. +For authentication the :file:`.mycnf` file of the user running the |fd| is used. Before proceeding, make sure that XtraBackup can connect to the database and create backups. Installation '''''''''''' -Make sure you have met the prerequisites. Install the package **bareos-filedaemon-percona_XtraBackup-python-plugin**. +Make sure you have met the prerequisites, after that install the package **bareos-filedaemon-percona_XtraBackup-python-plugin**. Configuration ''''''''''''' @@ -1099,8 +1099,8 @@ Troubleshooting ''''''''''''''' If things don't work as expected, make sure that -- Bareos FileDaemon (FD) works in general, so that you can make simple file - backups and restores - Bareos FD Python plugins work in general, try one of +- the |fd| (FD) works in general, so that you can make simple file backups and restores + - the Bareos FD Python plugins work in general, try one of the shipped simple sample plugins - Make sure *XtraBackup* works as user root, MySQL access needs to be configured properly @@ -1480,4 +1480,3 @@ Nagios:index:`\ `\ using the NSCA protocol. - From 186681808a12dd69e1df6973220acc78e1a45ac3 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 8 Jan 2020 10:33:43 +0100 Subject: [PATCH 18/25] Update docs/manuals/source/TasksAndConcepts/Plugins.rst --- docs/manuals/source/TasksAndConcepts/Plugins.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index a38afb03eed..c7636b8cec3 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1100,7 +1100,7 @@ Troubleshooting If things don't work as expected, make sure that - the |fd| (FD) works in general, so that you can make simple file backups and restores - - the Bareos FD Python plugins work in general, try one of +- the Bareos FD Python plugins work in general, try one of the shipped simple sample plugins - Make sure *XtraBackup* works as user root, MySQL access needs to be configured properly @@ -1479,4 +1479,3 @@ Some plugin examples are available on https://github.com/bareos/bareos-contrib. Nagios:index:`\ `\ using the NSCA protocol. - From 8ed36c5b0ca231f4b4c21c2242c9279d33361cda Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Thu, 9 Jan 2020 10:20:38 +0100 Subject: [PATCH 19/25] percona xtrabackup plugin: add debian packaging --- ...edaemon-percona_xtrabackup-python-plugin.install.in | 2 ++ ....bareos-filedaemon-percona_xtrabackup-python-plugin | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in create mode 100644 core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin diff --git a/core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in b/core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in new file mode 100644 index 00000000000..7f230d4ccb4 --- /dev/null +++ b/core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in @@ -0,0 +1,2 @@ +@plugindir@/bareos-fd-percona_xtrabackup.py* +@plugindir@/BareosFdPluginPerconaXtraBackup.py* diff --git a/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin b/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin new file mode 100644 index 00000000000..624b64d6d30 --- /dev/null +++ b/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin @@ -0,0 +1,10 @@ +Package: bareos-filedaemon-percona_xtrabackup-python-plugin +Architecture: any +Section: python +Pre-Depends: debconf (>= 1.4.30) | debconf-2.0 +Depends: bareos-common (= ${binary:Version}), bareos-filedaemon-python-plugin (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} +Description: Backup Archiving Recovery Open Sourced - file daemon LDAP plugin + Bareos is a set of programs to manage backup, recovery and verification of + data across a network of computers of different kinds. + . + This package provides the Percona XtraBackup Python plugin for the filedaemon. From 14562bb7f5d08df3fef7b3df4673297c541d7035 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Thu, 9 Jan 2020 12:25:00 +0100 Subject: [PATCH 20/25] moved bareos-filedaemon-percona_xtrabackup-python-plugin definition to ontrol.bareos-filedaemon-python-plugin --- ...bareos-filedaemon-percona_xtrabackup-python-plugin | 10 ---------- core/debian/control.bareos-filedaemon-python-plugin | 11 +++++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin diff --git a/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin b/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin deleted file mode 100644 index 624b64d6d30..00000000000 --- a/core/debian/control.bareos-filedaemon-percona_xtrabackup-python-plugin +++ /dev/null @@ -1,10 +0,0 @@ -Package: bareos-filedaemon-percona_xtrabackup-python-plugin -Architecture: any -Section: python -Pre-Depends: debconf (>= 1.4.30) | debconf-2.0 -Depends: bareos-common (= ${binary:Version}), bareos-filedaemon-python-plugin (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} -Description: Backup Archiving Recovery Open Sourced - file daemon LDAP plugin - Bareos is a set of programs to manage backup, recovery and verification of - data across a network of computers of different kinds. - . - This package provides the Percona XtraBackup Python plugin for the filedaemon. diff --git a/core/debian/control.bareos-filedaemon-python-plugin b/core/debian/control.bareos-filedaemon-python-plugin index fb0a6326648..89addedb55e 100644 --- a/core/debian/control.bareos-filedaemon-python-plugin +++ b/core/debian/control.bareos-filedaemon-python-plugin @@ -19,3 +19,14 @@ Description: Backup Archiving Recovery Open Sourced - file daemon LDAP plugin data across a network of computers of different kinds. . This package provides the LDAP Python plugin for the filedaemon. + +Package: bareos-filedaemon-percona_xtrabackup-python-plugin +Architecture: any +Section: python +Pre-Depends: debconf (>= 1.4.30) | debconf-2.0 +Depends: bareos-common (= ${binary:Version}), bareos-filedaemon-python-plugin (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} +Description: Backup Archiving Recovery Open Sourced - file daemon LDAP plugin + Bareos is a set of programs to manage backup, recovery and verification of + data across a network of computers of different kinds. + . + This package provides the Percona XtraBackup Python plugin for the filedaemon. From f909b407ad64ec8bb10039eec6de5cc777d4e8b3 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Thu, 9 Jan 2020 12:42:01 +0100 Subject: [PATCH 21/25] do not use _ in package name --- ...reos-filedaemon-percona-xtrabackup-python-plugin.install.in} | 0 core/debian/control.bareos-filedaemon-python-plugin | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename core/debian/{bareos-filedaemon-percona_xtrabackup-python-plugin.install.in => bareos-filedaemon-percona-xtrabackup-python-plugin.install.in} (100%) diff --git a/core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in b/core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in similarity index 100% rename from core/debian/bareos-filedaemon-percona_xtrabackup-python-plugin.install.in rename to core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in diff --git a/core/debian/control.bareos-filedaemon-python-plugin b/core/debian/control.bareos-filedaemon-python-plugin index 89addedb55e..1d951fda0dc 100644 --- a/core/debian/control.bareos-filedaemon-python-plugin +++ b/core/debian/control.bareos-filedaemon-python-plugin @@ -20,7 +20,7 @@ Description: Backup Archiving Recovery Open Sourced - file daemon LDAP plugin . This package provides the LDAP Python plugin for the filedaemon. -Package: bareos-filedaemon-percona_xtrabackup-python-plugin +Package: bareos-filedaemon-percona-xtrabackup-python-plugin Architecture: any Section: python Pre-Depends: debconf (>= 1.4.30) | debconf-2.0 From 0d68db983728f28715b1e4af5efce17e4d9c3446 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Thu, 9 Jan 2020 16:19:04 +0100 Subject: [PATCH 22/25] Percona Xtrabackup Plugin: rename percona_xtrabackup to percona-xtrabackup everywhere --- ...ercona-xtrabackup-python-plugin.install.in | 2 +- core/platforms/packaging/bareos.spec | 22 +++++++++---------- core/src/plugins/filed/CMakeLists.txt | 2 +- ...kup.py => bareos-fd-percona-xtrabackup.py} | 0 .../source/TasksAndConcepts/Plugins.rst | 2 +- systemtests/CMakeLists.txt | 8 +++---- .../bareos-dir.d/catalog/MyCatalog.conf.in | 0 .../bareos-dir.d/client/bareos-fd.conf.in | 0 .../bareos-dir.d/console/bareos-mon.conf.in | 0 .../bareos-dir.d/director/bareos-dir.conf.in | 0 .../bareos-dir.d/fileset/Catalog.conf.in | 0 .../fileset/PerconaXtraBackupTest.conf.in | 2 +- .../bareos-dir.d/fileset/SelfTest.conf.in | 0 .../bareos-dir.d/job/BackupCatalog.conf.in | 0 .../bareos-dir.d/job/RestoreFiles.conf.in | 0 .../bareos-dir.d/job/backup-bareos-fd.conf.in | 0 .../bareos-dir.d/jobdefs/DefaultJob.conf.in | 0 .../bareos-dir.d/messages/Daemon.conf.in | 0 .../bareos-dir.d/messages/Standard.conf.in | 0 .../bareos-dir.d/pool/Differential.conf | 0 .../etc/bareos/bareos-dir.d/pool/Full.conf | 0 .../bareos/bareos-dir.d/pool/Incremental.conf | 0 .../etc/bareos/bareos-dir.d/pool/Scratch.conf | 0 .../bareos/bareos-dir.d/profile/operator.conf | 0 .../bareos/bareos-dir.d/storage/File.conf.in | 0 .../bareos/bareos-fd.d/client/myself.conf.in | 0 .../bareos-fd.d/director/bareos-dir.conf.in | 0 .../bareos-fd.d/director/bareos-mon.conf.in | 0 .../bareos/bareos-fd.d/messages/Standard.conf | 0 .../bareos-sd.d/device/FileStorage.conf.in | 0 .../bareos-sd.d/director/bareos-dir.conf.in | 0 .../bareos-sd.d/director/bareos-mon.conf.in | 0 .../bareos/bareos-sd.d/messages/Standard.conf | 0 .../bareos-sd.d/storage/bareos-sd.conf.in | 0 .../etc/bareos/bconsole.conf.in | 0 .../client/FileDaemon-local.conf.in | 0 .../director/Director-local.conf.in | 0 .../tray-monitor.d/monitor/bareos-mon.conf.in | 0 .../storage/StorageDaemon-local.conf.in | 0 .../testrunner | 0 40 files changed, 19 insertions(+), 19 deletions(-) rename core/src/plugins/filed/{bareos-fd-percona_xtrabackup.py => bareos-fd-percona-xtrabackup.py} (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in (83%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/messages/Daemon.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/messages/Standard.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Differential.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Full.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Incremental.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/pool/Scratch.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/profile/operator.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-dir.d/storage/File.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/client/myself.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-fd.d/messages/Standard.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/device/FileStorage.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/messages/Standard.conf (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/bconsole.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/director/Director-local.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in (100%) rename systemtests/tests/{python-fd-percona_xtrabackup-plugin-test => python-fd-percona-xtrabackup-plugin-test}/testrunner (100%) diff --git a/core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in b/core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in index 7f230d4ccb4..b828764c6e1 100644 --- a/core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in +++ b/core/debian/bareos-filedaemon-percona-xtrabackup-python-plugin.install.in @@ -1,2 +1,2 @@ -@plugindir@/bareos-fd-percona_xtrabackup.py* +@plugindir@/bareos-fd-percona-xtrabackup.py* @plugindir@/BareosFdPluginPerconaXtraBackup.py* diff --git a/core/platforms/packaging/bareos.spec b/core/platforms/packaging/bareos.spec index 7b4447cff07..81783fcd467 100644 --- a/core/platforms/packaging/bareos.spec +++ b/core/platforms/packaging/bareos.spec @@ -577,7 +577,7 @@ Requires: python-pycurl Requires: python-lxml Requires: python-ovirt-engine-sdk4 -%package filedaemon-percona_xtrabackup-python-plugin +%package filedaemon-percona-xtrabackup-python-plugin Summary: LDAP Python plugin for Bareos File daemon Group: Productivity/Archiving/Backup Requires: bareos-filedaemon = %{version} @@ -609,7 +609,7 @@ This package contains the LDAP python plugin for the file daemon This package contains the Ovirt python plugin for the file daemon -%description filedaemon-percona_xtrabackup-python-plugin +%description filedaemon-percona-xtrabackup-python-plugin %{dscr} This package contains the Percona python plugin for the file daemon @@ -1399,13 +1399,13 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-ovirt.conf.example %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-ovirt.conf.example -%files filedaemon-percona_xtrabackup-python-plugin +%files filedaemon-percona-xtrabackup-python-plugin %defattr(-, root, root) -%{plugin_dir}/bareos-fd-percona_xtrabackup.py* +%{plugin_dir}/bareos-fd-percona-xtrabackup.py* %{plugin_dir}/BareosFdPluginPerconaXtraBackup.py* -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona_xtrabackup.conf.example -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona_xtrabackup.conf.example -#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona_xtrabackup.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-percona-xtrabackup.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-percona-xtrabackup.conf.example +#%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/restore-percona-xtrabackup.conf.example %files director-python-plugin %defattr(-, root, root) @@ -1670,10 +1670,10 @@ fi; \ %posttrans filedaemon-ldap-python-plugin %posttrans_restore_file /etc/%{name}/bareos-dir.d/plugin-python-ldap.conf -#post filedaemon-percona_xtrabackup-python-plugin -#post_backup_file /etc/#{name}/bareos-dir.d/plugin-python-percona_xtrabackup.conf -#posttrans filedaemon-percona_xtrabackup-python-plugin -#posttrans_restore_file /etc/#{name}/bareos-dir.d/plugin-python-percona_xtrabackup.conf +#post filedaemon-percona-xtrabackup-python-plugin +#post_backup_file /etc/#{name}/bareos-dir.d/plugin-python-percona-xtrabackup.conf +#posttrans filedaemon-percona-xtrabackup-python-plugin +#posttrans_restore_file /etc/#{name}/bareos-dir.d/plugin-python-percona-xtrabackup.conf %endif diff --git a/core/src/plugins/filed/CMakeLists.txt b/core/src/plugins/filed/CMakeLists.txt index e55c4cdbcee..b2a7e0cbd69 100644 --- a/core/src/plugins/filed/CMakeLists.txt +++ b/core/src/plugins/filed/CMakeLists.txt @@ -143,7 +143,7 @@ set(PYFILES bareos-fd-ovirt.py BareosFdPluginOvirt.py - bareos-fd-percona_xtrabackup.py + bareos-fd-percona-xtrabackup.py BareosFdPluginPerconaXtraBackup.py ) diff --git a/core/src/plugins/filed/bareos-fd-percona_xtrabackup.py b/core/src/plugins/filed/bareos-fd-percona-xtrabackup.py similarity index 100% rename from core/src/plugins/filed/bareos-fd-percona_xtrabackup.py rename to core/src/plugins/filed/bareos-fd-percona-xtrabackup.py diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index c7636b8cec3..7a4b8666512 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -1050,7 +1050,7 @@ Now include the plugin as command-plugin in the Fileset resource: } File = /etc #... - Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona_xtrabackup:mycnf=/root/.my.cnf" + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona-xtrabackup:mycnf=/root/.my.cnf" } } diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index f04069fbfed..d1c8de465ca 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -114,9 +114,9 @@ macro(handle_python_plugin_modules test_name) list(APPEND PYMODULES_TO_LINK_TO_SRC) endif() - if(${test_name} STREQUAL python-fd-percona_xtrabackup-plugin-test) + if(${test_name} STREQUAL python-fd-percona-xtrabackup-plugin-test) list(APPEND PYMODULES_TO_LINK_TO_SRC filed/BareosFdPluginPerconaXtraBackup.py - filed/bareos-fd-percona_xtrabackup.py + filed/bareos-fd-percona-xtrabackup.py ) endif() @@ -398,9 +398,9 @@ else() endif() if(TARGET python-fd AND XTRABACKUP) - list(APPEND SYSTEM_TESTS "python-fd-percona_xtrabackup-plugin-test") + list(APPEND SYSTEM_TESTS "python-fd-percona-xtrabackup-plugin-test") else() - list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona_xtrabackup-plugin-test") + list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona-xtrabackup-plugin-test") endif() if(TARGET python-dir) diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/client/bareos-fd.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/console/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/Catalog.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in similarity index 83% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in index 4db71b42590..c9f103d1433 100644 --- a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in +++ b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in @@ -5,6 +5,6 @@ FileSet { Options { signature = MD5 } - Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona_xtrabackup" + Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona-xtrabackup" } } diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/SelfTest.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/BackupCatalog.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/RestoreFiles.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/job/backup-bareos-fd.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/jobdefs/DefaultJob.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Daemon.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/messages/Standard.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Differential.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Full.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Incremental.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/pool/Scratch.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/profile/operator.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/storage/File.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/client/myself.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/director/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-fd.d/messages/Standard.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/device/FileStorage.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-dir.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/director/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/messages/Standard.conf diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-sd.d/storage/bareos-sd.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bconsole.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bconsole.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/bconsole.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bconsole.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/client/FileDaemon-local.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/director/Director-local.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/monitor/bareos-mon.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/tray-monitor.d/storage/StorageDaemon-local.conf.in diff --git a/systemtests/tests/python-fd-percona_xtrabackup-plugin-test/testrunner b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner similarity index 100% rename from systemtests/tests/python-fd-percona_xtrabackup-plugin-test/testrunner rename to systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner From f2e253d43c97af162370aa25d99647892ea8ab62 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Fri, 10 Jan 2020 13:32:45 +0100 Subject: [PATCH 23/25] detect both xtrabackup and mariabackup --- core/CMakeLists.txt | 1 + core/cmake/BareosFindPrograms.cmake | 2 ++ systemtests/CMakeLists.txt | 9 +++++- .../fileset/PerconaXtraBackupTest.conf.in | 2 +- .../testrunner | 28 +++++++------------ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 92edf0b70b3..2187246c603 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -923,6 +923,7 @@ message(" GAWK: ${GAWK}") message(" RPCGEN: ${RPCGEN}") message(" MTX: ${MTX}") message(" XTRABACKUP: ${XTRABACKUP}") +message(" MARIABACKUP: ${MARIABACKUP}") message(" DEVELOPER: ${developer}") message(" LocalBuildDefinitionsFile: ${BareosLocalBuildDefinitionsFile}") message(" HAVE_IS_TRIVIALLY_COPYABLE: ${HAVE_IS_TRIVIALLY_COPYABLE}") diff --git a/core/cmake/BareosFindPrograms.cmake b/core/cmake/BareosFindPrograms.cmake index ff46060be6f..0a23b390961 100644 --- a/core/cmake/BareosFindPrograms.cmake +++ b/core/cmake/BareosFindPrograms.cmake @@ -39,3 +39,5 @@ find_program(GDB gdb) find_program(DBX dbx) find_program(MDB mdb) find_program(XTRABACKUP xtrabackup) +find_program(MARIABACKUP mariabackup) + diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt index d1c8de465ca..492b59fed75 100644 --- a/systemtests/CMakeLists.txt +++ b/systemtests/CMakeLists.txt @@ -397,7 +397,14 @@ else() list(APPEND SYSTEM_TESTS_DISABLED "python-fd-ovirt-plugin-test") endif() -if(TARGET python-fd AND XTRABACKUP) +if(XTRABACKUP) + set (XTRABACKUP_OR_MARIABACKUP ${XTRABACKUP}) +endif() +if(MARIABACKUP) + set (XTRABACKUP_OR_MARIABACKUP ${MARIABACKUP}) + set (extradumpoptions "--user=root") +endif() +if(TARGET python-fd AND XTRABACKUP_OR_MARIABACKUP ) list(APPEND SYSTEM_TESTS "python-fd-percona-xtrabackup-plugin-test") else() list(APPEND SYSTEM_TESTS_DISABLED "python-fd-percona-xtrabackup-plugin-test") diff --git a/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in index c9f103d1433..60702c1d7a5 100644 --- a/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in +++ b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/etc/bareos/bareos-dir.d/fileset/PerconaXtraBackupTest.conf.in @@ -5,6 +5,6 @@ FileSet { Options { signature = MD5 } - Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona-xtrabackup" + Plugin = "python:module_path=@python_plugin_module_src_test_dir@:module_name=bareos-fd-percona-xtrabackup:dumpbinary=@XTRABACKUP_OR_MARIABACKUP@:extradumpoptions=@extradumpoptions@" } } diff --git a/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner index a1d2c9e8c85..9c07b973e8b 100755 --- a/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner +++ b/systemtests/tests/python-fd-percona-xtrabackup-plugin-test/testrunner @@ -15,55 +15,47 @@ ${scripts}/cleanup ${scripts}/setup xtrabackup_test_db="${db_name}_xtrabackup" +BCONSOLE="${current_test_directory}/bin/bconsole" start_test -echo "------ running PERCONA tests" echo "drop database ${xtrabackup_test_db}" | mysql echo "create database ${xtrabackup_test_db}" | mysql echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | mysql ${xtrabackup_test_db} echo "insert into test (data) VALUES ('test entry 1') " | mysql ${xtrabackup_test_db} -BCONSOLE="${current_test_directory}/bin/bconsole" - run_bareos echo "@$out $tmp/log1.out" | $BCONSOLE COMMAND="run job=$JobName" -echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" -#sleep 5 -echo -e "wait JobName=$JobName" | $BCONSOLE -#sleep 2 +echo "$COMMAND yes" | $BCONSOLE +echo "wait JobName=$JobName" | $BCONSOLE echo "status dir" | $BCONSOLE # insert data and run incremental echo "insert into test (data) VALUES ('test entry 2') " | mysql ${xtrabackup_test_db} COMMAND="$COMMAND level=Incremental" -echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" -#sleep 5 -echo -e "wait JobName=$JobName" | $BCONSOLE -#sleep 2 +echo "$COMMAND yes" | $BCONSOLE #| grep "Job queued. JobId=" +echo "wait JobName=$JobName" | $BCONSOLE echo "status dir" | $BCONSOLE # run incremental again without any new data -echo -e "$COMMAND \ryes\rwait" | $BCONSOLE #| grep "Job queued. JobId=" -#sleep 5 -echo -e "wait JobName=$JobName" | $BCONSOLE -#sleep 2 +echo "$COMMAND yes" | $BCONSOLE #| grep "Job queued. JobId=" +echo "wait JobName=$JobName" | $BCONSOLE echo "status dir" | $BCONSOLE # run restore -RESTORECMD="restore client=bareos-fd fileset=PerconaXtraBackupTest yes restorejob=RestoreFile select all\rdone\r" +RESTORECMD="restore client=bareos-fd fileset=PerconaXtraBackupTest yes restorejob=RestoreFile select all done" echo "@$out $tmp/log2.out" | $BCONSOLE -JOBID=`echo -e "$RESTORECMD" | $BCONSOLE | grep "Job queued. JobId=" | sed s'/.*=//' ` +JOBID=`echo "$RESTORECMD" | $BCONSOLE | grep "Job queued. JobId=" | sed s'/.*=//' ` -echo -e "wait jobid=$JOBID" | $BCONSOLE | grep -q "JobStatus=OK" +echo "wait jobid=$JOBID" | $BCONSOLE | grep -q "JobStatus=OK" if [[ $? != 0 ]]; then echo "Restore Job $JOBID failed" estat=1 From 4de9a8d56e27a22d93dd17991b08be3bb7f9facc Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Fri, 10 Jan 2020 14:56:11 +0100 Subject: [PATCH 24/25] AUTHORS: added all missing contributors from bareos-contrib repo --- AUTHORS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AUTHORS b/AUTHORS index 1a25af52d0e..d046b3ca097 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,12 +7,15 @@ Adrian Close Aitor Matilla Alan Brown Aleksandar Milivojevic +Aleksandr Kozlov +Aleksey Volkov Alessandro Rigopoulos Alexander Bergolth Alexander Kushnirenko Alexandre Baron Alexandre Simon Allan Black +Andreas Thienemann Andre Noll Andreas Helmcke Andreas Piesk @@ -59,6 +62,7 @@ Enes Yalcin Eric Bollengier Erich Prinz Eugene Sobolev +Evan J. Felix Evgeni Golov Felix Geyer Felix Schwarz @@ -72,8 +76,10 @@ Graham Keeling Grzegorz Grabowski Holger Weiss Howard Thomson +I Smith Jaime Ventura Jakub Hradil +Jakub Kramarz James Harper Jan Görig Jan Huisink @@ -111,6 +117,7 @@ Maik Aussendorf Marc Cousin Marc Schiffbauer Marcelo Medeiros +Marco Lertora Marco van Wieringen Martin Schmid Martin Simmons @@ -160,8 +167,10 @@ Sergey Svishchev Simon Krahé Simone Caronni Stefan Reddig +Stefan Warten Stephan Duehr Sébastien Marchal +Thomas Duemesnil Thomas Glatthor Thomas Lohman Thorsten Enge From ddc168f7770e6e320a66a95d04eca6370ca8c338 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 13 Jan 2020 12:04:23 +0100 Subject: [PATCH 25/25] docs: add Percona XtraBackup plugin to release notes --- docs/manuals/source/Appendix/ReleaseNotes.rst | 2 ++ docs/manuals/source/TasksAndConcepts/Plugins.rst | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/manuals/source/Appendix/ReleaseNotes.rst b/docs/manuals/source/Appendix/ReleaseNotes.rst index df8f85e98d6..993cdb8007f 100644 --- a/docs/manuals/source/Appendix/ReleaseNotes.rst +++ b/docs/manuals/source/Appendix/ReleaseNotes.rst @@ -57,6 +57,8 @@ New Features * Client initiated connection: Run dedicated jobs when a client connects to the |dir|. Introduced a new configuration directive RunOnIncomingConnectInterval, see the documentation here: :config:option:`dir/job/RunOnIncomingConnectInterval` * Python-bareos: Depending on the distribution, the Python module is packaged for Python 2 and/or Python 3. Previously it has only been packaged for Python 2 * Python-bareos: There are two variants of the Console protocol. The protocol used before Bareos-18.2 and the protocol used thereafter. The protocol since Bareos-18.2 supports TLS-PSK and PAM authentication, see :ref:`bareos-18.2.5`. Beginning with this version, Python-bareos also supports both protocols. As TLS-PSK for Python (module **sslpsk**) is not available for all platforms, Python-bareos has integrated an automatic fallback to the old protocol. +* Percona XtraBackup Plugin: The :ref:`PerconaXtrabackupPlugin` can be used to backup MySQL Databases. + It uses the command line tool *Percona XtraBackup* to create backups. The plugin was formerly part of the bareos-contrib source code repository. Changed Features ^^^^^^^^^^^^^^^^ diff --git a/docs/manuals/source/TasksAndConcepts/Plugins.rst b/docs/manuals/source/TasksAndConcepts/Plugins.rst index 7a4b8666512..28d5b3b64e5 100644 --- a/docs/manuals/source/TasksAndConcepts/Plugins.rst +++ b/docs/manuals/source/TasksAndConcepts/Plugins.rst @@ -987,8 +987,8 @@ can also be used to extract single files from the disk image. .. _PerconaXtrabackupPlugin: .. _backup-mysql-XtraBackup: -Backup of MySQL Databases using the Bareos MySQL Percona XtraBackup Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Percona XtraBackup Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~ :index:`\ ` :index:`\ `