Skip to content

Commit

Permalink
Merge pull request #502 from bareos/dev/maik/master/postgres-backup-c…
Browse files Browse the repository at this point in the history 
…leanup

Python Postgres Plugin
  • Loading branch information
@franku
franku committed May 7, 2020
2 parents e87e0fc + a9523ed commit b1a7943
Show file tree
Hide file tree
Showing 45 changed files with 1,894 additions and 84 deletions.
15 changes: 15 additions & 0 deletions core/platforms/packaging/bareos.spec
View file
Expand Up @@ -577,6 +577,11 @@ Requires: python-pycurl
Requires: python-lxml
Requires: python-ovirt-engine-sdk4

%package filedaemon-postgresql-python-plugin
Summary: PostgreSQL Python plugin for Bareos File daemon
Group: Productivity/Archiving/Backup
Requires: bareos-filedaemon = %{version}

%package filedaemon-percona-xtrabackup-python-plugin
Summary: Percona xtrabackup Python plugin for Bareos File daemon
Group: Productivity/Archiving/Backup
Expand Down Expand Up @@ -609,6 +614,11 @@ This package contains the LDAP python plugin for the file daemon

This package contains the Ovirt python plugin for the file daemon

%description filedaemon-postgresql-python-plugin
%{dscr}

This package contains the PostgreSQL python plugin for the file daemon

%description filedaemon-percona-xtrabackup-python-plugin
%{dscr}

Expand Down Expand Up @@ -1402,6 +1412,11 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d
%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/fileset/plugin-ovirt.conf.example
%attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/%{name}/bareos-dir.d/job/backup-ovirt.conf.example

%files filedaemon-postgresql-python-plugin
%defattr(-, root, root)
%{plugin_dir}/BareosFdPluginPostgres.py*
%{plugin_dir}/bareos-fd-postgres.py*

%files filedaemon-percona-xtrabackup-python-plugin
%defattr(-, root, root)
%{plugin_dir}/bareos-fd-percona-xtrabackup.py*
Expand Down
215 changes: 144 additions & 71 deletions core/src/plugins/filed/BareosFdPluginBaseclass.py
View file
Expand Up @@ -58,6 +58,7 @@ def __init__(self, context, plugindef, mandatory_options=None):
self.jobName = bareosfd.GetValue(context, bVariable["bVarJobName"])
self.workingdir = bareosfd.GetValue(context, bVariable["bVarWorkingDir"])
self.FNAME = "undef"
self.filetype = "undef"
self.file = None
bareosfd.DebugMessage(
context, 100, "FDName = %s - BareosFdPluginBaseclass\n" % (self.fdname)
Expand Down Expand Up @@ -113,10 +114,8 @@ def check_options(self, context, mandatory_options=None):
Here we just verify that eventual mandatory options are set.
If you have more to veriy, just overwrite ths method in your class
"""

if mandatory_options is None:
return bRCs["bRC_OK"]

for option in mandatory_options:
if option not in self.options:
bareosfd.DebugMessage(
Expand All @@ -128,99 +127,168 @@ def check_options(self, context, mandatory_options=None):
"Mandatory option '%s' not defined.\n" % (option),
)
return bRCs["bRC_Error"]

bareosfd.DebugMessage(
context, 100, "Using Option %s=%s\n" % (option, self.options[option])
)

return bRCs["bRC_OK"]

def plugin_io(self, context, IOP):
bareosfd.DebugMessage(
context, 100, "plugin_io called with function %s\n" % (IOP.func)
)
bareosfd.DebugMessage(context, 100, "FNAME is set to %s\n" % (self.FNAME))

if IOP.func == bIOPS["IO_OPEN"]:
self.FNAME = IOP.fname
try:
if IOP.flags & (os.O_CREAT | os.O_WRONLY):
bareosfd.DebugMessage(
context,
100,
"Open file %s for writing with %s\n" % (self.FNAME, IOP),
)

dirname = os.path.dirname(self.FNAME)
if not os.path.exists(dirname):
bareosfd.DebugMessage(
context,
100,
"Directory %s does not exist, creating it now\n"
% (dirname),
)
os.makedirs(dirname)
self.file = open(self.FNAME, "wb")
else:
def plugin_io_open(self, context, IOP):
self.FNAME = IOP.fname
if os.path.isdir(self.FNAME):
bareosfd.DebugMessage(context, 100, "%s is a directory\n" % (IOP.fname))
self.fileType = "FT_DIR"
bareosfd.DebugMessage(
context,
100,
"Did not open file %s of type %s\n" % (self.FNAME, self.fileType),
)
return bRCs["bRC_OK"]
elif os.path.islink(self.FNAME):
self.fileType = "FT_LNK"
bareosfd.DebugMessage(
context,
100,
"Did not open file %s of type %s\n" % (self.FNAME, self.fileType),
)
return bRCs["bRC_OK"]
else:
self.fileType = "FT_REG"
bareosfd.DebugMessage(
context,
150,
"file %s has type %s - trying to open it\n"
% (self.FNAME, self.fileType),
)
try:
if IOP.flags & (os.O_CREAT | os.O_WRONLY):
bareosfd.DebugMessage(
context,
100,
"Open file %s for writing with %s\n" % (self.FNAME, IOP),
)
dirname = os.path.dirname(self.FNAME)
if not os.path.exists(dirname):
bareosfd.DebugMessage(
context,
100,
"Open file %s for reading with %s\n" % (self.FNAME, IOP),
"Directory %s does not exist, creating it now\n" % (dirname),
)
self.file = open(self.FNAME, "rb")
except:
IOP.status = -1
return bRCs["bRC_Error"]

return bRCs["bRC_OK"]
os.makedirs(dirname)
self.file = open(self.FNAME, "wb")
else:
bareosfd.DebugMessage(
context,
100,
"Open file %s for reading with %s\n" % (self.FNAME, IOP),
)
self.file = open(self.FNAME, "rb")
except:
IOP.status = -1
return bRCs["bRC_Error"]
return bRCs["bRC_OK"]

elif IOP.func == bIOPS["IO_CLOSE"]:
bareosfd.DebugMessage(context, 100, "Closing file " + "\n")
def plugin_io_close(self, context, IOP):
bareosfd.DebugMessage(context, 100, "Closing file " + "\n")
if self.fileType == "FT_REG":
self.file.close()
return bRCs["bRC_OK"]
return bRCs["bRC_OK"]

elif IOP.func == bIOPS["IO_SEEK"]:
return bRCs["bRC_OK"]
def plugin_io_seek(self, context, IOP):
return bRCs["bRC_OK"]

elif IOP.func == bIOPS["IO_READ"]:
def plugin_io_read(self, context, IOP):
if self.fileType == "FT_REG":
bareosfd.DebugMessage(
context, 200, "Reading %d from file %s\n" % (IOP.count, self.FNAME)
)
IOP.buf = bytearray(IOP.count)
IOP.status = self.file.readinto(IOP.buf)
IOP.io_errno = 0
return bRCs["bRC_OK"]

elif IOP.func == bIOPS["IO_WRITE"]:
try:
IOP.status = self.file.readinto(IOP.buf)
IOP.io_errno = 0
except Exception as e:
bareosfd.JobMessage(
context,
bJobMessageType["M_ERROR"],
"Could net read %d bytes from file %s. \"%s\"" % (IOP.count, file_to_backup, e.message),
)
IOP.io_errno = e.errno
return bRCs["bRC_Error"]
else:
bareosfd.DebugMessage(
context, 200, "Writing buffer to file %s\n" % (self.FNAME)
context,
100,
"Did not read from file %s of type %s\n" % (self.FNAME, self.fileType),
)
self.file.write(IOP.buf)
IOP.status = IOP.count
IOP.buf = bytearray()
IOP.status = 0
IOP.io_errno = 0
return bRCs["bRC_OK"]
return bRCs["bRC_OK"]

def plugin_io_write(self, context, IOP):
bareosfd.DebugMessage(
context, 200, "Writing buffer to file %s\n" % (self.FNAME)
)
try:
self.file.write(IOP.buf)
except Exception as e:
bareosfd.JobMessage(
context,
bJobMessageType["M_ERROR"],
"Could net write to file %s. \"%s\"" % (file_to_backup, e.message),
)
IOP.io_errno = e.errno
IOP.status = 0
return bRCs["bRC_Error"]
IOP.status = IOP.count
IOP.io_errno = 0
return bRCs["bRC_OK"]

def plugin_io(self, context, IOP):
"""
Basic IO operations. Some tweaks here: IOP.fname is only set on file-open
We need to capture it on open and keep it for the remaining procedures
Now (since 2020) separated into sub-methods to ease overloading in derived classes
"""
bareosfd.DebugMessage(
context,
250,
"plugin_io called with function %s filename %s\n" % (IOP.func, IOP.fname),
)
bareosfd.DebugMessage(context, 250, "self.FNAME is set to %s\n" % (self.FNAME))
if IOP.func == bIOPS["IO_OPEN"]:
return self.plugin_io_open(context, IOP)
elif IOP.func == bIOPS["IO_CLOSE"]:
return self.plugin_io_close(context, IOP)
elif IOP.func == bIOPS["IO_SEEK"]:
return self.plugin_io_seek(context, IOP)
elif IOP.func == bIOPS["IO_READ"]:
return self.plugin_io_read(context, IOP)
elif IOP.func == bIOPS["IO_WRITE"]:
return self.plugin_io_write(context, IOP)

def handle_plugin_event(self, context, event):
if event == bEventType["bEventJobEnd"]:
bareosfd.DebugMessage(
context, 100, "handle_plugin_event called with bEventJobEnd\n"
)
return self.end_job(context)

elif event == bEventType["bEventEndBackupJob"]:
bareosfd.DebugMessage(
context, 100, "handle_plugin_event called with bEventEndBackupJob\n"
)
return self.end_backup_job(context)

elif event == bEventType["bEventEndFileSet"]:
bareosfd.DebugMessage(
context, 100, "handle_plugin_event called with bEventEndFileSet\n"
)
return self.end_fileset(context)

elif event == bEventType["bEventStartBackupJob"]:
bareosfd.DebugMessage(
context, 100, "handle_plugin_event() called with bEventStartBackupJob\n"
)

return self.start_backup_job(context)

elif event == bEventType["bEventStartRestoreJob"]:
Expand All @@ -229,14 +297,12 @@ def handle_plugin_event(self, context, event):
100,
"handle_plugin_event() called with bEventStartRestoreJob\n",
)

return self.start_restore_job(context)

else:
bareosfd.DebugMessage(
context, 100, "handle_plugin_event called with event %s\n" % (event)
)

return bRCs["bRC_OK"]

def start_backup_job(self, context):
Expand All @@ -246,6 +312,20 @@ def start_backup_job(self, context):
"""
return bRCs["bRC_OK"]

def end_job(self, context):
"""
Called if job ends regularyly (not for cancelled jobs)
Overload this to arrange whatever you have to do at this time.
"""
return bRCs["bRC_OK"]

def end_backup_job(self, context):
"""
Called if backup job ends, before ClientAfterJob
Overload this to arrange whatever you have to do at this time.
"""
return bRCs["bRC_OK"]

def start_backup_file(self, context, savepkt):
"""
Base method, we do not add anything, overload this method with your
Expand All @@ -260,6 +340,12 @@ def end_backup_file(self, context):
)
return bRCs["bRC_OK"]

def end_fileset(self, context):
bareosfd.DebugMessage(
context, 100, "end_fileset() entry point in Python called\n"
)
return bRCs["bRC_OK"]

def start_restore_job(self, context):
"""
Start of Restore Job. Called , if you have Restore objects.
Expand Down Expand Up @@ -298,21 +384,8 @@ def create_file(self, context, restorepkt):
100,
"create_file() entry point in Python called with %s\n" % (restorepkt),
)
FNAME = restorepkt.ofname
dirname = os.path.dirname(FNAME)
if not os.path.exists(dirname):
bareosfd.DebugMessage(
context, 200, "Directory %s does not exist, creating it now\n" % dirname
)
os.makedirs(dirname)
# open creates the file, if not yet existing, we close it again right
# aways it will be opened again in plugin_io.
# But: only do this for regular files, prevent from
# IOError: (21, 'Is a directory', '/tmp/bareos-restores/my/dir/')
# if it's a directory
if restorepkt.type == bFileType["FT_REG"]:
open(FNAME, "wb").close()
restorepkt.create_status = bCFs["CF_EXTRACT"]
# We leave file creation up to the core for the default case
restorepkt.create_status = bCFs["CF_CORE"]
return bRCs["bRC_OK"]

def set_file_attributes(self, context, restorepkt):
Expand Down

0 comments on commit b1a7943

Please sign in to comment.