Skip to content

Commit

Permalink
python plugins: enable io in core functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@pstorz @arogge
pstorz authored and arogge committed Dec 7, 2022
1 parent 83497b0 commit 8f1fe6e
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 88 deletions.
51 changes: 42 additions & 9 deletions core/src/filed/fd_plugins.cc
View file
Expand Up @@ -1225,6 +1225,7 @@ int PluginCreateFile(JobControlRecord* jcr,
rp.RegexWhere = jcr->RegexWhere;
rp.replace = jcr->fd_impl->replace;
rp.create_status = CF_ERROR;
rp.filedes = -1;

Dmsg4(debuglevel,
"call plugin createFile stream=%d type=%d LinkFI=%d File=%s\n",
Expand All @@ -1246,6 +1247,7 @@ int PluginCreateFile(JobControlRecord* jcr,
return CF_ERROR;
}

bfd->filedes = rp.filedes;
switch (rp.create_status) {
case CF_ERROR:
Qmsg1(jcr, M_ERROR, 0,
Expand Down Expand Up @@ -1326,6 +1328,7 @@ bool PluginSetAttributes(JobControlRecord* jcr,
if (IsBopen(ofd)) { bclose(ofd); }
PmStrcpy(attr->ofname, "*None*");
}
rp.filedes = -1;

return true;
}
Expand Down Expand Up @@ -1795,10 +1798,8 @@ void FreePlugins(JobControlRecord* jcr)
jcr->plugin_ctx_list = NULL;
}

/**
* Entry point for opening the file this is a wrapper around the pluginIO
* entry point in the plugin.
*/
/** Entry point for opening the file this is a wrapper around
the pluginIO entry point in the plugin. */
static int MyPluginBopen(BareosFilePacket* bfd,
const char* fname,
int flags,
Expand All @@ -1815,6 +1816,7 @@ static int MyPluginBopen(BareosFilePacket* bfd,
memset(&io, 0, sizeof(io));
io.pkt_size = sizeof(io);
io.pkt_end = sizeof(io);
io.filedes = -1;

io.func = IO_OPEN;
io.fname = fname;
Expand All @@ -1830,10 +1832,39 @@ static int MyPluginBopen(BareosFilePacket* bfd,
errno = io.io_errno;
bfd->lerror = io.lerror;
}

Dmsg1(debuglevel, "Return from plugin open status=%d\n", io.status);

return io.status;
// The plugin has two options for the read/write:
// 1.: - Set io.do_io_in_core to true, and
// - Set the io.filedes to the filedescriptor of the file that was
// opened in the plugin. In this case the core code will read from
// write to that filedescriptor during backup and restore.
// 2.: - Set io.do_io_in_core to false , and
// - Set/leave the io.filedes on/to -1. In this case the plugin code
// itself will be called for every read/write during backup and
// restore.

bfd->do_io_in_core = io.do_io_in_core;
bfd->filedes = io.filedes;
if (bfd->do_io_in_core) {
if (io.filedes != -1) {
Dmsg1(
debuglevel,
"bopen: plugin asks for core to do the read/write via filedescriptor "
"%d\n",
io.filedes);
return io.filedes;
} else {
Dmsg1(debuglevel,
"bopen: ERROR: plugin wants to do read/write itself but did not "
"return a valid filedescriptor: %d\n",
io.filedes);
return -2;
}
} else {
Dmsg1(debuglevel,
"bopen: plugin wants to do read/write itself. status: %d\n",
io.status);
return io.status;
}
}

/**
Expand All @@ -1853,6 +1884,7 @@ static int MyPluginBclose(BareosFilePacket* bfd)
memset(&io, 0, sizeof(io));
io.pkt_size = sizeof(io);
io.pkt_end = sizeof(io);
io.filedes = -1;

io.func = IO_CLOSE;

Expand All @@ -1867,7 +1899,7 @@ static int MyPluginBclose(BareosFilePacket* bfd)
}

Dmsg1(debuglevel, "plugin_bclose stat=%d\n", io.status);

bfd->filedes = -1;
return io.status;
}

Expand All @@ -1888,6 +1920,7 @@ static ssize_t MyPluginBread(BareosFilePacket* bfd, void* buf, size_t count)
memset(&io, 0, sizeof(io));
io.pkt_size = sizeof(io);
io.pkt_end = sizeof(io);
io.filedes = -1;

io.func = IO_READ;
io.count = count;
Expand Down
31 changes: 17 additions & 14 deletions core/src/filed/fd_plugins.h
View file
Expand Up @@ -124,6 +124,7 @@ struct restore_pkt {
int replace; /* Replace flag */
int create_status; /* Status from createFile() */
uint32_t delta_seq; /* Delta sequence number */
int filedes; /* file descriptor to read/write in core */
int32_t pkt_end; /* End packet sentinel */
};

Expand All @@ -137,20 +138,22 @@ enum
};

struct io_pkt {
int32_t pkt_size; /* Size of this packet */
int32_t func; /* Function code */
int32_t count; /* Read/write count */
int32_t flags; /* Open flags */
mode_t mode; /* Permissions for created files */
char* buf; /* Read/write buffer */
const char* fname; /* Open filename */
int32_t status; /* Return status */
int32_t io_errno; /* Errno code */
int32_t lerror; /* Win32 error code */
int32_t whence; /* Lseek argument */
boffset_t offset; /* Lseek argument */
bool win32; /* Win32 GetLastError returned */
int32_t pkt_end; /* End packet sentinel */
int32_t pkt_size; /* Size of this packet */
int32_t func; /* Function code */
int32_t count; /* Read/write count */
int32_t flags; /* Open flags */
mode_t mode; /* Permissions for created files */
char* buf; /* Read/write buffer */
const char* fname; /* Open filename */
int32_t status; /* Return status */
int32_t io_errno; /* Errno code */
int32_t lerror; /* Win32 error code */
int32_t whence; /* Lseek argument */
boffset_t offset; /* Lseek argument */
bool win32; /* Win32 GetLastError returned */
int filedes; /* file descriptor to read/write in core */
bool do_io_in_core; /* do io in core */
int32_t pkt_end; /* End packet sentinel */
};

struct acl_pkt {
Expand Down
20 changes: 7 additions & 13 deletions core/src/filed/restore.cc
View file
Expand Up @@ -569,13 +569,11 @@ void DoRestore(JobControlRecord* jcr)

BuildAttrOutputFnames(jcr, attr);

/*
* Try to actually create the file, which returns a status telling
* us if we need to extract or not.
*/
/* Try to actually create the file, which returns a status telling
* us if we need to extract or not. */
jcr->fd_impl->num_files_examined++;
rctx.extract = false;
status = CF_CORE; /* By default, let Bareos's core handle it */
status = CF_CORE; /* By default, let Bareos' core handle it */

if (jcr->IsPlugin()) {
status
Expand Down Expand Up @@ -610,10 +608,8 @@ void DoRestore(JobControlRecord* jcr)
FromBase64(&rsrc_len, attr->attrEx);
if (attr->type == FT_REG && rsrc_len > 0) { rctx.extract = true; }

/*
* Do not count the resource forks as regular files being
* restored.
*/
/* Do not count the resource forks as regular files being
* restored. */
if (rsrc_len == 0) { jcr->JobFiles++; }
} else {
jcr->JobFiles++;
Expand Down Expand Up @@ -1301,10 +1297,8 @@ int32_t ExtractData(JobControlRecord* jcr,
// If extracting, close any previous stream
static bool ClosePreviousStream(JobControlRecord* jcr, r_ctx& rctx)
{
/*
* If extracting, it was from previous stream, so
* close the output file and validate the signature.
*/
/* If extracting, it was from previous stream, so
* close the output file and validate the signature. */
if (rctx.extract) {
if (rctx.size > 0 && !IsBopen(&rctx.bfd)) {
Jmsg0(rctx.jcr, M_ERROR, 0,
Expand Down
43 changes: 28 additions & 15 deletions core/src/findlib/bfile.cc
View file
Expand Up @@ -861,8 +861,13 @@ ssize_t bread(BareosFilePacket* bfd, void* buf, size_t count)
{
bfd->rw_bytes = 0;

if (bfd->cmd_plugin && plugin_bread) { return plugin_bread(bfd, buf, count); }

if (bfd->cmd_plugin && plugin_bread) {
// invalid filehandle -> plugin does read
if (bfd->fh == INVALID_HANDLE_VALUE) {
return plugin_bread(bfd, buf, count);
}
Dmsg1(400, "bread handled in core via bfd->fh=%d\n", bfd->fh);
}
if (bfd->use_backup_api) {
if (!p_BackupRead(bfd->fh, (BYTE*)buf, count, &bfd->rw_bytes,
0, /* no Abort */
Expand Down Expand Up @@ -890,9 +895,14 @@ ssize_t bwrite(BareosFilePacket* bfd, void* buf, size_t count)
bfd->rw_bytes = 0;

if (bfd->cmd_plugin && plugin_bwrite) {
return plugin_bwrite(bfd, buf, count);
// invalid filehandle -> plugin does read
if (bfd->fh == INVALID_HANDLE_VALUE) {
return plugin_bwrite(bfd, buf, count);
}
Dmsg1(400, "bwrite handled in core via bfd->fh=%d\n", bfd->fh);
}


if (bfd->use_backup_api) {
if (!p_BackupWrite(bfd->fh, (BYTE*)buf, count, &bfd->rw_bytes,
0, /* No abort */
Expand Down Expand Up @@ -1046,9 +1056,9 @@ int bopen(BareosFilePacket* bfd,

if (bfd->cmd_plugin && plugin_bopen) {
Dmsg1(400, "call plugin_bopen fname=%s\n", fname);
bfd->filedes = plugin_bopen(bfd, fname, flags, mode);
Dmsg1(400, "Plugin bopen stat=%d\n", bfd->filedes);
return bfd->filedes;
int retval = plugin_bopen(bfd, fname, flags, mode);
Dmsg1(400, "Plugin bopen stat=%d\n", retval);
return retval;
}

/* Normal file open */
Expand Down Expand Up @@ -1123,6 +1133,7 @@ int bclose(BareosFilePacket* bfd)
if (bfd->cmd_plugin && plugin_bclose) {
status = plugin_bclose(bfd);
bfd->filedes = -1;
bfd->do_io_in_core = false;
bfd->cmd_plugin = false;
} else {
# if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
Expand All @@ -1139,30 +1150,32 @@ int bclose(BareosFilePacket* bfd)
bfd->BErrNo = errno;
bfd->filedes = -1;
bfd->cmd_plugin = false;
bfd->do_io_in_core = false;
}

return status;
}

ssize_t bread(BareosFilePacket* bfd, void* buf, size_t count)
{
ssize_t status;

if (bfd->cmd_plugin && plugin_bread) { return plugin_bread(bfd, buf, count); }
if (bfd->cmd_plugin && plugin_bread)
// plugin does read/write
if (!bfd->do_io_in_core) { return plugin_bread(bfd, buf, count); }

status = read(bfd->filedes, buf, count);
Dmsg1(400, "bread handled in core via bfd->filedes=%d\n", bfd->filedes);
ssize_t status = read(bfd->filedes, buf, count);
bfd->BErrNo = errno;
return status;
}

ssize_t bwrite(BareosFilePacket* bfd, void* buf, size_t count)
{
ssize_t status;
if (bfd->cmd_plugin && plugin_bwrite)
// plugin does read/write
if (!bfd->do_io_in_core) { return plugin_bwrite(bfd, buf, count); }

if (bfd->cmd_plugin && plugin_bwrite) {
return plugin_bwrite(bfd, buf, count);
}
status = write(bfd->filedes, buf, count);
Dmsg1(400, "bwrite handled in core via bfd->filedes=%d\n", bfd->filedes);
ssize_t status = write(bfd->filedes, buf, count);
bfd->BErrNo = errno;
return status;
}
Expand Down
8 changes: 4 additions & 4 deletions core/src/findlib/bfile.h
View file
Expand Up @@ -76,7 +76,7 @@ struct BareosFilePacket {
bool encrypted = false; /**< set if using ReadEncryptedFileRaw/WriteEncryptedFileRaw */
int mode = BF_CLOSED; /**< set if file is open */
HANDLE fh = INVALID_HANDLE_VALUE; /**< Win32 file handle */
int filedes = 0; /**< fd if doing Unix style */
int filedes = 0; /**< filedes if doing Unix style */
LPVOID lplugin_private_context = nullptr; /**< BackupRead/Write context */
PVOID pvContext = nullptr; /**< Encryption context */
POOLMEM* errmsg = nullptr; /**< error message buffer */
Expand All @@ -87,10 +87,10 @@ struct BareosFilePacket {
JobControlRecord* jcr = nullptr; /**< jcr for editing job codes */
PROCESS_WIN32_BACKUPAPIBLOCK_CONTEXT win32Decomplugin_private_context{}; /**< context for decomposition
of win32 backup streams */
int use_backup_decomp = 0; /**< set if using BackupRead Stream Decomposition */
int use_backup_decomp = 0; /**< set if using BackupRead Stream Decomposition */
bool reparse_point = false; /**< set if reparse point */
bool cmd_plugin = false; /**< set if we have a command plugin */
bool do_io_in_core{false}; /**< set if core should read/write from/to filedes */
bool do_io_in_core{false}; /**< set if core should read/write from/to filedes */
};
/* clang-format on */

Expand All @@ -111,7 +111,7 @@ struct BareosFilePacket {
int filedes{0}; /**< filedescriptor on Unix */
int flags_{0}; /**< open flags */
int BErrNo{0}; /**< errno */
int32_t lerror{0}; /**< not used - simplies Win32 builds */
int32_t lerror{0}; /**< not used - simplifies Win32 builds */
boffset_t offset{0}; /**< Delta offset */
JobControlRecord* jcr{nullptr}; /**< jcr for editing job codes */
PROCESS_WIN32_BACKUPAPIBLOCK_CONTEXT win32Decomplugin_private_context{}; /**< context for decomposition
Expand Down
6 changes: 2 additions & 4 deletions core/src/findlib/create_file.cc
View file
Expand Up @@ -437,10 +437,8 @@ int CreateFile(JobControlRecord* jcr,
if (!makepath(attr, attr->ofname, new_mode, parent_mode, uid, gid, 0)) {
return CF_ERROR;
}
/*
* If we are using the Win32 Backup API, we open the directory so
* that the security info will be read and saved.
*/
/* If we are using the Win32 Backup API, we open the directory so
* that the security info will be read and saved. */
if (!IsPortableBackup(bfd)) {
if (IsBopen(bfd)) {
Qmsg1(jcr, M_ERROR, 0, _("bpkt already open filedes=%d\n"),
Expand Down

0 comments on commit 8f1fe6e

Please sign in to comment.