From a8d827006b000fc224708f16083e4c25b2bf2ab0 Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Tue, 1 Sep 2015 20:44:24 +0200 Subject: [PATCH] Secure erase of files. Some customers need for compliance reasons the ability that we use a secure method for erasing files. This patch adds such ability by allowing an external program to be invoked to perform the secure erase. Strict industry standards and government regulations are in place that force organizations to mitigate the risk of unauthorized exposure of confidential corporate and government data. Regulations in the United States include HIPAA (Health Insurance Portability and Accountability Act); FACTA (The Fair and Accurate Credit Transactions Act of 2003); GLB (Gramm-Leach Bliley); Sarbanes-Oxley Act (SOx); and Payment Card Industry Data Security Standards (PCI DSS) and the Data Protection Act in the United Kingdom. Failure to comply can result in fines and damage to company reputation, as well as civil and criminal liability. Data erasure may not work completely on flash based media, such as Solid State Drives and USB Flash Drives, as these devices can store remnant data which is inaccessible to the erasure technique, and data can be retrieved from the individual flash memory chips inside the device. Data erasure through overwriting only works on hard drives that are functioning and writing to all sectors. Bad sectors cannot usually be overwritten, but may contain recoverable information. Bad sectors, however, may be invisible to the host system and thus to the erasing software. Disk encryption before use prevents this problem. Software-driven data erasure could also be compromised by malicious code. --- src/dird/dird.c | 7 +- src/dird/dird_conf.c | 7 +- src/dird/dird_conf.h | 1 + src/dird/msgchan.c | 2 +- src/dird/ndmp_dma.c | 2 +- src/dird/restore.c | 2 +- src/dird/ua_status.c | 6 +- src/dird/verify.c | 2 +- src/filed/accurate_lmdb.c | 2 +- src/filed/dir_cmd.c | 2 +- src/filed/filed.c | 3 + src/filed/filed_conf.c | 4 + src/filed/filed_conf.h | 1 + src/filed/status.c | 6 + src/findlib/create_file.c | 6 +- src/lib/bpipe.c | 4 - src/lib/bsys.c | 128 ++++++++++++++---- src/lib/crypto_cache.c | 6 +- src/lib/protos.h | 2 + src/lib/signal.c | 2 +- src/stored/backends/unix_file_device.c | 97 +++++++------ src/stored/dir_cmd.c | 4 +- src/stored/job.c | 2 +- src/stored/spool.c | 114 ++++++++++------ src/stored/status.c | 6 + src/stored/stored.c | 5 +- src/stored/stored_conf.c | 4 + src/stored/stored_conf.h | 1 + src/win32/stored/backends/win32_file_device.c | 2 +- 29 files changed, 296 insertions(+), 134 deletions(-) diff --git a/src/dird/dird.c b/src/dird/dird.c index 0c1cb5a820a..bbc88d45d6e 100644 --- a/src/dird/dird.c +++ b/src/dird/dird.c @@ -1000,7 +1000,10 @@ static bool check_resources() UnlockRes(); if (OK) { close_msg(NULL); /* close temp message handler */ - init_msg(NULL, me->messages); /* open daemon message handler */ + init_msg(NULL, me->messages); /* open daemon message handler */ + if (me->secure_erase_cmdline) { + set_secure_erase_cmdline(me->secure_erase_cmdline); + } } bail_out: @@ -1278,7 +1281,7 @@ static void cleanup_old_files() pm_strcpy(cleanup, basename); pm_strcat(cleanup, result->d_name); Dmsg1(100, "Unlink: %s\n", cleanup); - unlink(cleanup); + secure_erase(NULL, cleanup); } } diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 6c864b4d7ca..dd2e0dd23c5 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -145,8 +145,8 @@ static RES_ITEM dir_items[] = { { "NdmpLogLevel", CFG_TYPE_PINT32, ITEM(res_dir.ndmp_loglevel), 0, CFG_ITEM_DEFAULT, "4", "13.2.0-", NULL }, { "AbsoluteJobTimeout", CFG_TYPE_PINT32, ITEM(res_dir.jcr_watchdog_time), 0, 0, NULL, "14.2.0-", NULL }, { "Auditing", CFG_TYPE_BOOL, ITEM(res_dir.auditing), 0, CFG_ITEM_DEFAULT, "false", "14.2.0-", NULL }, - { "AuditEvents", CFG_TYPE_AUDIT, ITEM(res_dir.audit_events), 0, 0, NULL, - "14.2.0-", NULL }, + { "AuditEvents", CFG_TYPE_AUDIT, ITEM(res_dir.audit_events), 0, 0, NULL, "14.2.0-", NULL }, + { "SecureEraseCommand", CFG_TYPE_STR, ITEM(res_dir.secure_erase_cmdline), 0, 0, NULL, "15.2.1-", NULL }, { NULL, 0, { 0 }, 0, 0, NULL, NULL, NULL } }; @@ -2386,6 +2386,9 @@ void free_resource(RES *sres, int type) if (res->res_dir.audit_events) { delete res->res_dir.audit_events; } + if (res->res_dir.secure_erase_cmdline) { + free(res->res_dir.secure_erase_cmdline); + } break; case R_DEVICE: case R_COUNTER: diff --git a/src/dird/dird_conf.h b/src/dird/dird_conf.h index d7ab49d8e96..3d6c74fda33 100644 --- a/src/dird/dird_conf.h +++ b/src/dird/dird_conf.h @@ -145,6 +145,7 @@ class DIRRES: public BRSRES { uint32_t jcr_watchdog_time; /* Absolute time after which a Job gets terminated regardless of its progress */ uint32_t stats_collect_interval; /* Statistics collect interval in seconds */ char *verid; /* Custom Id to print in version command */ + char *secure_erase_cmdline; /* Cmdline to execute to perform secure erase of file */ s_password keyencrkey; /* Key Encryption Key */ }; diff --git a/src/dird/msgchan.c b/src/dird/msgchan.c index 5a76976e0f2..ad7cb1fd7b2 100644 --- a/src/dird/msgchan.c +++ b/src/dird/msgchan.c @@ -129,7 +129,7 @@ static inline bool send_bootstrap_file_to_sd(JCR *jcr, BSOCK *sd) sd->signal(BNET_EOD); fclose(bs); if (jcr->unlink_bsr) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); jcr->unlink_bsr = false; } return true; diff --git a/src/dird/ndmp_dma.c b/src/dird/ndmp_dma.c index e153bae76c3..7883a8af940 100644 --- a/src/dird/ndmp_dma.c +++ b/src/dird/ndmp_dma.c @@ -2833,7 +2833,7 @@ void ndmp_restore_cleanup(JCR *jcr, int TermCode) update_job_end(jcr, TermCode); if (jcr->unlink_bsr && jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); jcr->unlink_bsr = false; } diff --git a/src/dird/restore.c b/src/dird/restore.c index 45005afd6d0..ed613e040b8 100644 --- a/src/dird/restore.c +++ b/src/dird/restore.c @@ -457,7 +457,7 @@ void native_restore_cleanup(JCR *jcr, int TermCode) update_job_end(jcr, TermCode); if (jcr->unlink_bsr && jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); jcr->unlink_bsr = false; } diff --git a/src/dird/ua_status.c b/src/dird/ua_status.c index 5a6c5519763..5912a023ca8 100644 --- a/src/dird/ua_status.c +++ b/src/dird/ua_status.c @@ -397,7 +397,6 @@ void list_dir_status_header(UAContext *ua) dbdrivers.strcat(catalog->db_driver); cnt++; } - ua->send_msg(_("%s Version: %s (%s) %s %s %s\n"), my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER); bstrftime_nc(dt, sizeof(dt), daemon_start_time); @@ -409,6 +408,11 @@ void list_dir_status_header(UAContext *ua) edit_uint64_with_commas(sm_max_bytes, b3), edit_uint64_with_commas(sm_buffers, b4), edit_uint64_with_commas(sm_max_buffers, b5)); + + if (me->secure_erase_cmdline) { + ua->send_msg(_(" secure erase command='%s'\n"), me->secure_erase_cmdline); + } + len = list_dir_plugins(msg); if (len > 0) { ua->send_msg("%s\n", msg.c_str()); diff --git a/src/dird/verify.c b/src/dird/verify.c index d7d165fe06c..85de581c167 100644 --- a/src/dird/verify.c +++ b/src/dird/verify.c @@ -501,7 +501,7 @@ void verify_cleanup(JCR *jcr, int TermCode) } if (jcr->unlink_bsr && jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); jcr->unlink_bsr = false; } diff --git a/src/filed/accurate_lmdb.c b/src/filed/accurate_lmdb.c index d8d8946ea04..4882f76d5bd 100644 --- a/src/filed/accurate_lmdb.c +++ b/src/filed/accurate_lmdb.c @@ -570,7 +570,7 @@ void B_ACCURATE_LMDB::destroy(JCR *jcr) } if (m_lmdb_name) { - unlink(m_lmdb_name); + secure_erase(jcr, m_lmdb_name); free_pool_memory(m_lmdb_name); m_lmdb_name = NULL; } diff --git a/src/filed/dir_cmd.c b/src/filed/dir_cmd.c index cfeac04a584..113cb2978a3 100644 --- a/src/filed/dir_cmd.c +++ b/src/filed/dir_cmd.c @@ -1173,7 +1173,7 @@ static bool fileset_cmd(JCR *jcr) static void free_bootstrap(JCR *jcr) { if (jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); free_pool_memory(jcr->RestoreBootstrap); jcr->RestoreBootstrap = NULL; } diff --git a/src/filed/filed.c b/src/filed/filed.c index 0ccd11ba4db..7a1bc7ec74f 100644 --- a/src/filed/filed.c +++ b/src/filed/filed.c @@ -600,6 +600,9 @@ static bool check_resources() if (OK) { close_msg(NULL); /* close temp message handler */ init_msg(NULL, me->messages); /* open user specified message handler */ + if (me->secure_erase_cmdline) { + set_secure_erase_cmdline(me->secure_erase_cmdline); + } } return OK; diff --git a/src/filed/filed_conf.c b/src/filed/filed_conf.c index ebae753c75f..429de383e28 100644 --- a/src/filed/filed_conf.c +++ b/src/filed/filed_conf.c @@ -121,6 +121,7 @@ static RES_ITEM cli_items[] = { { "AbsoluteJobTimeout", CFG_TYPE_PINT32, ITEM(res_client.jcr_watchdog_time), 0, 0, NULL, NULL, NULL }, { "AlwaysUseLmdb", CFG_TYPE_BOOL, ITEM(res_client.always_use_lmdb), 0, CFG_ITEM_DEFAULT, "false", NULL, NULL }, { "LmdbThreshold", CFG_TYPE_PINT32, ITEM(res_client.lmdb_threshold), 0, 0, NULL, NULL, NULL }, + { "SecureEraseCommand", CFG_TYPE_STR, ITEM(res_client.secure_erase_cmdline), 0, 0, NULL, NULL, NULL }, { NULL, 0, { 0 }, 0, 0, NULL, NULL, NULL } }; @@ -360,6 +361,9 @@ void free_resource(RES *sres, int type) if (res->res_client.allowed_job_cmds) { delete res->res_client.allowed_job_cmds; } + if (res->res_client.secure_erase_cmdline) { + free(res->res_client.secure_erase_cmdline); + } break; case R_MSGS: if (res->res_msgs.mail_cmd) { diff --git a/src/filed/filed_conf.h b/src/filed/filed_conf.h index 1ee91441e53..67dec5e8cd5 100644 --- a/src/filed/filed_conf.h +++ b/src/filed/filed_conf.h @@ -116,6 +116,7 @@ class CLIENTRES : public BRSRES { alist *allowed_job_cmds; /* Only allow the following Job commands to be executed */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ char *verid; /* Custom Id to print in version command */ + char *secure_erase_cmdline; /* Cmdline to execute to perform secure erase of file */ uint64_t max_bandwidth_per_job; /* Bandwidth limitation (global) */ }; diff --git a/src/filed/status.c b/src/filed/status.c index 0804d0aaa8c..fa2e55c2079 100644 --- a/src/filed/status.c +++ b/src/filed/status.c @@ -150,6 +150,12 @@ static void list_status_header(STATUS_PKT *sp) "bwlimit=%skB/s\n"), sizeof(boffset_t), sizeof(size_t), debug_level, get_trace(), edit_uint64_with_commas(me->max_bandwidth_per_job / 1024, b1)); sendit(msg, len, sp); + + if (me->secure_erase_cmdline) { + len = Mmsg(msg, _(" secure erase command='%s'\n"), me->secure_erase_cmdline); + sendit(msg, len, sp); + } + len = list_fd_plugins(msg); if (len > 0) { sendit(msg, len, sp); diff --git a/src/findlib/create_file.c b/src/findlib/create_file.c index 4a316864dca..39f22841213 100644 --- a/src/findlib/create_file.c +++ b/src/findlib/create_file.c @@ -157,13 +157,15 @@ int create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace) if (exists && attr->type != FT_RAW && attr->type != FT_FIFO) { /* Get rid of old copy */ Dmsg1(400, "unlink %s\n", attr->ofname); - if (unlink(attr->ofname) == -1) { + if (secure_erase(jcr, attr->ofname) == -1) { berrno be; + Qmsg(jcr, M_ERROR, 0, _("File %s already exists and could not be replaced. ERR=%s.\n"), - attr->ofname, be.bstrerror()); + attr->ofname, be.bstrerror()); /* Continue despite error */ } } + /* * Here we do some preliminary work for all the above * types to create the path to the file if it does diff --git a/src/lib/bpipe.c b/src/lib/bpipe.c index 1158ef77d29..4149aeea93c 100644 --- a/src/lib/bpipe.c +++ b/src/lib/bpipe.c @@ -253,11 +253,7 @@ int close_bpipe(BPIPE *bpipe) } Dmsg1(800, "child status=%d\n", status & ~b_errno_exit); } else if (WIFSIGNALED(chldstatus)) { /* process died */ -#ifndef HAVE_WIN32 status = WTERMSIG(chldstatus); -#else - status = 1; /* fake child status */ -#endif Dmsg1(800, "Child died from signal %d\n", status); status |= b_errno_signal; /* exit signal returned */ } diff --git a/src/lib/bsys.c b/src/lib/bsys.c index 0f0f5e86088..1ec039f8efa 100644 --- a/src/lib/bsys.c +++ b/src/lib/bsys.c @@ -37,12 +37,13 @@ static pthread_mutex_t timer_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t timer = PTHREAD_COND_INITIALIZER; +static const char *secure_erase_cmdline = NULL; /* * This routine is a somewhat safer unlink in that it - * allows you to run a regex on the filename before - * excepting it. It also requires the file to be in - * the working directory. + * allows you to run a regex on the filename before + * excepting it. It also requires the file to be in + * the working directory. */ int safer_unlink(const char *pathname, const char *regx) { @@ -53,13 +54,17 @@ int safer_unlink(const char *pathname, const char *regx) regmatch_t pmatch[nmatch]; int rtn; - /* Name must start with working directory */ + /* + * Name must start with working directory + */ if (strncmp(pathname, working_directory, strlen(working_directory)) != 0) { Pmsg1(000, "Safe_unlink excluded: %s\n", pathname); return EROFS; } - /* Compile regex expression */ + /* + * Compile regex expression + */ rc = regcomp(&preg1, regx, REG_EXTENDED); if (rc != 0) { regerror(rc, &preg1, prbuf, sizeof(prbuf)); @@ -68,18 +73,86 @@ int safer_unlink(const char *pathname, const char *regx) return ENOENT; } - /* Unlink files that match regexes */ + /* + * Unlink files that match regexes + */ if (regexec(&preg1, pathname, nmatch, pmatch, 0) == 0) { Dmsg1(100, "safe_unlink unlinking: %s\n", pathname); - rtn = unlink(pathname); + rtn = secure_erase(NULL, pathname); } else { Pmsg2(000, "safe_unlink regex failed: regex=%s file=%s\n", regx, pathname); rtn = EROFS; } regfree(&preg1); + return rtn; } +/* + * This routine will use an external secure erase program to delete a file. + */ +int secure_erase(JCR *jcr, const char *pathname) +{ + int retval = -1; + + if (secure_erase_cmdline) { + int status; + BPIPE *bpipe; + POOL_MEM line(PM_NAME), + cmdline(PM_MESSAGE); + + Mmsg(cmdline,"%s \"%s\"", secure_erase_cmdline, pathname); + if (jcr) { + Jmsg(jcr, M_INFO, 0, _("secure_erase: executing %s\n"), cmdline.c_str()); + } + + bpipe = open_bpipe(cmdline.c_str(), 0, "r"); + if (bpipe == NULL) { + berrno be; + + if (jcr) { + Jmsg(jcr, M_FATAL, 0, _("secure_erase: %s could not execute. ERR=%s\n"), + secure_erase_cmdline, be.bstrerror()); + } + goto bail_out; + } + + while (fgets(line.c_str(), line.size(), bpipe->rfd)) { + strip_trailing_junk(line.c_str()); + if (jcr) { + Jmsg(jcr, M_INFO, 0, _("secure_erase: %s\n"), line.c_str()); + } + } + + status = close_bpipe(bpipe); + if (status != 0) { + berrno be; + + if (jcr) { + Jmsg(jcr, M_FATAL, 0, _("secure_erase: %s returned non-zero status=%d. ERR=%s\n"), + secure_erase_cmdline, be.code(status), be.bstrerror(status)); + } + goto bail_out; + } + + Dmsg0(100, "wpipe_command OK\n"); + retval = 0; + } else { + retval = unlink(pathname); + } + + return retval; + +bail_out: + errno = EROFS; + return retval; +} + +void set_secure_erase_cmdline(const char *cmdline) +{ + secure_erase_cmdline = cmdline; +} + /* * This routine will sleep (sec, microsec). Note, however, that if a * signal occurs, it will return early. It is up to the caller @@ -495,7 +568,7 @@ void b_memset(const char *file, int line, void *mem, int val, size_t num) #endif #if !defined(HAVE_WIN32) -static int del_pid_file_ok = FALSE; +static bool del_pid_file_ok = false; #endif /* @@ -557,7 +630,7 @@ void create_pid_file(char *dir, const char *progname, int port) len = sprintf(pidbuf, "%d\n", (int)getpid()); write(pidfd, pidbuf, len); close(pidfd); - del_pid_file_ok = TRUE; /* we created it so we can delete it */ + del_pid_file_ok = true; /* we created it so we can delete it */ } else { berrno be; Emsg2(M_ERROR_TERM, 0, _("Could not open pid file. %s ERR=%s\n"), fname, @@ -579,7 +652,7 @@ int delete_pid_file(char *dir, const char *progname, int port) free_pool_memory(fname); return 0; } - del_pid_file_ok = FALSE; + del_pid_file_ok = false; Mmsg(&fname, "%s/%s.%d.pid", dir, progname, port); unlink(fname); free_pool_memory(fname); @@ -613,23 +686,24 @@ void read_state_file(char *dir, const char *progname, int port) int hdr_size = sizeof(hdr); Mmsg(&fname, "%s/%s.%d.state", dir, progname, port); - /* If file exists, see what we have */ -// Dmsg1(10, "O_BINARY=%d\n", O_BINARY); + /* + * If file exists, see what we have + */ if ((sfd = open(fname, O_RDONLY|O_BINARY)) < 0) { berrno be; Dmsg3(010, "Could not open state file. sfd=%d size=%d: ERR=%s\n", - sfd, sizeof(hdr), be.bstrerror()); + sfd, sizeof(hdr), be.bstrerror()); goto bail_out; } if ((status = read(sfd, &hdr, hdr_size)) != hdr_size) { berrno be; Dmsg4(010, "Could not read state file. sfd=%d status=%d size=%d: ERR=%s\n", - sfd, (int)status, hdr_size, be.bstrerror()); + sfd, (int)status, hdr_size, be.bstrerror()); goto bail_out; } if (hdr.version != state_hdr.version) { Dmsg2(010, "Bad hdr version. Wanted %d got %d\n", - state_hdr.version, hdr.version); + state_hdr.version, hdr.version); goto bail_out; } hdr.id[13] = 0; @@ -637,7 +711,7 @@ void read_state_file(char *dir, const char *progname, int port) Dmsg0(000, "State file header id invalid.\n"); goto bail_out; } -// Dmsg1(010, "Read header of %d bytes.\n", sizeof(hdr)); + if (!read_last_jobs_list(sfd, hdr.last_jobs_addr)) { goto bail_out; } @@ -646,9 +720,11 @@ void read_state_file(char *dir, const char *progname, int port) if (sfd >= 0) { close(sfd); } + if (!ok) { - unlink(fname); - } + secure_erase(NULL, fname); + } + free_pool_memory(fname); } @@ -665,41 +741,45 @@ void write_state_file(char *dir, const char *progname, int port) P(state_mutex); /* Only one job at a time can call here */ Mmsg(&fname, "%s/%s.%d.state", dir, progname, port); - /* Create new state file */ - unlink(fname); + + /* + * Create new state file + */ + secure_erase(NULL, fname); if ((sfd = open(fname, O_CREAT|O_WRONLY|O_BINARY, 0640)) < 0) { berrno be; Dmsg2(000, "Could not create state file. %s ERR=%s\n", fname, be.bstrerror()); Emsg2(M_ERROR, 0, _("Could not create state file. %s ERR=%s\n"), fname, be.bstrerror()); goto bail_out; } + if (write(sfd, &state_hdr, sizeof(state_hdr)) != sizeof(state_hdr)) { berrno be; Dmsg1(000, "Write hdr error: ERR=%s\n", be.bstrerror()); goto bail_out; } -// Dmsg1(010, "Wrote header of %d bytes\n", sizeof(state_hdr)); + state_hdr.last_jobs_addr = sizeof(state_hdr); state_hdr.reserved[0] = write_last_jobs_list(sfd, state_hdr.last_jobs_addr); -// Dmsg1(010, "write last job end = %d\n", (int)state_hdr.reserved[0]); if (lseek(sfd, 0, SEEK_SET) < 0) { berrno be; Dmsg1(000, "lseek error: ERR=%s\n", be.bstrerror()); goto bail_out; } + if (write(sfd, &state_hdr, sizeof(state_hdr)) != sizeof(state_hdr)) { berrno be; Pmsg1(000, _("Write final hdr error: ERR=%s\n"), be.bstrerror()); goto bail_out; } ok = true; -// Dmsg1(010, "rewrote header = %d\n", sizeof(state_hdr)); bail_out: if (sfd >= 0) { close(sfd); } + if (!ok) { - unlink(fname); + secure_erase(NULL, fname); } V(state_mutex); free_pool_memory(fname); diff --git a/src/lib/crypto_cache.c b/src/lib/crypto_cache.c index 1622a8ca879..6c3fcc1181b 100644 --- a/src/lib/crypto_cache.c +++ b/src/lib/crypto_cache.c @@ -115,7 +115,7 @@ void read_crypto_cache(const char *cache_file) } if (!ok) { - unlink(cache_file); + secure_erase(NULL, cache_file); if (cached_crypto_keys) { cached_crypto_keys->destroy(); delete cached_crypto_keys; @@ -152,7 +152,7 @@ void write_crypto_cache(const char *cache_file) */ P(crypto_cache_lock); - unlink(cache_file); + secure_erase(NULL, cache_file); if ((fd = open(cache_file, O_CREAT | O_WRONLY | O_BINARY, 0640)) < 0) { berrno be; @@ -186,7 +186,7 @@ void write_crypto_cache(const char *cache_file) } if (!ok) { - unlink(cache_file); + secure_erase(NULL, cache_file); } V(crypto_cache_lock); diff --git a/src/lib/protos.h b/src/lib/protos.h index eb76502eecd..3154b016442 100644 --- a/src/lib/protos.h +++ b/src/lib/protos.h @@ -131,6 +131,8 @@ int Zdeflate(char *in, int in_len, char *out, int &out_len); int Zinflate(char *in, int in_len, char *out, int &out_len); void stack_trace(); int safer_unlink(const char *pathname, const char *regex); +int secure_erase(JCR *jcr, const char *pathname); +void set_secure_erase_cmdline(const char *cmdline); /* compression.c */ const char *cmprs_algo_to_text(uint32_t compression_algorithm); diff --git a/src/lib/signal.c b/src/lib/signal.c index fa5044bd0e2..387a9ca1267 100644 --- a/src/lib/signal.c +++ b/src/lib/signal.c @@ -193,7 +193,7 @@ extern "C" void signal_handler(int sig) Pmsg2(000, "chdir to %s failed. ERR=%s\n", working_directory, be.bstrerror()); strcpy((char *)working_directory, "/tmp/"); } - unlink("./core"); /* get rid of any old core file */ + secure_erase(NULL, "./core"); /* get rid of any old core file */ #ifdef DEVELOPER /* When DEVELOPER not set, this is done below */ /* diff --git a/src/stored/backends/unix_file_device.c b/src/stored/backends/unix_file_device.c index 12a2a662bc2..53cb8ff41a3 100644 --- a/src/stored/backends/unix_file_device.c +++ b/src/stored/backends/unix_file_device.c @@ -219,68 +219,77 @@ boffset_t unix_file_device::d_lseek(DCR *dcr, boffset_t offset, int whence) bool unix_file_device::d_truncate(DCR *dcr) { struct stat st; + POOL_MEM archive_name(PM_FNAME); - if (ftruncate(m_fd, 0) != 0) { - berrno be; + /* + * When secure erase is configured never truncate the file. + */ + if (!me->secure_erase_cmdline) { + if (ftruncate(m_fd, 0) != 0) { + berrno be; - Mmsg2(errmsg, _("Unable to truncate device %s. ERR=%s\n"), prt_name, be.bstrerror()); - return false; + Mmsg2(errmsg, _("Unable to truncate device %s. ERR=%s\n"), prt_name, be.bstrerror()); + return false; + } + + if (fstat(m_fd, &st) != 0) { + berrno be; + + Mmsg2(errmsg, _("Unable to stat device %s. ERR=%s\n"), prt_name, be.bstrerror()); + return false; + } + + if (st.st_size == 0) { + goto bail_out; + } + + Mmsg2(errmsg, _("Device %s doesn't support ftruncate(). Recreating file %s.\n"), + prt_name, archive_name.c_str()); } /* - * Check for a successful ftruncate() and issue a work-around for devices - * (mostly cheap NAS) that don't support truncation. * Workaround supplied by Martin Schmid as a solution to bug #1011. + * Used when secure erase is configured or when ftruncate() doesn't have the + * wanted result. As work around we perform the following steps: + * * 1. close file * 2. delete file * 3. open new file with same mode * 4. change ownership to original */ - if (fstat(m_fd, &st) != 0) { - berrno be; - - Mmsg2(errmsg, _("Unable to stat device %s. ERR=%s\n"), prt_name, be.bstrerror()); - return false; + pm_strcpy(archive_name, dev_name); + if (!IsPathSeparator(archive_name.c_str()[strlen(archive_name.c_str())-1])) { + pm_strcat(archive_name, "/"); } + pm_strcat(archive_name, dcr->VolumeName); - if (st.st_size != 0) { /* ftruncate() didn't work */ - POOL_MEM archive_name(PM_FNAME); - - pm_strcpy(archive_name, dev_name); - if (!IsPathSeparator(archive_name.c_str()[strlen(archive_name.c_str())-1])) { - pm_strcat(archive_name, "/"); - } - pm_strcat(archive_name, dcr->VolumeName); - - Mmsg2(errmsg, _("Device %s doesn't support ftruncate(). Recreating file %s.\n"), - prt_name, archive_name.c_str()); - - /* - * Close file and blow it away - */ - ::close(m_fd); - ::unlink(archive_name.c_str()); + /* + * Close file and blow it away + */ + ::close(m_fd); + secure_erase(dcr->jcr, archive_name.c_str()); - /* - * Recreate the file -- of course, empty - */ - oflags = O_CREAT | O_RDWR | O_BINARY; - if ((m_fd = ::open(archive_name.c_str(), oflags, st.st_mode)) < 0) { - berrno be; + /* + * Recreate the file -- of course, empty + */ + oflags = O_CREAT | O_RDWR | O_BINARY; + if ((m_fd = ::open(archive_name.c_str(), oflags, st.st_mode)) < 0) { + berrno be; - dev_errno = errno; - Mmsg2(errmsg, _("Could not reopen: %s, ERR=%s\n"), archive_name.c_str(), be.bstrerror()); - Dmsg1(100, "reopen failed: %s", errmsg); - Emsg0(M_FATAL, 0, errmsg); - return false; - } + dev_errno = errno; + Mmsg2(errmsg, _("Could not reopen: %s, ERR=%s\n"), archive_name.c_str(), be.bstrerror()); + Dmsg1(100, "reopen failed: %s", errmsg); + Emsg0(M_FATAL, 0, errmsg); - /* - * Reset proper owner - */ - chown(archive_name.c_str(), st.st_uid, st.st_gid); + return false; } + /* + * Reset proper owner + */ + chown(archive_name.c_str(), st.st_uid, st.st_gid); + +bail_out: return true; } diff --git a/src/stored/dir_cmd.c b/src/stored/dir_cmd.c index 4f9197d9474..f80ecda5988 100644 --- a/src/stored/dir_cmd.c +++ b/src/stored/dir_cmd.c @@ -1203,7 +1203,7 @@ static inline bool get_bootstrap_file(JCR *jcr, BSOCK *sock) bool ok = false; if (jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); free_pool_memory(jcr->RestoreBootstrap); } P(bsr_mutex); @@ -1240,7 +1240,7 @@ static inline bool get_bootstrap_file(JCR *jcr, BSOCK *sock) ok = true; bail_out: - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); free_pool_memory(jcr->RestoreBootstrap); jcr->RestoreBootstrap = NULL; if (!ok) { diff --git a/src/stored/job.c b/src/stored/job.c index 1d924bc9993..f4cc23fa6f7 100644 --- a/src/stored/job.c +++ b/src/stored/job.c @@ -515,7 +515,7 @@ void stored_free_jcr(JCR *jcr) */ free_restore_volume_list(jcr); if (jcr->RestoreBootstrap) { - unlink(jcr->RestoreBootstrap); + secure_erase(jcr, jcr->RestoreBootstrap); free_pool_memory(jcr->RestoreBootstrap); jcr->RestoreBootstrap = NULL; } diff --git a/src/stored/spool.c b/src/stored/spool.c index 431593b9d98..98e411b4431 100644 --- a/src/stored/spool.c +++ b/src/stored/spool.c @@ -2,6 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2004-2012 Free Software Foundation Europe e.V. + Copyright (C) 2015-2015 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -30,7 +31,7 @@ /* Forward referenced subroutines */ static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name); static bool open_data_spool_file(DCR *dcr); -static bool close_data_spool_file(DCR *dcr); +static bool close_data_spool_file(DCR *dcr, bool end_of_spool); static bool despool_data(DCR *dcr, bool commit); static int read_block_from_spool_file(DCR *dcr); static bool open_attr_spool_file(JCR *jcr, BSOCK *bs); @@ -108,6 +109,7 @@ bool begin_data_spool(DCR *dcr) V(mutex); } } + return status; } @@ -115,8 +117,9 @@ bool discard_data_spool(DCR *dcr) { if (dcr->spooling) { Dmsg0(100, "Data spooling discarded\n"); - return close_data_spool_file(dcr); + return close_data_spool_file(dcr, true); } + return true; } @@ -129,31 +132,33 @@ bool commit_data_spool(DCR *dcr) status = despool_data(dcr, true /*commit*/); if (!status) { Dmsg1(100, _("Bad return from despool WroteVol=%d\n"), dcr->WroteVol); - close_data_spool_file(dcr); + close_data_spool_file(dcr, true); return false; } - return close_data_spool_file(dcr); + return close_data_spool_file(dcr, true); } + return true; } static void make_unique_data_spool_filename(DCR *dcr, POOLMEM **name) { const char *dir; + if (dcr->dev->device->spool_directory) { dir = dcr->dev->device->spool_directory; } else { dir = working_directory; } - Mmsg(name, "%s/%s.data.%u.%s.%s.spool", dir, my_name, dcr->jcr->JobId, - dcr->jcr->Job, dcr->device->name()); -} + Mmsg(name, "%s/%s.data.%u.%s.%s.spool", dir, my_name, + dcr->jcr->JobId, dcr->jcr->Job, dcr->device->name()); +} static bool open_data_spool_file(DCR *dcr) { - POOLMEM *name = get_pool_memory(PM_MESSAGE); int spool_fd; + POOLMEM *name = get_pool_memory(PM_MESSAGE); make_unique_data_spool_filename(dcr, &name); if ((spool_fd = open(name, O_CREAT | O_TRUNC | O_RDWR | O_BINARY, 0640)) >= 0) { @@ -168,16 +173,19 @@ static bool open_data_spool_file(DCR *dcr) } Dmsg1(100, "Created spool file: %s\n", name); free_pool_memory(name); + return true; } -static bool close_data_spool_file(DCR *dcr) +static bool close_data_spool_file(DCR *dcr, bool end_of_spool) { - POOLMEM *name = get_pool_memory(PM_MESSAGE); + POOLMEM *name = get_pool_memory(PM_MESSAGE); P(mutex); spool_stats.data_jobs--; - spool_stats.total_data_jobs++; + if (end_of_spool) { + spool_stats.total_data_jobs++; + } if (spool_stats.data_size < dcr->job_spool_size) { spool_stats.data_size = 0; } else { @@ -193,9 +201,10 @@ static bool close_data_spool_file(DCR *dcr) close(dcr->spool_fd); dcr->spool_fd = -1; dcr->spooling = false; - unlink(name); + secure_erase(dcr->jcr, name); Dmsg1(100, "Deleted spool file: %s\n", name); free_pool_memory(name); + return true; } @@ -333,47 +342,62 @@ static bool despool_data(DCR *dcr, bool commit) } Jmsg(jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"), - despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60, - edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1)); + despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60, + edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1)); dcr->block = block; /* reset block */ - lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */ - if (ftruncate(rdcr->spool_fd, 0) != 0) { - berrno be; + /* + * See if we are using secure erase. + */ + if (me->secure_erase_cmdline) { + close_data_spool_file(dcr, false); + begin_data_spool(dcr); + } else { + lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */ + if (ftruncate(rdcr->spool_fd, 0) != 0) { + berrno be; - Jmsg(jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), be.bstrerror()); - /* Note, try continuing despite ftruncate problem */ - } + Jmsg(jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), be.bstrerror()); + /* + * Note, try continuing despite ftruncate problem + */ + } - P(mutex); - if (spool_stats.data_size < dcr->job_spool_size) { - spool_stats.data_size = 0; - } else { - spool_stats.data_size -= dcr->job_spool_size; + P(mutex); + if (spool_stats.data_size < dcr->job_spool_size) { + spool_stats.data_size = 0; + } else { + spool_stats.data_size -= dcr->job_spool_size; + } + V(mutex); + P(dcr->dev->spool_mutex); + dcr->dev->spool_size -= dcr->job_spool_size; + dcr->job_spool_size = 0; /* zap size in input dcr */ + V(dcr->dev->spool_mutex); } - V(mutex); - P(dcr->dev->spool_mutex); - dcr->dev->spool_size -= dcr->job_spool_size; - dcr->job_spool_size = 0; /* zap size in input dcr */ - V(dcr->dev->spool_mutex); + free_memory(rdev->dev_name); free_pool_memory(rdev->errmsg); - /* Be careful to NULL the jcr and free rdev after free_dcr() */ + + /* + * Be careful to NULL the jcr and free rdev after free_dcr() + */ rdcr->jcr = NULL; rdcr->set_dev(NULL); free_dcr(rdcr); free(rdev); dcr->spooling = true; /* turn on spooling again */ dcr->despooling = false; + /* - * Note, if committing we leave the device blocked. It will be removed in - * release_device(); + * Note, if committing we leave the device blocked. It will be removed in release_device(); */ if (!commit) { dcr->dev->dunblock(); } jcr->sendJobStatus(JS_Running); + return ok; } @@ -567,7 +591,9 @@ static bool write_spool_data(DCR *dcr) DEV_BLOCK *block = dcr->block; JCR *jcr = dcr->jcr; - /* Write data */ + /* + * Write data + */ for (int retry = 0; retry <= 1; retry++) { status = write(dcr->spool_fd, block->buf, (size_t)block->binbuf); if (status == -1) { @@ -593,25 +619,29 @@ static bool write_spool_data(DCR *dcr) /* Note, try continuing despite ftruncate problem */ } } + if (!despool_data(dcr, false)) { Jmsg(jcr, M_FATAL, 0, _("Fatal despooling error.")); jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ return false; } + if (!write_spool_header(dcr)) { return false; } + continue; /* try again */ } + return true; } + Jmsg(jcr, M_FATAL, 0, _("Retrying after data spooling error failed.\n")); jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */ + return false; } - - bool are_attributes_spooled(JCR *jcr) { return jcr->spool_attributes && jcr->dir_bsock->m_spool_fd != -1; @@ -666,8 +696,11 @@ static void make_unique_spool_filename(JCR *jcr, POOLMEM **name, int fd) */ static bool blast_attr_spool_file(JCR *jcr, boffset_t size) { - /* send full spool file name */ - POOLMEM *name = get_pool_memory(PM_MESSAGE); + POOLMEM *name = get_pool_memory(PM_MESSAGE); + + /* + * Send full spool file name + */ make_unique_spool_filename(jcr, &name, jcr->dir_bsock->m_fd); bash_spaces(name); jcr->dir_bsock->fsend("BlastAttr Job=%s File=%s\n", jcr->Job, name); @@ -682,6 +715,7 @@ static bool blast_attr_spool_file(JCR *jcr, boffset_t size) if (!bstrcmp(jcr->dir_bsock->msg, "1000 OK BlastAttr\n")) { return false; } + return true; } @@ -759,7 +793,7 @@ bool commit_attribute_spool(JCR *jcr) static bool open_attr_spool_file(JCR *jcr, BSOCK *bs) { - POOLMEM *name = get_pool_memory(PM_MESSAGE); + POOLMEM *name = get_pool_memory(PM_MESSAGE); make_unique_spool_filename(jcr, &name, bs->m_fd); bs->m_spool_fd = open(name, O_CREAT | O_TRUNC | O_RDWR | O_BINARY, 0640); @@ -800,7 +834,7 @@ static bool close_attr_spool_file(JCR *jcr, BSOCK *bs) make_unique_spool_filename(jcr, &name, bs->m_fd); close(bs->m_spool_fd); - unlink(name); + secure_erase(jcr, name); free_pool_memory(name); bs->m_spool_fd = -1; bs->clear_spooling(); diff --git a/src/stored/status.c b/src/stored/status.c index 1c8955a9502..877e1f64f02 100644 --- a/src/stored/status.c +++ b/src/stored/status.c @@ -434,6 +434,12 @@ static void list_status_header(STATUS_PKT *sp) edit_uint64_with_commas(me->max_bandwidth_per_job / 1024, b1)); sendit(msg, len, sp); + + if (me->secure_erase_cmdline) { + len = Mmsg(msg, _(" secure erase command='%s'\n"), me->secure_erase_cmdline); + sendit(msg, len, sp); + } + len = list_sd_plugins(msg); if (len > 0) { sendit(msg.c_str(), len, sp); diff --git a/src/stored/stored.c b/src/stored/stored.c index 07529521667..84dc468fe6e 100644 --- a/src/stored/stored.c +++ b/src/stored/stored.c @@ -517,6 +517,9 @@ static int check_resources() close_msg(NULL); /* close temp message handler */ init_msg(NULL, me->messages); /* open daemon message handler */ set_working_directory(me->working_directory); + if (me->secure_erase_cmdline) { + set_secure_erase_cmdline(me->secure_erase_cmdline); + } } return OK; @@ -587,7 +590,7 @@ static void cleanup_old_files() pm_strcpy(cleanup, basename); pm_strcat(cleanup, result->d_name); Dmsg1(500, "Unlink: %s\n", cleanup); - unlink(cleanup); + secure_erase(NULL, cleanup); } } free(entry); diff --git a/src/stored/stored_conf.c b/src/stored/stored_conf.c index 24913d31ed0..e47def582c2 100644 --- a/src/stored/stored_conf.c +++ b/src/stored/stored_conf.c @@ -108,6 +108,7 @@ static RES_ITEM store_items[] = { { "StatisticsCollectInterval", CFG_TYPE_PINT32, ITEM(res_store.stats_collect_interval), 0, CFG_ITEM_DEFAULT, "30", NULL, NULL }, { "DeviceReserveByMediaType", CFG_TYPE_BOOL, ITEM(res_store.device_reserve_by_mediatype), 0, CFG_ITEM_DEFAULT, "false", NULL, NULL }, { "FileDeviceConcurrentRead", CFG_TYPE_BOOL, ITEM(res_store.filedevice_concurrent_read), 0, CFG_ITEM_DEFAULT, "false", NULL, NULL }, + { "SecureEraseCommand", CFG_TYPE_STR, ITEM(res_store.secure_erase_cmdline), 0, 0, NULL, NULL, NULL }, { NULL, 0, { 0 }, 0, 0, NULL, NULL, NULL } }; @@ -629,6 +630,9 @@ void free_resource(RES *sres, int type) if (res->res_store.verid) { free(res->res_store.verid); } + if (res->res_store.secure_erase_cmdline) { + free(res->res_store.secure_erase_cmdline); + } break; case R_DEVICE: if (res->res_dev.media_type) { diff --git a/src/stored/stored_conf.h b/src/stored/stored_conf.h index 4276676e1be..63f36452287 100644 --- a/src/stored/stored_conf.h +++ b/src/stored/stored_conf.h @@ -123,6 +123,7 @@ class STORES : public BRSRES { char *tls_cipherlist; /* TLS Cipher List */ alist *tls_allowed_cns; /* TLS Allowed Clients */ char *verid; /* Custom Id to print in version command */ + char *secure_erase_cmdline; /* Cmdline to execute to perform secure erase of file */ uint64_t max_bandwidth_per_job; /* Bandwidth limitation (global) */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ diff --git a/src/win32/stored/backends/win32_file_device.c b/src/win32/stored/backends/win32_file_device.c index 0842201be55..cd97c3b93dd 100644 --- a/src/win32/stored/backends/win32_file_device.c +++ b/src/win32/stored/backends/win32_file_device.c @@ -259,7 +259,7 @@ bool win32_file_device::d_truncate(DCR *dcr) * Close file and blow it away */ ::close(m_fd); - ::unlink(archive_name.c_str()); + secure_erase(dcr->jcr, archive_name.c_str()); /* * Recreate the file -- of course, empty